From af55803d0dff9eb692c760c8d486e736c945061d Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 27 Jun 2026 00:58:05 +0200 Subject: chore: clean up comment formatting and wording --- tests/tokenizer_dump.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'tests/tokenizer_dump.c') diff --git a/tests/tokenizer_dump.c b/tests/tokenizer_dump.c index f6e26aa..4e8e8ec 100644 --- a/tests/tokenizer_dump.c +++ b/tests/tokenizer_dump.c @@ -1,9 +1,14 @@ -/* Parity driver for the BPE tokenizer. Loads vocab.json (merges.txt is read - * from the same directory), encodes each argument through the three encode - * entry points (and cross-checks they agree), then prints " - * ..." per line. check_tokenizer_parity.py builds this with - * `make parity-tokenizer-driver` and compares the ids against the stored - * reference vectors in tests/test-vectors/bpe/. Links libtok. */ +/* BPE tokenizer parity driver. + * Loads vocab.json; merges.txt is read from the same directory. + * Encodes each argument through the three encode entry points and checks that + * they agree. + * + * Prints one line per argument: " ...". + * + * check_tokenizer_parity.py builds this with `make parity-tokenizer-driver` and + * compares ids against reference vectors in tests/test-vectors/bpe/. + */ + #include "bpe.h" #include -- cgit v1.3.1