aboutsummaryrefslogtreecommitdiff
path: root/tests/check_tokenizer_parity.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-27 00:58:05 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-27 01:03:41 +0200
commitaf55803d0dff9eb692c760c8d486e736c945061d (patch)
tree0875a1112bf98304e445e0544c4309670d6f4769 /tests/check_tokenizer_parity.py
parent0f861b7f3098750ddee4f6570f3be228d1da5320 (diff)
downloadffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz
ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip
chore: clean up comment formatting and wording
Diffstat (limited to 'tests/check_tokenizer_parity.py')
-rwxr-xr-xtests/check_tokenizer_parity.py10
1 files changed, 5 insertions, 5 deletions
diff --git a/tests/check_tokenizer_parity.py b/tests/check_tokenizer_parity.py
index acea177..70eea3c 100755
--- a/tests/check_tokenizer_parity.py
+++ b/tests/check_tokenizer_parity.py
@@ -1,11 +1,11 @@
#!/usr/bin/env python3
-"""Check the C BPE tokenizer against stored reference vectors.
+"""Check the BPE tokenizer against stored reference vectors.
The reference ids live in tests/test-vectors/bpe/expected.json, generated from
-the Qwen tokenizer by devtools/gen_test_vectors.py. This builds the C BPE driver
-(tests/tokenizer_dump), runs it against the checked-in vocab.json + merges.txt,
-and compares the ids exactly. It imports no transformers and downloads no model,
-so it is hermetic and fast.
+the Qwen tokenizer by devtools/gen_test_vectors.py.
+
+This builds the C BPE driver (tests/tokenizer_dump), runs it against the checked-in
+vocab.json + merges.txt, and compares the ids exactly.
tests/check_tokenizer_parity.py
tests/check_tokenizer_parity.py --cc clang