diff options
Diffstat (limited to 'tests/test_bpe.c')
| -rw-r--r-- | tests/test_bpe.c | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/tests/test_bpe.c b/tests/test_bpe.c index 1674b4e..02e5078 100644 --- a/tests/test_bpe.c +++ b/tests/test_bpe.c @@ -1,8 +1,9 @@ -/* tests/test_tokenizer.c - hermetic byte-level BPE tokenizer tests. +/* Hermetic byte-level BPE tokenizer tests. * Synthesizes a complete GPT-2-style vocab.json (all 256 byte tokens plus * merged tokens) and merges.txt, then checks exact token ids through the * integer-pair fast path, and that dropping the integer tables (forcing the - * string-based fallback) produces identical ids. Runs via `make test`. */ + * string-based fallback) produces identical ids. Runs via make test. + */ #include "bpe.h" #include "tok_fixture.h" |
