aboutsummaryrefslogtreecommitdiff
path: root/tests/test_tokenizer.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 12:33:13 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit2c9592042faf7229dc801f6eaee34636267aec4c (patch)
tree723f2cdfb4c44f5d37ea7fac7719a8de3d871588 /tests/test_tokenizer.c
parent50e29c9504093fc2734f9ca99266a0caa5b238cb (diff)
downloadffwd-2c9592042faf7229dc801f6eaee34636267aec4c.tar.gz
ffwd-2c9592042faf7229dc801f6eaee34636267aec4c.zip
tokenizer: ASCII class table for pretokenizer codepoint checks
Diffstat (limited to 'tests/test_tokenizer.c')
-rw-r--r--tests/test_tokenizer.c32
1 files changed, 32 insertions, 0 deletions
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c
index 19c007a..8c20308 100644
--- a/tests/test_tokenizer.c
+++ b/tests/test_tokenizer.c
@@ -138,6 +138,38 @@ static void run_cases(const qwen_tokenizer_t *tok,
/* No applicable merges: raw byte tokens. */
const int dr[] = {'d', 'r'};
check_ids(tok, ws, "dr", dr, 2, label);
+
+ /* Pretokenizer codepoint classes (the ASCII class-table paths). */
+
+ /* A letter run stops at a digit; digits split one per piece. */
+ const int x123[] = {'x', '1', '2', '3'};
+ check_ids(tok, ws, "x123", x123, 4, label);
+
+ /* Punctuation is its own piece; " world" joins as one piece
+ * (Gworld -> Gwor, ld); trailing "!" is a symbol piece. */
+ const int punct[] = {259, ',', 263, 262, '!'};
+ check_ids(tok, ws, "hello, world!", punct, 5, label);
+
+ /* Tab and newline are spaces; each stays a piece between letters. */
+ const int tab[] = {'a', '\t', 'b'};
+ check_ids(tok, ws, "a\tb", tab, 3, label);
+ const int nl[] = {'a', '\n', 'b'};
+ check_ids(tok, ws, "a\nb", nl, 3, label);
+
+ /* A space is not absorbed into a digit piece (digits are not letters). */
+ const int sp_digit[] = {'7', ' ', '8'};
+ check_ids(tok, ws, "7 8", sp_digit, 3, label);
+
+ /* Contraction suffix splits off. */
+ const int its[] = {'i', 't', '\'', 's'};
+ check_ids(tok, ws, "it's", its, 4, label);
+
+ /* Non-ASCII falls through the table: U+00E9 is a letter (byte pair);
+ * U+2026 is symbolish, so it prefixes the following letter run. */
+ const int eacute[] = {0xC3, 0xA9};
+ check_ids(tok, ws, "\xC3\xA9", eacute, 2, label);
+ const int ellipsis[] = {'a', 0xE2, 0x80, 0xA6, 'b'};
+ check_ids(tok, ws, "a\xE2\x80\xA6" "b", ellipsis, 5, label);
}
int main(void)