diff options
| -rw-r--r-- | qwen_tokenizer.c | 28 | ||||
| -rw-r--r-- | tests/test_tokenizer.c | 32 |
2 files changed, 51 insertions, 9 deletions
diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c index 4a30b92..67dcaf7 100644 --- a/qwen_tokenizer.c +++ b/qwen_tokenizer.c @@ -496,12 +496,23 @@ static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, return 1; } -static int is_ascii_alpha_cp(int cp) { - return (cp >= 'A' && cp <= 'Z') || (cp >= 'a' && cp <= 'z'); -} +/* The pretokenizer classifies every codepoint, several times each, and the + * input is overwhelmingly ASCII; one table lookup replaces the range chains + * for cp < 0x80. Non-ASCII codepoints fall through to the full checks. + * ASCII has no symbolish codepoints, so that class needs no bit. */ +#define CPC_SPACE 1 +#define CPC_NUMBER 2 +#define CPC_LETTER 4 + +static const unsigned char ascii_cp_class[128] = { + ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE, + ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE, + ['0' ... '9'] = CPC_NUMBER, + ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER, +}; static int is_number_cp(int cp) { - if (cp >= '0' && cp <= '9') return 1; + if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_NUMBER; if (cp >= 0x0660 && cp <= 0x0669) return 1; if (cp >= 0x06F0 && cp <= 0x06F9) return 1; if (cp >= 0x0966 && cp <= 0x096F) return 1; @@ -510,8 +521,8 @@ static int is_number_cp(int cp) { } static int is_space_cp(int cp) { - return cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r' || - cp == '\f' || cp == '\v' || cp == 0x85 || cp == 0xA0 || + if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_SPACE; + return cp == 0x85 || cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000; @@ -640,9 +651,8 @@ static char *normalize_nfc_latin(const char *text) { } static int is_letter_cp(int cp) { - if (is_ascii_alpha_cp(cp)) return 1; - if (cp < 0x80 || is_space_cp(cp) || is_number_cp(cp) || - is_symbolish_cp(cp)) + if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_LETTER; + if (is_space_cp(cp) || is_number_cp(cp) || is_symbolish_cp(cp)) return 0; return 1; } diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index 19c007a..8c20308 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -138,6 +138,38 @@ static void run_cases(const qwen_tokenizer_t *tok, /* No applicable merges: raw byte tokens. */ const int dr[] = {'d', 'r'}; check_ids(tok, ws, "dr", dr, 2, label); + + /* Pretokenizer codepoint classes (the ASCII class-table paths). */ + + /* A letter run stops at a digit; digits split one per piece. */ + const int x123[] = {'x', '1', '2', '3'}; + check_ids(tok, ws, "x123", x123, 4, label); + + /* Punctuation is its own piece; " world" joins as one piece + * (Gworld -> Gwor, ld); trailing "!" is a symbol piece. */ + const int punct[] = {259, ',', 263, 262, '!'}; + check_ids(tok, ws, "hello, world!", punct, 5, label); + + /* Tab and newline are spaces; each stays a piece between letters. */ + const int tab[] = {'a', '\t', 'b'}; + check_ids(tok, ws, "a\tb", tab, 3, label); + const int nl[] = {'a', '\n', 'b'}; + check_ids(tok, ws, "a\nb", nl, 3, label); + + /* A space is not absorbed into a digit piece (digits are not letters). */ + const int sp_digit[] = {'7', ' ', '8'}; + check_ids(tok, ws, "7 8", sp_digit, 3, label); + + /* Contraction suffix splits off. */ + const int its[] = {'i', 't', '\'', 's'}; + check_ids(tok, ws, "it's", its, 4, label); + + /* Non-ASCII falls through the table: U+00E9 is a letter (byte pair); + * U+2026 is symbolish, so it prefixes the following letter run. */ + const int eacute[] = {0xC3, 0xA9}; + check_ids(tok, ws, "\xC3\xA9", eacute, 2, label); + const int ellipsis[] = {'a', 0xE2, 0x80, 0xA6, 'b'}; + check_ids(tok, ws, "a\xE2\x80\xA6" "b", ellipsis, 5, label); } int main(void) |
