diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:18:12 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:18:12 +0200 |
| commit | 35400f3a6b3b38d86eba4ab99a2984f584b4c961 (patch) | |
| tree | 1c49a65ddd6a9cef5d804976798303f13fe4c339 | |
| parent | ee07842fa0951a9a171ebd5e1011f25e2f8bf5fc (diff) | |
| download | ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.tar.gz ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.zip | |
tokenizer: full-Unicode punctuation and control classification
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | tools/gen_wordpiece_unicode.py | 93 | ||||
| -rw-r--r-- | wordpiece_tokenizer.c | 77 | ||||
| -rw-r--r-- | wordpiece_unicode.h | 73 |
4 files changed, 198 insertions, 47 deletions
@@ -400,7 +400,7 @@ qwen_kernels_avx.o: qwen_kernels_avx.c qwen_kernels_impl.h qwen_tokenizer.o: qwen_tokenizer.c qwen_tokenizer.h qwen_kernels.h $(CC) $(CFLAGS) -c -o $@ $< -wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h +wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h wordpiece_unicode.h $(CC) $(CFLAGS) -c -o $@ $< qwen_safetensors.o: qwen_safetensors.c qwen_safetensors.h diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py new file mode 100644 index 0000000..3eb5cc0 --- /dev/null +++ b/tools/gen_wordpiece_unicode.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Generate wordpiece_unicode.h: Unicode range tables for the WordPiece +BasicTokenizer's punctuation and control classification, matching transformers' +BertTokenizer._is_punctuation and _is_control exactly. + +Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint +whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus +tab/newline/carriage-return (which route to whitespace). The tables are emitted +as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. + +Run when bumping the target Unicode version (the C side is data-only otherwise): + + uv run python tools/gen_wordpiece_unicode.py > wordpiece_unicode.h +""" + +import unicodedata + +MAX_CP = 0x110000 + + +def merge(cps): + """Collapse a sorted codepoint list into contiguous [lo, hi] ranges.""" + ranges = [] + for cp in cps: + if ranges and cp == ranges[-1][1] + 1: + ranges[-1][1] = cp + else: + ranges.append([cp, cp]) + return ranges + + +def is_punct(cp): + if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126): + return True + return unicodedata.category(chr(cp)).startswith("P") + + +def is_control(cp): + if cp in (0x09, 0x0A, 0x0D): + return False + # Cc/Cf/Co are the assigned control, format and private-use chars. Cn + # (unassigned) is intentionally excluded: it never occurs in real text and + # would bind the table to a single Unicode version. + return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co") + + +def emit(name, ranges): + out = [f"static const wp_range_t {name}[] = {{"] + line = " " + for lo, hi in ranges: + s = "{0x%X, 0x%X}, " % (lo, hi) + if len(line) + len(s) > 96: + out.append(line.rstrip()) + line = " " + line += s + if line.strip(): + out.append(line.rstrip()) + out.append("};") + out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};") + return "\n".join(out) + + +def main(): + punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)]) + control = merge([cp for cp in range(MAX_CP) if is_control(cp)]) + print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata + * (Unicode {unicodedata.unidata_version}). Do not edit by hand. + * + * Range tables for the BasicTokenizer's _is_punctuation and _is_control, + * matching transformers' BertTokenizer. Punctuation is the ASCII set + * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is + * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to + * whitespace). Unassigned (Cn) codepoints are intentionally not control: they + * never occur in real text and would bind the table to one Unicode version. */ +#ifndef WORDPIECE_UNICODE_H +#define WORDPIECE_UNICODE_H + +#include <stdint.h> + +typedef struct {{ + uint32_t lo; + uint32_t hi; +}} wp_range_t; + +{emit("WP_PUNCT_RANGES", punct)} + +{emit("WP_CONTROL_RANGES", control)} + +#endif /* WORDPIECE_UNICODE_H */""") + + +if __name__ == "__main__": + main() diff --git a/wordpiece_tokenizer.c b/wordpiece_tokenizer.c index 6119c0d..1142167 100644 --- a/wordpiece_tokenizer.c +++ b/wordpiece_tokenizer.c @@ -7,16 +7,19 @@ * WordPiece: greedy longest-match-first against vocab.txt, continuation * pieces marked "##", whole token -> [UNK] when any piece fails to match. * - * Unicode coverage is scoped to the English/Western-European + CJK-split path - * that the first target encoders (bge-large-en, all-MiniLM-L6-v2) need: ASCII - * and Latin-1 Supplement casing/accents, the BERT CJK ranges, and the common - * punctuation/whitespace/control classes. Latin Extended beyond Latin-1, full - * Unicode category tables, and full NFD normalization are deliberate follow-ups - * (they matter for multilingual models, which mostly use SentencePiece anyway). + * Punctuation, whitespace and control classification is full-Unicode and exact + * against transformers: punctuation and control use generated category tables + * (wordpiece_unicode.h), whitespace is the complete Zs set, and the CJK split + * uses the BERT ideograph ranges. This makes cased multilingual encoders (e.g. + * bert-base-multilingual-cased) match Hugging Face. Casing and accent stripping + * still cover only ASCII + Latin-1 (wp_lower_deaccent), so do_lower_case + * multilingual checkpoints need full Unicode lowercase + NFD - a follow-up. */ #include "wordpiece_tokenizer.h" +#include "wordpiece_unicode.h" + #include <stdint.h> #include <stdio.h> #include <stdlib.h> @@ -144,51 +147,33 @@ static int wp_is_whitespace(uint32_t cp) { cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000; } +/* Binary search a sorted, non-overlapping [lo, hi] range table. */ +static int wp_in_ranges(uint32_t cp, const wp_range_t *r, int n) { + int lo = 0, hi = n - 1; + while (lo <= hi) { + int mid = (lo + hi) >> 1; + if (cp < r[mid].lo) + hi = mid - 1; + else if (cp > r[mid].hi) + lo = mid + 1; + else + return 1; + } + return 0; +} + +/* transformers' _is_control: tab/newline/carriage-return are whitespace, not + * control (they fall through to wp_is_whitespace); everything else is Unicode + * category Cc, Cf, or Co. */ static int wp_is_control(uint32_t cp) { if (cp == '\t' || cp == '\n' || cp == '\r') return 0; - if (cp < 0x20 || cp == 0x7F || (cp >= 0x80 && cp <= 0x9F)) - return 1; /* category Cc */ - return cp == 0x00AD || (cp >= 0x200B && cp <= 0x200F) || (cp >= 0x202A && cp <= 0x202E) || - (cp >= 0x2060 && cp <= 0x2064) || cp == 0xFEFF; /* common Cf */ + return wp_in_ranges(cp, WP_CONTROL_RANGES, WP_CONTROL_RANGES_N); } -static int wp_is_punct(uint32_t cp) { - if ((cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64) || (cp >= 91 && cp <= 96) || - (cp >= 123 && cp <= 126)) - return 1; /* all ASCII non-alnum, matching BERT */ - switch (cp) { - case 0x00A1: - case 0x00BF: - case 0x2010: - case 0x2011: - case 0x2012: - case 0x2013: - case 0x2014: - case 0x2015: - case 0x2018: - case 0x2019: - case 0x201A: - case 0x201B: - case 0x201C: - case 0x201D: - case 0x201E: - case 0x201F: - case 0x2026: - case 0x2032: - case 0x2033: - case 0x3001: - case 0x3002: - case 0xFF01: - case 0xFF0C: - case 0xFF1A: - case 0xFF1B: - case 0xFF1F: - return 1; - default: - return 0; - } -} +/* transformers' _is_punctuation: the ASCII non-alphanumeric set plus every + * Unicode category P* codepoint, both folded into the generated table. */ +static int wp_is_punct(uint32_t cp) { return wp_in_ranges(cp, WP_PUNCT_RANGES, WP_PUNCT_RANGES_N); } static int wp_is_cjk(uint32_t cp) { return (cp >= 0x4E00 && cp <= 0x9FFF) || (cp >= 0x3400 && cp <= 0x4DBF) || diff --git a/wordpiece_unicode.h b/wordpiece_unicode.h new file mode 100644 index 0000000..0c96c20 --- /dev/null +++ b/wordpiece_unicode.h @@ -0,0 +1,73 @@ +/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata + * (Unicode 13.0.0). Do not edit by hand. + * + * Range tables for the BasicTokenizer's _is_punctuation and _is_control, + * matching transformers' BertTokenizer. Punctuation is the ASCII set + * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is + * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to + * whitespace). Unassigned (Cn) codepoints are intentionally not control: they + * never occur in real text and would bind the table to one Unicode version. */ +#ifndef WORDPIECE_UNICODE_H +#define WORDPIECE_UNICODE_H + +#include <stdint.h> + +typedef struct { + uint32_t lo; + uint32_t hi; +} wp_range_t; + +static const wp_range_t WP_PUNCT_RANGES[] = { + {0x21, 0x2F}, {0x3A, 0x40}, {0x5B, 0x60}, {0x7B, 0x7E}, {0xA1, 0xA1}, {0xA7, 0xA7}, + {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387}, + {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3}, + {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D}, {0x61B, 0x61B}, + {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9}, + {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD}, + {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4}, + {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D}, + {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB}, + {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED}, + {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, {0x1944, 0x1945}, + {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF}, + {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027}, + {0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E}, + {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF}, + {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF}, + {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003}, + {0x3008, 0x3011}, {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0}, + {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E}, + {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC}, + {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F}, + {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19}, + {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B}, + {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20}, + {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65}, + {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F}, + {0x10857, 0x10857}, {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58}, + {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C}, + {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC}, + {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8}, + {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D}, + {0x112A9, 0x112A9}, {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D}, + {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C}, + {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2}, + {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45}, + {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474}, + {0x16A6E, 0x16A6F}, {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44}, + {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B}, + {0x1E95E, 0x1E95F}, +}; +enum { WP_PUNCT_RANGES_N = (int)(sizeof(WP_PUNCT_RANGES) / sizeof(WP_PUNCT_RANGES[0])) }; + +static const wp_range_t WP_CONTROL_RANGES[] = { + {0x0, 0x8}, {0xB, 0xC}, {0xE, 0x1F}, {0x7F, 0x9F}, {0xAD, 0xAD}, {0x600, 0x605}, + {0x61C, 0x61C}, {0x6DD, 0x6DD}, {0x70F, 0x70F}, {0x8E2, 0x8E2}, {0x180E, 0x180E}, + {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x2064}, {0x2066, 0x206F}, {0xE000, 0xF8FF}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0x110BD, 0x110BD}, {0x110CD, 0x110CD}, + {0x13430, 0x13438}, {0x1BCA0, 0x1BCA3}, {0x1D173, 0x1D17A}, {0xE0001, 0xE0001}, + {0xE0020, 0xE007F}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}, +}; +enum { WP_CONTROL_RANGES_N = (int)(sizeof(WP_CONTROL_RANGES) / sizeof(WP_CONTROL_RANGES[0])) }; + +#endif /* WORDPIECE_UNICODE_H */ |
