aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:18:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:18:12 +0200
commit35400f3a6b3b38d86eba4ab99a2984f584b4c961 (patch)
tree1c49a65ddd6a9cef5d804976798303f13fe4c339
parentee07842fa0951a9a171ebd5e1011f25e2f8bf5fc (diff)
downloadffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.tar.gz
ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.zip
tokenizer: full-Unicode punctuation and control classification
-rw-r--r--Makefile2
-rw-r--r--tools/gen_wordpiece_unicode.py93
-rw-r--r--wordpiece_tokenizer.c77
-rw-r--r--wordpiece_unicode.h73
4 files changed, 198 insertions, 47 deletions
diff --git a/Makefile b/Makefile
index 506fdb4..9de5f83 100644
--- a/Makefile
+++ b/Makefile
@@ -400,7 +400,7 @@ qwen_kernels_avx.o: qwen_kernels_avx.c qwen_kernels_impl.h
qwen_tokenizer.o: qwen_tokenizer.c qwen_tokenizer.h qwen_kernels.h
$(CC) $(CFLAGS) -c -o $@ $<
-wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h
+wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h wordpiece_unicode.h
$(CC) $(CFLAGS) -c -o $@ $<
qwen_safetensors.o: qwen_safetensors.c qwen_safetensors.h
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py
new file mode 100644
index 0000000..3eb5cc0
--- /dev/null
+++ b/tools/gen_wordpiece_unicode.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Generate wordpiece_unicode.h: Unicode range tables for the WordPiece
+BasicTokenizer's punctuation and control classification, matching transformers'
+BertTokenizer._is_punctuation and _is_control exactly.
+
+Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint
+whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus
+tab/newline/carriage-return (which route to whitespace). The tables are emitted
+as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
+
+Run when bumping the target Unicode version (the C side is data-only otherwise):
+
+ uv run python tools/gen_wordpiece_unicode.py > wordpiece_unicode.h
+"""
+
+import unicodedata
+
+MAX_CP = 0x110000
+
+
+def merge(cps):
+ """Collapse a sorted codepoint list into contiguous [lo, hi] ranges."""
+ ranges = []
+ for cp in cps:
+ if ranges and cp == ranges[-1][1] + 1:
+ ranges[-1][1] = cp
+ else:
+ ranges.append([cp, cp])
+ return ranges
+
+
+def is_punct(cp):
+ if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126):
+ return True
+ return unicodedata.category(chr(cp)).startswith("P")
+
+
+def is_control(cp):
+ if cp in (0x09, 0x0A, 0x0D):
+ return False
+ # Cc/Cf/Co are the assigned control, format and private-use chars. Cn
+ # (unassigned) is intentionally excluded: it never occurs in real text and
+ # would bind the table to a single Unicode version.
+ return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co")
+
+
+def emit(name, ranges):
+ out = [f"static const wp_range_t {name}[] = {{"]
+ line = " "
+ for lo, hi in ranges:
+ s = "{0x%X, 0x%X}, " % (lo, hi)
+ if len(line) + len(s) > 96:
+ out.append(line.rstrip())
+ line = " "
+ line += s
+ if line.strip():
+ out.append(line.rstrip())
+ out.append("};")
+ out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};")
+ return "\n".join(out)
+
+
+def main():
+ punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)])
+ control = merge([cp for cp in range(MAX_CP) if is_control(cp)])
+ print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
+ * (Unicode {unicodedata.unidata_version}). Do not edit by hand.
+ *
+ * Range tables for the BasicTokenizer's _is_punctuation and _is_control,
+ * matching transformers' BertTokenizer. Punctuation is the ASCII set
+ * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is
+ * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to
+ * whitespace). Unassigned (Cn) codepoints are intentionally not control: they
+ * never occur in real text and would bind the table to one Unicode version. */
+#ifndef WORDPIECE_UNICODE_H
+#define WORDPIECE_UNICODE_H
+
+#include <stdint.h>
+
+typedef struct {{
+ uint32_t lo;
+ uint32_t hi;
+}} wp_range_t;
+
+{emit("WP_PUNCT_RANGES", punct)}
+
+{emit("WP_CONTROL_RANGES", control)}
+
+#endif /* WORDPIECE_UNICODE_H */""")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/wordpiece_tokenizer.c b/wordpiece_tokenizer.c
index 6119c0d..1142167 100644
--- a/wordpiece_tokenizer.c
+++ b/wordpiece_tokenizer.c
@@ -7,16 +7,19 @@
* WordPiece: greedy longest-match-first against vocab.txt, continuation
* pieces marked "##", whole token -> [UNK] when any piece fails to match.
*
- * Unicode coverage is scoped to the English/Western-European + CJK-split path
- * that the first target encoders (bge-large-en, all-MiniLM-L6-v2) need: ASCII
- * and Latin-1 Supplement casing/accents, the BERT CJK ranges, and the common
- * punctuation/whitespace/control classes. Latin Extended beyond Latin-1, full
- * Unicode category tables, and full NFD normalization are deliberate follow-ups
- * (they matter for multilingual models, which mostly use SentencePiece anyway).
+ * Punctuation, whitespace and control classification is full-Unicode and exact
+ * against transformers: punctuation and control use generated category tables
+ * (wordpiece_unicode.h), whitespace is the complete Zs set, and the CJK split
+ * uses the BERT ideograph ranges. This makes cased multilingual encoders (e.g.
+ * bert-base-multilingual-cased) match Hugging Face. Casing and accent stripping
+ * still cover only ASCII + Latin-1 (wp_lower_deaccent), so do_lower_case
+ * multilingual checkpoints need full Unicode lowercase + NFD - a follow-up.
*/
#include "wordpiece_tokenizer.h"
+#include "wordpiece_unicode.h"
+
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
@@ -144,51 +147,33 @@ static int wp_is_whitespace(uint32_t cp) {
cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000;
}
+/* Binary search a sorted, non-overlapping [lo, hi] range table. */
+static int wp_in_ranges(uint32_t cp, const wp_range_t *r, int n) {
+ int lo = 0, hi = n - 1;
+ while (lo <= hi) {
+ int mid = (lo + hi) >> 1;
+ if (cp < r[mid].lo)
+ hi = mid - 1;
+ else if (cp > r[mid].hi)
+ lo = mid + 1;
+ else
+ return 1;
+ }
+ return 0;
+}
+
+/* transformers' _is_control: tab/newline/carriage-return are whitespace, not
+ * control (they fall through to wp_is_whitespace); everything else is Unicode
+ * category Cc, Cf, or Co. */
static int wp_is_control(uint32_t cp) {
if (cp == '\t' || cp == '\n' || cp == '\r')
return 0;
- if (cp < 0x20 || cp == 0x7F || (cp >= 0x80 && cp <= 0x9F))
- return 1; /* category Cc */
- return cp == 0x00AD || (cp >= 0x200B && cp <= 0x200F) || (cp >= 0x202A && cp <= 0x202E) ||
- (cp >= 0x2060 && cp <= 0x2064) || cp == 0xFEFF; /* common Cf */
+ return wp_in_ranges(cp, WP_CONTROL_RANGES, WP_CONTROL_RANGES_N);
}
-static int wp_is_punct(uint32_t cp) {
- if ((cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64) || (cp >= 91 && cp <= 96) ||
- (cp >= 123 && cp <= 126))
- return 1; /* all ASCII non-alnum, matching BERT */
- switch (cp) {
- case 0x00A1:
- case 0x00BF:
- case 0x2010:
- case 0x2011:
- case 0x2012:
- case 0x2013:
- case 0x2014:
- case 0x2015:
- case 0x2018:
- case 0x2019:
- case 0x201A:
- case 0x201B:
- case 0x201C:
- case 0x201D:
- case 0x201E:
- case 0x201F:
- case 0x2026:
- case 0x2032:
- case 0x2033:
- case 0x3001:
- case 0x3002:
- case 0xFF01:
- case 0xFF0C:
- case 0xFF1A:
- case 0xFF1B:
- case 0xFF1F:
- return 1;
- default:
- return 0;
- }
-}
+/* transformers' _is_punctuation: the ASCII non-alphanumeric set plus every
+ * Unicode category P* codepoint, both folded into the generated table. */
+static int wp_is_punct(uint32_t cp) { return wp_in_ranges(cp, WP_PUNCT_RANGES, WP_PUNCT_RANGES_N); }
static int wp_is_cjk(uint32_t cp) {
return (cp >= 0x4E00 && cp <= 0x9FFF) || (cp >= 0x3400 && cp <= 0x4DBF) ||
diff --git a/wordpiece_unicode.h b/wordpiece_unicode.h
new file mode 100644
index 0000000..0c96c20
--- /dev/null
+++ b/wordpiece_unicode.h
@@ -0,0 +1,73 @@
+/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
+ * (Unicode 13.0.0). Do not edit by hand.
+ *
+ * Range tables for the BasicTokenizer's _is_punctuation and _is_control,
+ * matching transformers' BertTokenizer. Punctuation is the ASCII set
+ * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is
+ * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to
+ * whitespace). Unassigned (Cn) codepoints are intentionally not control: they
+ * never occur in real text and would bind the table to one Unicode version. */
+#ifndef WORDPIECE_UNICODE_H
+#define WORDPIECE_UNICODE_H
+
+#include <stdint.h>
+
+typedef struct {
+ uint32_t lo;
+ uint32_t hi;
+} wp_range_t;
+
+static const wp_range_t WP_PUNCT_RANGES[] = {
+ {0x21, 0x2F}, {0x3A, 0x40}, {0x5B, 0x60}, {0x7B, 0x7E}, {0xA1, 0xA1}, {0xA7, 0xA7},
+ {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387},
+ {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3},
+ {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D}, {0x61B, 0x61B},
+ {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9},
+ {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD},
+ {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4},
+ {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D},
+ {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB},
+ {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED},
+ {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, {0x1944, 0x1945},
+ {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF},
+ {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027},
+ {0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E},
+ {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF},
+ {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF},
+ {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003},
+ {0x3008, 0x3011}, {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0},
+ {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E},
+ {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC},
+ {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F},
+ {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19},
+ {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B},
+ {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20},
+ {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65},
+ {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F},
+ {0x10857, 0x10857}, {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58},
+ {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C},
+ {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC},
+ {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8},
+ {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D},
+ {0x112A9, 0x112A9}, {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D},
+ {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C},
+ {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2},
+ {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45},
+ {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474},
+ {0x16A6E, 0x16A6F}, {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44},
+ {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B},
+ {0x1E95E, 0x1E95F},
+};
+enum { WP_PUNCT_RANGES_N = (int)(sizeof(WP_PUNCT_RANGES) / sizeof(WP_PUNCT_RANGES[0])) };
+
+static const wp_range_t WP_CONTROL_RANGES[] = {
+ {0x0, 0x8}, {0xB, 0xC}, {0xE, 0x1F}, {0x7F, 0x9F}, {0xAD, 0xAD}, {0x600, 0x605},
+ {0x61C, 0x61C}, {0x6DD, 0x6DD}, {0x70F, 0x70F}, {0x8E2, 0x8E2}, {0x180E, 0x180E},
+ {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x2064}, {0x2066, 0x206F}, {0xE000, 0xF8FF},
+ {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0x110BD, 0x110BD}, {0x110CD, 0x110CD},
+ {0x13430, 0x13438}, {0x1BCA0, 0x1BCA3}, {0x1D173, 0x1D17A}, {0xE0001, 0xE0001},
+ {0xE0020, 0xE007F}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD},
+};
+enum { WP_CONTROL_RANGES_N = (int)(sizeof(WP_CONTROL_RANGES) / sizeof(WP_CONTROL_RANGES[0])) };
+
+#endif /* WORDPIECE_UNICODE_H */