aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:18:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:18:12 +0200
commit35400f3a6b3b38d86eba4ab99a2984f584b4c961 (patch)
tree1c49a65ddd6a9cef5d804976798303f13fe4c339 /tools
parentee07842fa0951a9a171ebd5e1011f25e2f8bf5fc (diff)
downloadffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.tar.gz
ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.zip
tokenizer: full-Unicode punctuation and control classification
Diffstat (limited to 'tools')
-rw-r--r--tools/gen_wordpiece_unicode.py93
1 files changed, 93 insertions, 0 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py
new file mode 100644
index 0000000..3eb5cc0
--- /dev/null
+++ b/tools/gen_wordpiece_unicode.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Generate wordpiece_unicode.h: Unicode range tables for the WordPiece
+BasicTokenizer's punctuation and control classification, matching transformers'
+BertTokenizer._is_punctuation and _is_control exactly.
+
+Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint
+whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus
+tab/newline/carriage-return (which route to whitespace). The tables are emitted
+as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
+
+Run when bumping the target Unicode version (the C side is data-only otherwise):
+
+ uv run python tools/gen_wordpiece_unicode.py > wordpiece_unicode.h
+"""
+
+import unicodedata
+
+MAX_CP = 0x110000
+
+
+def merge(cps):
+ """Collapse a sorted codepoint list into contiguous [lo, hi] ranges."""
+ ranges = []
+ for cp in cps:
+ if ranges and cp == ranges[-1][1] + 1:
+ ranges[-1][1] = cp
+ else:
+ ranges.append([cp, cp])
+ return ranges
+
+
+def is_punct(cp):
+ if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126):
+ return True
+ return unicodedata.category(chr(cp)).startswith("P")
+
+
+def is_control(cp):
+ if cp in (0x09, 0x0A, 0x0D):
+ return False
+ # Cc/Cf/Co are the assigned control, format and private-use chars. Cn
+ # (unassigned) is intentionally excluded: it never occurs in real text and
+ # would bind the table to a single Unicode version.
+ return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co")
+
+
+def emit(name, ranges):
+ out = [f"static const wp_range_t {name}[] = {{"]
+ line = " "
+ for lo, hi in ranges:
+ s = "{0x%X, 0x%X}, " % (lo, hi)
+ if len(line) + len(s) > 96:
+ out.append(line.rstrip())
+ line = " "
+ line += s
+ if line.strip():
+ out.append(line.rstrip())
+ out.append("};")
+ out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};")
+ return "\n".join(out)
+
+
+def main():
+ punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)])
+ control = merge([cp for cp in range(MAX_CP) if is_control(cp)])
+ print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
+ * (Unicode {unicodedata.unidata_version}). Do not edit by hand.
+ *
+ * Range tables for the BasicTokenizer's _is_punctuation and _is_control,
+ * matching transformers' BertTokenizer. Punctuation is the ASCII set
+ * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is
+ * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to
+ * whitespace). Unassigned (Cn) codepoints are intentionally not control: they
+ * never occur in real text and would bind the table to one Unicode version. */
+#ifndef WORDPIECE_UNICODE_H
+#define WORDPIECE_UNICODE_H
+
+#include <stdint.h>
+
+typedef struct {{
+ uint32_t lo;
+ uint32_t hi;
+}} wp_range_t;
+
+{emit("WP_PUNCT_RANGES", punct)}
+
+{emit("WP_CONTROL_RANGES", control)}
+
+#endif /* WORDPIECE_UNICODE_H */""")
+
+
+if __name__ == "__main__":
+ main()