aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:27:41 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:27:41 +0200
commit187e5e8ad924cde6436ed799a0b8f3a7e0ed2efd (patch)
tree82aa04a609acab612585aa1b41591e8f20b742c1 /tools
parent35400f3a6b3b38d86eba4ab99a2984f584b4c961 (diff)
downloadffwd-187e5e8ad924cde6436ed799a0b8f3a7e0ed2efd.tar.gz
ffwd-187e5e8ad924cde6436ed799a0b8f3a7e0ed2efd.zip
tokenizer: Unicode case-fold and accent strip for multilingual WordPiece
Diffstat (limited to 'tools')
-rw-r--r--tools/gen_wordpiece_unicode.py58
1 files changed, 52 insertions, 6 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py
index 3eb5cc0..24d56c9 100644
--- a/tools/gen_wordpiece_unicode.py
+++ b/tools/gen_wordpiece_unicode.py
@@ -44,6 +44,29 @@ def is_control(cp):
return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co")
+# Hangul syllables decompose by formula (the standard algorithm), so they are
+# excluded from the fold table to keep it small; the C side handles them.
+HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172
+
+
+def fold(cp):
+ """do_lower_case fold: Python str.lower then NFD then drop combining marks
+ (Mn), matching transformers' lowercase + _run_strip_accents. Returns the
+ output codepoint list (empty when the codepoint is a dropped mark)."""
+ s = chr(cp).lower()
+ return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"]
+
+
+def emit_fold(entries):
+ out = ["static const wp_fold_t WP_FOLD[] = {"]
+ for cp, f in entries:
+ outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3])
+ out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f)))
+ out.append("};")
+ out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };")
+ return "\n".join(out)
+
+
def emit(name, ranges):
out = [f"static const wp_range_t {name}[] = {{"]
line = " "
@@ -63,15 +86,30 @@ def emit(name, ranges):
def main():
punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)])
control = merge([cp for cp in range(MAX_CP) if is_control(cp)])
+ hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT)
+ fold_entries = [
+ (cp, f)
+ for cp in range(MAX_CP)
+ if cp not in hangul
+ for f in [fold(cp)]
+ if f != [cp]
+ ]
print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
* (Unicode {unicodedata.unidata_version}). Do not edit by hand.
*
- * Range tables for the BasicTokenizer's _is_punctuation and _is_control,
- * matching transformers' BertTokenizer. Punctuation is the ASCII set
- * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is
- * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to
- * whitespace). Unassigned (Cn) codepoints are intentionally not control: they
- * never occur in real text and would bind the table to one Unicode version. */
+ * Tables for the BasicTokenizer, matching transformers' BertTokenizer.
+ *
+ * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted
+ * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus
+ * every category P* codepoint. Control is categories Cc, Cf, Co minus
+ * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is
+ * intentionally not control: it never occurs in real text and would bind the
+ * table to one Unicode version.
+ *
+ * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks),
+ * sorted by codepoint. n is the output length (0 drops a combining mark; up to
+ * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and
+ * are excluded here. */
#ifndef WORDPIECE_UNICODE_H
#define WORDPIECE_UNICODE_H
@@ -82,10 +120,18 @@ typedef struct {{
uint32_t hi;
}} wp_range_t;
+typedef struct {{
+ uint32_t cp;
+ uint32_t out[3];
+ uint8_t n;
+}} wp_fold_t;
+
{emit("WP_PUNCT_RANGES", punct)}
{emit("WP_CONTROL_RANGES", control)}
+{emit_fold(fold_entries)}
+
#endif /* WORDPIECE_UNICODE_H */""")