diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:27:41 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:27:41 +0200 |
| commit | 187e5e8ad924cde6436ed799a0b8f3a7e0ed2efd (patch) | |
| tree | 82aa04a609acab612585aa1b41591e8f20b742c1 /tools | |
| parent | 35400f3a6b3b38d86eba4ab99a2984f584b4c961 (diff) | |
| download | ffwd-187e5e8ad924cde6436ed799a0b8f3a7e0ed2efd.tar.gz ffwd-187e5e8ad924cde6436ed799a0b8f3a7e0ed2efd.zip | |
tokenizer: Unicode case-fold and accent strip for multilingual WordPiece
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/gen_wordpiece_unicode.py | 58 |
1 files changed, 52 insertions, 6 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py index 3eb5cc0..24d56c9 100644 --- a/tools/gen_wordpiece_unicode.py +++ b/tools/gen_wordpiece_unicode.py @@ -44,6 +44,29 @@ def is_control(cp): return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co") +# Hangul syllables decompose by formula (the standard algorithm), so they are +# excluded from the fold table to keep it small; the C side handles them. +HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172 + + +def fold(cp): + """do_lower_case fold: Python str.lower then NFD then drop combining marks + (Mn), matching transformers' lowercase + _run_strip_accents. Returns the + output codepoint list (empty when the codepoint is a dropped mark).""" + s = chr(cp).lower() + return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"] + + +def emit_fold(entries): + out = ["static const wp_fold_t WP_FOLD[] = {"] + for cp, f in entries: + outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3]) + out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f))) + out.append("};") + out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };") + return "\n".join(out) + + def emit(name, ranges): out = [f"static const wp_range_t {name}[] = {{"] line = " " @@ -63,15 +86,30 @@ def emit(name, ranges): def main(): punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)]) control = merge([cp for cp in range(MAX_CP) if is_control(cp)]) + hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT) + fold_entries = [ + (cp, f) + for cp in range(MAX_CP) + if cp not in hangul + for f in [fold(cp)] + if f != [cp] + ] print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata * (Unicode {unicodedata.unidata_version}). Do not edit by hand. * - * Range tables for the BasicTokenizer's _is_punctuation and _is_control, - * matching transformers' BertTokenizer. Punctuation is the ASCII set - * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is - * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to - * whitespace). Unassigned (Cn) codepoints are intentionally not control: they - * never occur in real text and would bind the table to one Unicode version. */ + * Tables for the BasicTokenizer, matching transformers' BertTokenizer. + * + * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted + * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus + * every category P* codepoint. Control is categories Cc, Cf, Co minus + * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is + * intentionally not control: it never occurs in real text and would bind the + * table to one Unicode version. + * + * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks), + * sorted by codepoint. n is the output length (0 drops a combining mark; up to + * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and + * are excluded here. */ #ifndef WORDPIECE_UNICODE_H #define WORDPIECE_UNICODE_H @@ -82,10 +120,18 @@ typedef struct {{ uint32_t hi; }} wp_range_t; +typedef struct {{ + uint32_t cp; + uint32_t out[3]; + uint8_t n; +}} wp_fold_t; + {emit("WP_PUNCT_RANGES", punct)} {emit("WP_CONTROL_RANGES", control)} +{emit_fold(fold_entries)} + #endif /* WORDPIECE_UNICODE_H */""") |
