aboutsummaryrefslogtreecommitdiff
path: root/tools/gen_wordpiece_unicode.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:31:06 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitdbdf714bb57578431924e5d069089bdfd9f4c604 (patch)
tree4f5b59ef132220310e1e4b1dde64f5e38782369a /tools/gen_wordpiece_unicode.py
parente9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff)
downloadffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz
ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to 'tools/gen_wordpiece_unicode.py')
-rw-r--r--tools/gen_wordpiece_unicode.py139
1 files changed, 0 insertions, 139 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py
deleted file mode 100644
index 24649f1..0000000
--- a/tools/gen_wordpiece_unicode.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python3
-"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece
-BasicTokenizer's punctuation and control classification, matching transformers'
-BertTokenizer._is_punctuation and _is_control exactly.
-
-Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint
-whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus
-tab/newline/carriage-return (which route to whitespace). The tables are emitted
-as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
-
-Run when bumping the target Unicode version (the C side is data-only otherwise):
-
- uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h
-"""
-
-import unicodedata
-
-MAX_CP = 0x110000
-
-
-def merge(cps):
- """Collapse a sorted codepoint list into contiguous [lo, hi] ranges."""
- ranges = []
- for cp in cps:
- if ranges and cp == ranges[-1][1] + 1:
- ranges[-1][1] = cp
- else:
- ranges.append([cp, cp])
- return ranges
-
-
-def is_punct(cp):
- if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126):
- return True
- return unicodedata.category(chr(cp)).startswith("P")
-
-
-def is_control(cp):
- if cp in (0x09, 0x0A, 0x0D):
- return False
- # Cc/Cf/Co are the assigned control, format and private-use chars. Cn
- # (unassigned) is intentionally excluded: it never occurs in real text and
- # would bind the table to a single Unicode version.
- return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co")
-
-
-# Hangul syllables decompose by formula (the standard algorithm), so they are
-# excluded from the fold table to keep it small; the C side handles them.
-HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172
-
-
-def fold(cp):
- """do_lower_case fold: Python str.lower then NFD then drop combining marks
- (Mn), matching transformers' lowercase + _run_strip_accents. Returns the
- output codepoint list (empty when the codepoint is a dropped mark)."""
- s = chr(cp).lower()
- return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"]
-
-
-def emit_fold(entries):
- out = ["static const wp_fold_t WP_FOLD[] = {"]
- for cp, f in entries:
- outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3])
- out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f)))
- out.append("};")
- out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };")
- return "\n".join(out)
-
-
-def emit(name, ranges):
- out = [f"static const wp_range_t {name}[] = {{"]
- line = " "
- for lo, hi in ranges:
- s = "{0x%X, 0x%X}, " % (lo, hi)
- if len(line) + len(s) > 96:
- out.append(line.rstrip())
- line = " "
- line += s
- if line.strip():
- out.append(line.rstrip())
- out.append("};")
- out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};")
- return "\n".join(out)
-
-
-def main():
- punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)])
- control = merge([cp for cp in range(MAX_CP) if is_control(cp)])
- hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT)
- fold_entries = [
- (cp, f)
- for cp in range(MAX_CP)
- if cp not in hangul
- for f in [fold(cp)]
- if f != [cp]
- ]
- print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
- * (Unicode {unicodedata.unidata_version}). Do not edit by hand.
- *
- * Tables for the BasicTokenizer, matching transformers' BertTokenizer.
- *
- * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted
- * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus
- * every category P* codepoint. Control is categories Cc, Cf, Co minus
- * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is
- * intentionally not control: it never occurs in real text and would bind the
- * table to one Unicode version.
- *
- * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks),
- * sorted by codepoint. n is the output length (0 drops a combining mark; up to
- * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and
- * are excluded here. */
-#ifndef WORDPIECE_UNICODE_H
-#define WORDPIECE_UNICODE_H
-
-#include <stdint.h>
-
-typedef struct {{
- uint32_t lo;
- uint32_t hi;
-}} wp_range_t;
-
-typedef struct {{
- uint32_t cp;
- uint32_t out[3];
- uint8_t n;
-}} wp_fold_t;
-
-{emit("WP_PUNCT_RANGES", punct)}
-
-{emit("WP_CONTROL_RANGES", control)}
-
-{emit_fold(fold_entries)}
-
-#endif /* WORDPIECE_UNICODE_H */""")
-
-
-if __name__ == "__main__":
- main()