From dbdf714bb57578431924e5d069089bdfd9f4c604 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 20 Jun 2026 20:31:06 +0200 Subject: devtools: add scripts for unicode generation, model inspection, embedding ranking, and more --- devtools/gen_wordpiece_unicode.py | 139 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100755 devtools/gen_wordpiece_unicode.py (limited to 'devtools/gen_wordpiece_unicode.py') diff --git a/devtools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py new file mode 100755 index 0000000..2f8642e --- /dev/null +++ b/devtools/gen_wordpiece_unicode.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece +BasicTokenizer's punctuation and control classification, matching transformers' +BertTokenizer._is_punctuation and _is_control exactly. + +Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint +whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus +tab/newline/carriage-return (which route to whitespace). The tables are emitted +as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. + +Run when bumping the target Unicode version (the C side is data-only otherwise): + + uv run python devtools/gen_wordpiece_unicode.py > unicode.h +""" + +import unicodedata + +MAX_CP = 0x110000 + + +def merge(cps): + """Collapse a sorted codepoint list into contiguous [lo, hi] ranges.""" + ranges = [] + for cp in cps: + if ranges and cp == ranges[-1][1] + 1: + ranges[-1][1] = cp + else: + ranges.append([cp, cp]) + return ranges + + +def is_punct(cp): + if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126): + return True + return unicodedata.category(chr(cp)).startswith("P") + + +def is_control(cp): + if cp in (0x09, 0x0A, 0x0D): + return False + # Cc/Cf/Co are the assigned control, format and private-use chars. Cn + # (unassigned) is intentionally excluded: it never occurs in real text and + # would bind the table to a single Unicode version. + return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co") + + +# Hangul syllables decompose by formula (the standard algorithm), so they are +# excluded from the fold table to keep it small; the C side handles them. +HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172 + + +def fold(cp): + """do_lower_case fold: Python str.lower then NFD then drop combining marks + (Mn), matching transformers' lowercase + _run_strip_accents. Returns the + output codepoint list (empty when the codepoint is a dropped mark).""" + s = chr(cp).lower() + return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"] + + +def emit_fold(entries): + out = ["static const wp_fold_t WP_FOLD[] = {"] + for cp, f in entries: + outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3]) + out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f))) + out.append("};") + out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };") + return "\n".join(out) + + +def emit(name, ranges): + out = [f"static const wp_range_t {name}[] = {{"] + line = " " + for lo, hi in ranges: + s = "{0x%X, 0x%X}, " % (lo, hi) + if len(line) + len(s) > 96: + out.append(line.rstrip()) + line = " " + line += s + if line.strip(): + out.append(line.rstrip()) + out.append("};") + out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};") + return "\n".join(out) + + +def main(): + punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)]) + control = merge([cp for cp in range(MAX_CP) if is_control(cp)]) + hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT) + fold_entries = [ + (cp, f) + for cp in range(MAX_CP) + if cp not in hangul + for f in [fold(cp)] + if f != [cp] + ] + print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata + * (Unicode {unicodedata.unidata_version}). Do not edit by hand. + * + * Tables for the BasicTokenizer, matching transformers' BertTokenizer. + * + * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted + * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus + * every category P* codepoint. Control is categories Cc, Cf, Co minus + * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is + * intentionally not control: it never occurs in real text and would bind the + * table to one Unicode version. + * + * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks), + * sorted by codepoint. n is the output length (0 drops a combining mark; up to + * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and + * are excluded here. */ +#ifndef WORDPIECE_UNICODE_H +#define WORDPIECE_UNICODE_H + +#include + +typedef struct {{ + uint32_t lo; + uint32_t hi; +}} wp_range_t; + +typedef struct {{ + uint32_t cp; + uint32_t out[3]; + uint8_t n; +}} wp_fold_t; + +{emit("WP_PUNCT_RANGES", punct)} + +{emit("WP_CONTROL_RANGES", control)} + +{emit_fold(fold_entries)} + +#endif /* WORDPIECE_UNICODE_H */""") + + +if __name__ == "__main__": + main() -- cgit v1.3.1