diff options
Diffstat (limited to 'tools/gen_wordpiece_unicode.py')
| -rw-r--r-- | tools/gen_wordpiece_unicode.py | 139 |
1 files changed, 0 insertions, 139 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py deleted file mode 100644 index 24649f1..0000000 --- a/tools/gen_wordpiece_unicode.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece -BasicTokenizer's punctuation and control classification, matching transformers' -BertTokenizer._is_punctuation and _is_control exactly. - -Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint -whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus -tab/newline/carriage-return (which route to whitespace). The tables are emitted -as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. - -Run when bumping the target Unicode version (the C side is data-only otherwise): - - uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h -""" - -import unicodedata - -MAX_CP = 0x110000 - - -def merge(cps): - """Collapse a sorted codepoint list into contiguous [lo, hi] ranges.""" - ranges = [] - for cp in cps: - if ranges and cp == ranges[-1][1] + 1: - ranges[-1][1] = cp - else: - ranges.append([cp, cp]) - return ranges - - -def is_punct(cp): - if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126): - return True - return unicodedata.category(chr(cp)).startswith("P") - - -def is_control(cp): - if cp in (0x09, 0x0A, 0x0D): - return False - # Cc/Cf/Co are the assigned control, format and private-use chars. Cn - # (unassigned) is intentionally excluded: it never occurs in real text and - # would bind the table to a single Unicode version. - return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co") - - -# Hangul syllables decompose by formula (the standard algorithm), so they are -# excluded from the fold table to keep it small; the C side handles them. -HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172 - - -def fold(cp): - """do_lower_case fold: Python str.lower then NFD then drop combining marks - (Mn), matching transformers' lowercase + _run_strip_accents. Returns the - output codepoint list (empty when the codepoint is a dropped mark).""" - s = chr(cp).lower() - return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"] - - -def emit_fold(entries): - out = ["static const wp_fold_t WP_FOLD[] = {"] - for cp, f in entries: - outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3]) - out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f))) - out.append("};") - out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };") - return "\n".join(out) - - -def emit(name, ranges): - out = [f"static const wp_range_t {name}[] = {{"] - line = " " - for lo, hi in ranges: - s = "{0x%X, 0x%X}, " % (lo, hi) - if len(line) + len(s) > 96: - out.append(line.rstrip()) - line = " " - line += s - if line.strip(): - out.append(line.rstrip()) - out.append("};") - out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};") - return "\n".join(out) - - -def main(): - punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)]) - control = merge([cp for cp in range(MAX_CP) if is_control(cp)]) - hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT) - fold_entries = [ - (cp, f) - for cp in range(MAX_CP) - if cp not in hangul - for f in [fold(cp)] - if f != [cp] - ] - print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata - * (Unicode {unicodedata.unidata_version}). Do not edit by hand. - * - * Tables for the BasicTokenizer, matching transformers' BertTokenizer. - * - * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted - * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus - * every category P* codepoint. Control is categories Cc, Cf, Co minus - * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is - * intentionally not control: it never occurs in real text and would bind the - * table to one Unicode version. - * - * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks), - * sorted by codepoint. n is the output length (0 drops a combining mark; up to - * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and - * are excluded here. */ -#ifndef WORDPIECE_UNICODE_H -#define WORDPIECE_UNICODE_H - -#include <stdint.h> - -typedef struct {{ - uint32_t lo; - uint32_t hi; -}} wp_range_t; - -typedef struct {{ - uint32_t cp; - uint32_t out[3]; - uint8_t n; -}} wp_fold_t; - -{emit("WP_PUNCT_RANGES", punct)} - -{emit("WP_CONTROL_RANGES", control)} - -{emit_fold(fold_entries)} - -#endif /* WORDPIECE_UNICODE_H */""") - - -if __name__ == "__main__": - main() |
