diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:18:12 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:18:12 +0200 |
| commit | 35400f3a6b3b38d86eba4ab99a2984f584b4c961 (patch) | |
| tree | 1c49a65ddd6a9cef5d804976798303f13fe4c339 /tools | |
| parent | ee07842fa0951a9a171ebd5e1011f25e2f8bf5fc (diff) | |
| download | ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.tar.gz ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.zip | |
tokenizer: full-Unicode punctuation and control classification
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/gen_wordpiece_unicode.py | 93 |
1 files changed, 93 insertions, 0 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py new file mode 100644 index 0000000..3eb5cc0 --- /dev/null +++ b/tools/gen_wordpiece_unicode.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Generate wordpiece_unicode.h: Unicode range tables for the WordPiece +BasicTokenizer's punctuation and control classification, matching transformers' +BertTokenizer._is_punctuation and _is_control exactly. + +Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint +whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus +tab/newline/carriage-return (which route to whitespace). The tables are emitted +as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. + +Run when bumping the target Unicode version (the C side is data-only otherwise): + + uv run python tools/gen_wordpiece_unicode.py > wordpiece_unicode.h +""" + +import unicodedata + +MAX_CP = 0x110000 + + +def merge(cps): + """Collapse a sorted codepoint list into contiguous [lo, hi] ranges.""" + ranges = [] + for cp in cps: + if ranges and cp == ranges[-1][1] + 1: + ranges[-1][1] = cp + else: + ranges.append([cp, cp]) + return ranges + + +def is_punct(cp): + if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126): + return True + return unicodedata.category(chr(cp)).startswith("P") + + +def is_control(cp): + if cp in (0x09, 0x0A, 0x0D): + return False + # Cc/Cf/Co are the assigned control, format and private-use chars. Cn + # (unassigned) is intentionally excluded: it never occurs in real text and + # would bind the table to a single Unicode version. + return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co") + + +def emit(name, ranges): + out = [f"static const wp_range_t {name}[] = {{"] + line = " " + for lo, hi in ranges: + s = "{0x%X, 0x%X}, " % (lo, hi) + if len(line) + len(s) > 96: + out.append(line.rstrip()) + line = " " + line += s + if line.strip(): + out.append(line.rstrip()) + out.append("};") + out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};") + return "\n".join(out) + + +def main(): + punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)]) + control = merge([cp for cp in range(MAX_CP) if is_control(cp)]) + print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata + * (Unicode {unicodedata.unidata_version}). Do not edit by hand. + * + * Range tables for the BasicTokenizer's _is_punctuation and _is_control, + * matching transformers' BertTokenizer. Punctuation is the ASCII set + * (33-47/58-64/91-96/123-126) plus every category P* codepoint. Control is + * categories Cc, Cf, Co minus tab/newline/carriage-return (those route to + * whitespace). Unassigned (Cn) codepoints are intentionally not control: they + * never occur in real text and would bind the table to one Unicode version. */ +#ifndef WORDPIECE_UNICODE_H +#define WORDPIECE_UNICODE_H + +#include <stdint.h> + +typedef struct {{ + uint32_t lo; + uint32_t hi; +}} wp_range_t; + +{emit("WP_PUNCT_RANGES", punct)} + +{emit("WP_CONTROL_RANGES", control)} + +#endif /* WORDPIECE_UNICODE_H */""") + + +if __name__ == "__main__": + main() |
