1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
#!/usr/bin/env python3
"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece
BasicTokenizer's punctuation and control classification, matching transformers'
BertTokenizer._is_punctuation and _is_control exactly.
Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint
whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus
tab/newline/carriage-return (which route to whitespace). The tables are emitted
as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
Run when bumping the target Unicode version (the C side is data-only otherwise):
uv run python devtools/gen_wordpiece_unicode.py > unicode.h
"""
import unicodedata
MAX_CP = 0x110000
def merge(cps):
"""Collapse a sorted codepoint list into contiguous [lo, hi] ranges."""
ranges = []
for cp in cps:
if ranges and cp == ranges[-1][1] + 1:
ranges[-1][1] = cp
else:
ranges.append([cp, cp])
return ranges
def is_punct(cp):
if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126):
return True
return unicodedata.category(chr(cp)).startswith("P")
def is_control(cp):
if cp in (0x09, 0x0A, 0x0D):
return False
# Cc/Cf/Co are the assigned control, format and private-use chars. Cn
# (unassigned) is intentionally excluded: it never occurs in real text and
# would bind the table to a single Unicode version.
return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co")
# Hangul syllables decompose by formula (the standard algorithm), so they are
# excluded from the fold table to keep it small; the C side handles them.
HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172
def fold(cp):
"""do_lower_case fold: Python str.lower then NFD then drop combining marks
(Mn), matching transformers' lowercase + _run_strip_accents. Returns the
output codepoint list (empty when the codepoint is a dropped mark)."""
s = chr(cp).lower()
return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"]
def emit_fold(entries):
out = ["static const wp_fold_t WP_FOLD[] = {"]
for cp, f in entries:
outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3])
out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f)))
out.append("};")
out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };")
return "\n".join(out)
def emit(name, ranges):
out = [f"static const wp_range_t {name}[] = {{"]
line = " "
for lo, hi in ranges:
s = "{0x%X, 0x%X}, " % (lo, hi)
if len(line) + len(s) > 96:
out.append(line.rstrip())
line = " "
line += s
if line.strip():
out.append(line.rstrip())
out.append("};")
out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};")
return "\n".join(out)
def main():
punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)])
control = merge([cp for cp in range(MAX_CP) if is_control(cp)])
hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT)
fold_entries = [
(cp, f)
for cp in range(MAX_CP)
if cp not in hangul
for f in [fold(cp)]
if f != [cp]
]
print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata
* (Unicode {unicodedata.unidata_version}). Do not edit by hand.
*
* Tables for the BasicTokenizer, matching transformers' BertTokenizer.
*
* WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted
* [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus
* every category P* codepoint. Control is categories Cc, Cf, Co minus
* tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is
* intentionally not control: it never occurs in real text and would bind the
* table to one Unicode version.
*
* WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks),
* sorted by codepoint. n is the output length (0 drops a combining mark; up to
* 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and
* are excluded here. */
#ifndef WORDPIECE_UNICODE_H
#define WORDPIECE_UNICODE_H
#include <stdint.h>
typedef struct {{
uint32_t lo;
uint32_t hi;
}} wp_range_t;
typedef struct {{
uint32_t cp;
uint32_t out[3];
uint8_t n;
}} wp_fold_t;
{emit("WP_PUNCT_RANGES", punct)}
{emit("WP_CONTROL_RANGES", control)}
{emit_fold(fold_entries)}
#endif /* WORDPIECE_UNICODE_H */""")
if __name__ == "__main__":
main()
|