aboutsummaryrefslogtreecommitdiff
path: root/devtools/gen_wordpiece_unicode.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:31:06 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitdbdf714bb57578431924e5d069089bdfd9f4c604 (patch)
tree4f5b59ef132220310e1e4b1dde64f5e38782369a /devtools/gen_wordpiece_unicode.py
parente9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff)
downloadffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz
ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to '')
-rwxr-xr-x[-rw-r--r--]devtools/gen_wordpiece_unicode.py (renamed from tools/gen_wordpiece_unicode.py)6
1 files changed, 3 insertions, 3 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py
index 24649f1..2f8642e 100644..100755
--- a/tools/gen_wordpiece_unicode.py
+++ b/devtools/gen_wordpiece_unicode.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece
+"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece
BasicTokenizer's punctuation and control classification, matching transformers'
BertTokenizer._is_punctuation and _is_control exactly.
@@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
Run when bumping the target Unicode version (the C side is data-only otherwise):
- uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h
+ uv run python devtools/gen_wordpiece_unicode.py > unicode.h
"""
import unicodedata
@@ -94,7 +94,7 @@ def main():
for f in [fold(cp)]
if f != [cp]
]
- print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
+ print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata
* (Unicode {unicodedata.unidata_version}). Do not edit by hand.
*
* Tables for the BasicTokenizer, matching transformers' BertTokenizer.