diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:31:06 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | dbdf714bb57578431924e5d069089bdfd9f4c604 (patch) | |
| tree | 4f5b59ef132220310e1e4b1dde64f5e38782369a /devtools/gen_wordpiece_unicode.py | |
| parent | e9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff) | |
| download | ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip | |
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to '')
| -rwxr-xr-x[-rw-r--r--] | devtools/gen_wordpiece_unicode.py (renamed from tools/gen_wordpiece_unicode.py) | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py index 24649f1..2f8642e 100644..100755 --- a/tools/gen_wordpiece_unicode.py +++ b/devtools/gen_wordpiece_unicode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece +"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece BasicTokenizer's punctuation and control classification, matching transformers' BertTokenizer._is_punctuation and _is_control exactly. @@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. Run when bumping the target Unicode version (the C side is data-only otherwise): - uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h + uv run python devtools/gen_wordpiece_unicode.py > unicode.h """ import unicodedata @@ -94,7 +94,7 @@ def main(): for f in [fold(cp)] if f != [cp] ] - print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata + print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata * (Unicode {unicodedata.unidata_version}). Do not edit by hand. * * Tables for the BasicTokenizer, matching transformers' BertTokenizer. |
