diff options
Diffstat (limited to '')
| -rwxr-xr-x[-rw-r--r--] | devtools/gen_wordpiece_unicode.py (renamed from tools/gen_wordpiece_unicode.py) | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py index 24649f1..2f8642e 100644..100755 --- a/tools/gen_wordpiece_unicode.py +++ b/devtools/gen_wordpiece_unicode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece +"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece BasicTokenizer's punctuation and control classification, matching transformers' BertTokenizer._is_punctuation and _is_control exactly. @@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. Run when bumping the target Unicode version (the C side is data-only otherwise): - uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h + uv run python devtools/gen_wordpiece_unicode.py > unicode.h """ import unicodedata @@ -94,7 +94,7 @@ def main(): for f in [fold(cp)] if f != [cp] ] - print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata + print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata * (Unicode {unicodedata.unidata_version}). Do not edit by hand. * * Tables for the BasicTokenizer, matching transformers' BertTokenizer. |
