aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/gen_wordpiece_unicode.py4
1 files changed, 2 insertions, 2 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py
index 24d56c9..24649f1 100644
--- a/tools/gen_wordpiece_unicode.py
+++ b/tools/gen_wordpiece_unicode.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Generate wordpiece_unicode.h: Unicode range tables for the WordPiece
+"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece
BasicTokenizer's punctuation and control classification, matching transformers'
BertTokenizer._is_punctuation and _is_control exactly.
@@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
Run when bumping the target Unicode version (the C side is data-only otherwise):
- uv run python tools/gen_wordpiece_unicode.py > wordpiece_unicode.h
+ uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h
"""
import unicodedata