aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:51:35 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:51:35 +0200
commit51a716cc7bc555d765cf4ab97c6ee4935683d129 (patch)
tree90cafc39d949001aebf077ee5a80f51489d950c8
parent78f5588d761f0df66eb11c2b79bee7c22fb0d132 (diff)
downloadffwd-51a716cc7bc555d765cf4ab97c6ee4935683d129.tar.gz
ffwd-51a716cc7bc555d765cf4ab97c6ee4935683d129.zip
tools: point wordpiece unicode generator at src/tokenizer_unicode.h
-rw-r--r--tools/gen_wordpiece_unicode.py4
1 files changed, 2 insertions, 2 deletions
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py
index 24d56c9..24649f1 100644
--- a/tools/gen_wordpiece_unicode.py
+++ b/tools/gen_wordpiece_unicode.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Generate wordpiece_unicode.h: Unicode range tables for the WordPiece
+"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece
BasicTokenizer's punctuation and control classification, matching transformers'
BertTokenizer._is_punctuation and _is_control exactly.
@@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
Run when bumping the target Unicode version (the C side is data-only otherwise):
- uv run python tools/gen_wordpiece_unicode.py > wordpiece_unicode.h
+ uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h
"""
import unicodedata