From 51a716cc7bc555d765cf4ab97c6ee4935683d129 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 16 Jun 2026 16:51:35 +0200 Subject: tools: point wordpiece unicode generator at src/tokenizer_unicode.h --- tools/gen_wordpiece_unicode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py index 24d56c9..24649f1 100644 --- a/tools/gen_wordpiece_unicode.py +++ b/tools/gen_wordpiece_unicode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate wordpiece_unicode.h: Unicode range tables for the WordPiece +"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece BasicTokenizer's punctuation and control classification, matching transformers' BertTokenizer._is_punctuation and _is_control exactly. @@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. Run when bumping the target Unicode version (the C side is data-only otherwise): - uv run python tools/gen_wordpiece_unicode.py > wordpiece_unicode.h + uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h """ import unicodedata -- cgit v1.3.1