aboutsummaryrefslogtreecommitdiff
path: root/devtools/gen_test_vectors.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:31:06 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitdbdf714bb57578431924e5d069089bdfd9f4c604 (patch)
tree4f5b59ef132220310e1e4b1dde64f5e38782369a /devtools/gen_test_vectors.py
parente9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff)
downloadffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz
ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to 'devtools/gen_test_vectors.py')
-rw-r--r--devtools/gen_test_vectors.py209
1 files changed, 209 insertions, 0 deletions
diff --git a/devtools/gen_test_vectors.py b/devtools/gen_test_vectors.py
new file mode 100644
index 0000000..44206ec
--- /dev/null
+++ b/devtools/gen_test_vectors.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Generate tokenizer parity vectors for the hermetic tokenizer checks.
+
+tests/check_tokenizer_parity.py and tests/check_wordpiece_parity.py read
+the stored vectors generated by this script. Regenerate the vectors when
+the test corpus changes or the reference tokenizer is bumped, then check
+the result into git.
+
+For each tokenizer family it writes, under tests/test-vectors/<family>/:
+ - the vocab files the C driver loads (vocab.json + merges.txt for BPE;
+ vocab.txt + tokenizer_config.json for WordPiece),
+ - expected.json: provenance plus a list of {text, ids} cases.
+
+- The BPE reference is the fast Qwen tokenizer (what the C BPE backend
+targets).
+- The WordPiece reference is the slow BertTokenizer (the exact reference
+for the C WordPiece backend).
+- Both encode with add_special_tokens=False, matching the C drivers,
+which emit core ids and leave [CLS]/[SEP] to the frontend.
+
+ uv run --with transformers devtools/gen_test_vectors.py
+ uv run --with transformers devtools/gen_test_vectors.py --family bpe
+ uv run --with transformers devtools/gen_test_vectors.py --bpe-model DIR --fuzz 128
+"""
+
+import argparse
+import json
+import random
+import re
+import shutil
+import sys
+from pathlib import Path
+
+# BPE corpus: ASCII, accents, decomposed Unicode, emoji, digits, and long
+# repeated strings that stress the merge loop.
+BPE_TEXTS = [
+ "",
+ "hello world",
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "punctuation: hello, world!!! (test)",
+ "repeated spaces\tand\nnewlines",
+ "unicode: cafe, caf\xe9, Istanbul, İstanbul",
+ "unicode decomposed: café, façade",
+ "emoji: embeddings \U0001f50d are useful",
+ "numbers: 12345 67.89 -0.12",
+ "a " * 256,
+ "antidisestablishmentarianism" * 64,
+ ("document: " + "dense retrieval systems need stable tokenization. " * 256),
+]
+
+BPE_FUZZ_ALPHABET = (
+ "abcdefghijklmnopqrstuvwxyz"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "0123456789"
+ " \t\n\r"
+ ".,!?;:()[]{}'\"-/"
+ "éèêëáàâäíìîïóòôöúùûüñç"
+ "¿¡«»—…"
+ "中文测试Русскийعربيहिन्दी"
+ "🔍👍🏽🇺🇸"
+)
+
+# WordPiece corpus: stays within the implemented coverage (ASCII, Latin-1
+# accents, common punctuation, CJK char-splitting) plus a multilingual set.
+WORDPIECE_TEXTS = [
+ "",
+ "hello world",
+ "The capital of France is Paris.",
+ "Tokenization, sub-words, and WordPiece!",
+ "unaffable antidisestablishmentarianism",
+ "café résumé naïve Zürich Málaga",
+ "COVID-19 affected 2020-2021 globally.",
+ "don't can't won't it's o'clock",
+ "e-mail user@example.com or http://test.org/path?q=1",
+ "numbers: 12345 67.89 -0.12 3.14159 (approx)",
+ "repeated spaces\tand\nnewlines\rhere",
+ "Hyphenated state-of-the-art well-known co-operate",
+ "MixedCase camelCase snake_case ALLCAPS",
+ "中 文 字 符",
+ "a " * 200,
+ "supercalifragilisticexpialidocious pneumonoultramicroscopicsilicovolcanoconiosis",
+ "Müller wohnt in Zürich, Straße.",
+ "L'élève préfère — n'est-ce pas?",
+ "¿Cómo estás? ¡Muy bien!",
+ "你好,世界。こんにちは",
+]
+
+WORDPIECE_FUZZ_ALPHABET = (
+ "abcdefghijklmnopqrstuvwxyz"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "0123456789"
+ " \t\n\r"
+ ".,!?;:()[]{}'\"-/@#"
+ "éèêëáàâäíìîïóòôöúùûüñçßæøþ"
+)
+
+
+def repo_root() -> Path:
+ return Path(__file__).resolve().parents[1]
+
+
+def transformers_version() -> str:
+ import transformers
+ return transformers.__version__
+
+
+def snapshot_revision(local_path: str) -> str:
+ """Best-effort: pull the commit hash from an HF cache path, else ''."""
+ m = re.search(r"/snapshots/([0-9a-f]{8,})/", local_path + "/")
+ return m.group(1) if m else ""
+
+
+def resolve_file(model: str, filename: str) -> Path:
+ """Return a local path to one of the model's files, from a local dir or the
+ HF cache (download only if absent)."""
+ local = Path(model) / filename
+ if local.exists():
+ return local
+ from huggingface_hub import hf_hub_download
+ return Path(hf_hub_download(model, filename))
+
+
+def fuzz_cases(alphabet: str, count: int, seed: int) -> list[str]:
+ rng = random.Random(seed)
+ out = []
+ for _ in range(count):
+ n = rng.randint(0, 96)
+ out.append("".join(rng.choice(alphabet) for _ in range(n)))
+ return out
+
+
+def write_vectors(out_dir: Path, family: str, model: str, revision: str,
+ texts: list[str], encode) -> None:
+ out_dir.mkdir(parents=True, exist_ok=True)
+ cases = [{"text": t, "ids": encode(t)} for t in texts]
+ payload = {
+ "family": family,
+ "model": model,
+ "revision": revision,
+ "transformers_version": transformers_version(),
+ "add_special_tokens": False,
+ "cases": cases,
+ }
+ (out_dir / "expected.json").write_text(
+ json.dumps(payload, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
+ print(f"ok: wrote {len(cases)} {family} cases to {out_dir}")
+
+
+def gen_bpe(root: Path, model: str, fuzz: int, seed: int) -> None:
+ from transformers import AutoTokenizer
+ tok = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
+ out_dir = root / "tests/test-vectors/bpe"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ # Copy the shipped vocab.json + merges.txt the C BPE loader reads. The fast
+ # tokenizer's save_vocabulary does not re-emit them, so copy the originals
+ # the reference tokenizer was built from.
+ src = ""
+ for fn in ("vocab.json", "merges.txt"):
+ path = resolve_file(model, fn)
+ src = str(path)
+ shutil.copyfile(path, out_dir / fn)
+ texts = list(BPE_TEXTS) + fuzz_cases(BPE_FUZZ_ALPHABET, fuzz, seed)
+ revision = snapshot_revision(src)
+ write_vectors(out_dir, "bpe", model, revision, texts,
+ lambda t: tok.encode(t, add_special_tokens=False))
+
+
+def gen_wordpiece(root: Path, model: str, fuzz: int, seed: int) -> None:
+ from transformers import BertTokenizer
+ tok = BertTokenizer.from_pretrained(model)
+ out_dir = root / "tests/test-vectors/wordpiece"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ # Copy vocab.txt and write the tokenizer_config.json the C loader reads for
+ # do_lower_case (save_pretrained would emit tokenizer.json, which the C
+ # side ignores).
+ src = resolve_file(model, "vocab.txt")
+ shutil.copyfile(src, out_dir / "vocab.txt")
+ lower = "true" if tok.do_lower_case else "false"
+ (out_dir / "tokenizer_config.json").write_text(
+ '{"do_lower_case": %s}\n' % lower, encoding="utf-8")
+ texts = list(WORDPIECE_TEXTS) + fuzz_cases(WORDPIECE_FUZZ_ALPHABET, fuzz, seed)
+ revision = snapshot_revision(str(src))
+ write_vectors(out_dir, "wordpiece", model, revision, texts,
+ lambda t: tok.encode(t, add_special_tokens=False))
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--family", choices=("bpe", "wordpiece", "both"), default="both")
+ ap.add_argument("--bpe-model", default="Qwen/Qwen3-Embedding-0.6B",
+ help="HF name or local dir for the BPE reference tokenizer")
+ ap.add_argument("--wordpiece-model", default="bert-base-uncased",
+ help="HF name or local dir for the WordPiece reference tokenizer")
+ ap.add_argument("--fuzz", type=int, default=64,
+ help="deterministic random cases to bake in per family")
+ ap.add_argument("--seed", type=int, default=1)
+ args = ap.parse_args()
+
+ root = repo_root()
+ if args.family in ("bpe", "both"):
+ gen_bpe(root, args.bpe_model, args.fuzz, args.seed)
+ if args.family in ("wordpiece", "both"):
+ gen_wordpiece(root, args.wordpiece_model, args.fuzz, args.seed)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())