diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:31:06 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | dbdf714bb57578431924e5d069089bdfd9f4c604 (patch) | |
| tree | 4f5b59ef132220310e1e4b1dde64f5e38782369a /devtools/gen_test_vectors.py | |
| parent | e9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff) | |
| download | ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip | |
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to 'devtools/gen_test_vectors.py')
| -rw-r--r-- | devtools/gen_test_vectors.py | 209 |
1 files changed, 209 insertions, 0 deletions
diff --git a/devtools/gen_test_vectors.py b/devtools/gen_test_vectors.py new file mode 100644 index 0000000..44206ec --- /dev/null +++ b/devtools/gen_test_vectors.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Generate tokenizer parity vectors for the hermetic tokenizer checks. + +tests/check_tokenizer_parity.py and tests/check_wordpiece_parity.py read +the stored vectors generated by this script. Regenerate the vectors when +the test corpus changes or the reference tokenizer is bumped, then check +the result into git. + +For each tokenizer family it writes, under tests/test-vectors/<family>/: + - the vocab files the C driver loads (vocab.json + merges.txt for BPE; + vocab.txt + tokenizer_config.json for WordPiece), + - expected.json: provenance plus a list of {text, ids} cases. + +- The BPE reference is the fast Qwen tokenizer (what the C BPE backend +targets). +- The WordPiece reference is the slow BertTokenizer (the exact reference +for the C WordPiece backend). +- Both encode with add_special_tokens=False, matching the C drivers, +which emit core ids and leave [CLS]/[SEP] to the frontend. + + uv run --with transformers devtools/gen_test_vectors.py + uv run --with transformers devtools/gen_test_vectors.py --family bpe + uv run --with transformers devtools/gen_test_vectors.py --bpe-model DIR --fuzz 128 +""" + +import argparse +import json +import random +import re +import shutil +import sys +from pathlib import Path + +# BPE corpus: ASCII, accents, decomposed Unicode, emoji, digits, and long +# repeated strings that stress the merge loop. +BPE_TEXTS = [ + "", + "hello world", + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "punctuation: hello, world!!! (test)", + "repeated spaces\tand\nnewlines", + "unicode: cafe, caf\xe9, Istanbul, İstanbul", + "unicode decomposed: café, façade", + "emoji: embeddings \U0001f50d are useful", + "numbers: 12345 67.89 -0.12", + "a " * 256, + "antidisestablishmentarianism" * 64, + ("document: " + "dense retrieval systems need stable tokenization. " * 256), +] + +BPE_FUZZ_ALPHABET = ( + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + " \t\n\r" + ".,!?;:()[]{}'\"-/" + "éèêëáàâäíìîïóòôöúùûüñç" + "¿¡«»—…" + "中文测试Русскийعربيहिन्दी" + "🔍👍🏽🇺🇸" +) + +# WordPiece corpus: stays within the implemented coverage (ASCII, Latin-1 +# accents, common punctuation, CJK char-splitting) plus a multilingual set. +WORDPIECE_TEXTS = [ + "", + "hello world", + "The capital of France is Paris.", + "Tokenization, sub-words, and WordPiece!", + "unaffable antidisestablishmentarianism", + "café résumé naïve Zürich Málaga", + "COVID-19 affected 2020-2021 globally.", + "don't can't won't it's o'clock", + "e-mail user@example.com or http://test.org/path?q=1", + "numbers: 12345 67.89 -0.12 3.14159 (approx)", + "repeated spaces\tand\nnewlines\rhere", + "Hyphenated state-of-the-art well-known co-operate", + "MixedCase camelCase snake_case ALLCAPS", + "中 文 字 符", + "a " * 200, + "supercalifragilisticexpialidocious pneumonoultramicroscopicsilicovolcanoconiosis", + "Müller wohnt in Zürich, Straße.", + "L'élève préfère — n'est-ce pas?", + "¿Cómo estás? ¡Muy bien!", + "你好,世界。こんにちは", +] + +WORDPIECE_FUZZ_ALPHABET = ( + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + " \t\n\r" + ".,!?;:()[]{}'\"-/@#" + "éèêëáàâäíìîïóòôöúùûüñçßæøþ" +) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def transformers_version() -> str: + import transformers + return transformers.__version__ + + +def snapshot_revision(local_path: str) -> str: + """Best-effort: pull the commit hash from an HF cache path, else ''.""" + m = re.search(r"/snapshots/([0-9a-f]{8,})/", local_path + "/") + return m.group(1) if m else "" + + +def resolve_file(model: str, filename: str) -> Path: + """Return a local path to one of the model's files, from a local dir or the + HF cache (download only if absent).""" + local = Path(model) / filename + if local.exists(): + return local + from huggingface_hub import hf_hub_download + return Path(hf_hub_download(model, filename)) + + +def fuzz_cases(alphabet: str, count: int, seed: int) -> list[str]: + rng = random.Random(seed) + out = [] + for _ in range(count): + n = rng.randint(0, 96) + out.append("".join(rng.choice(alphabet) for _ in range(n))) + return out + + +def write_vectors(out_dir: Path, family: str, model: str, revision: str, + texts: list[str], encode) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + cases = [{"text": t, "ids": encode(t)} for t in texts] + payload = { + "family": family, + "model": model, + "revision": revision, + "transformers_version": transformers_version(), + "add_special_tokens": False, + "cases": cases, + } + (out_dir / "expected.json").write_text( + json.dumps(payload, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + print(f"ok: wrote {len(cases)} {family} cases to {out_dir}") + + +def gen_bpe(root: Path, model: str, fuzz: int, seed: int) -> None: + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(model, trust_remote_code=True) + out_dir = root / "tests/test-vectors/bpe" + out_dir.mkdir(parents=True, exist_ok=True) + # Copy the shipped vocab.json + merges.txt the C BPE loader reads. The fast + # tokenizer's save_vocabulary does not re-emit them, so copy the originals + # the reference tokenizer was built from. + src = "" + for fn in ("vocab.json", "merges.txt"): + path = resolve_file(model, fn) + src = str(path) + shutil.copyfile(path, out_dir / fn) + texts = list(BPE_TEXTS) + fuzz_cases(BPE_FUZZ_ALPHABET, fuzz, seed) + revision = snapshot_revision(src) + write_vectors(out_dir, "bpe", model, revision, texts, + lambda t: tok.encode(t, add_special_tokens=False)) + + +def gen_wordpiece(root: Path, model: str, fuzz: int, seed: int) -> None: + from transformers import BertTokenizer + tok = BertTokenizer.from_pretrained(model) + out_dir = root / "tests/test-vectors/wordpiece" + out_dir.mkdir(parents=True, exist_ok=True) + # Copy vocab.txt and write the tokenizer_config.json the C loader reads for + # do_lower_case (save_pretrained would emit tokenizer.json, which the C + # side ignores). + src = resolve_file(model, "vocab.txt") + shutil.copyfile(src, out_dir / "vocab.txt") + lower = "true" if tok.do_lower_case else "false" + (out_dir / "tokenizer_config.json").write_text( + '{"do_lower_case": %s}\n' % lower, encoding="utf-8") + texts = list(WORDPIECE_TEXTS) + fuzz_cases(WORDPIECE_FUZZ_ALPHABET, fuzz, seed) + revision = snapshot_revision(str(src)) + write_vectors(out_dir, "wordpiece", model, revision, texts, + lambda t: tok.encode(t, add_special_tokens=False)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--family", choices=("bpe", "wordpiece", "both"), default="both") + ap.add_argument("--bpe-model", default="Qwen/Qwen3-Embedding-0.6B", + help="HF name or local dir for the BPE reference tokenizer") + ap.add_argument("--wordpiece-model", default="bert-base-uncased", + help="HF name or local dir for the WordPiece reference tokenizer") + ap.add_argument("--fuzz", type=int, default=64, + help="deterministic random cases to bake in per family") + ap.add_argument("--seed", type=int, default=1) + args = ap.parse_args() + + root = repo_root() + if args.family in ("bpe", "both"): + gen_bpe(root, args.bpe_model, args.fuzz, args.seed) + if args.family in ("wordpiece", "both"): + gen_wordpiece(root, args.wordpiece_model, args.fuzz, args.seed) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) |
