diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:45:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | bc8bebe78358e3a37bad0867b2d94955cac01d47 (patch) | |
| tree | d5a49a34b95b45cec0917a2a07707159825fb912 /bench/bench_qwen_tokenizer.py | |
| parent | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff) | |
| download | ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip | |
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/bench_qwen_tokenizer.py')
| -rwxr-xr-x | bench/bench_qwen_tokenizer.py | 59 |
1 files changed, 59 insertions, 0 deletions
diff --git a/bench/bench_qwen_tokenizer.py b/bench/bench_qwen_tokenizer.py new file mode 100755 index 0000000..a69921d --- /dev/null +++ b/bench/bench_qwen_tokenizer.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Benchmark Qwen tokenizer encode paths. + +Builds the C tokenizer bench via the Makefile (`make -C bench bench_tokenizer`) +and runs it against a Qwen model's vocab.json with a curated text set. + + bench/bench_qwen_tokenizer.py --model-dir DIR +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +DEFAULT_TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: dense retrieval systems need stable tokenization.", + "unicode: cafe, café, Istanbul, İstanbul, emoji \U0001f50d", + "mixed whitespace:\n one\t two three\n\nfour", + ("long document: " + "embedding inference should batch tokenization and " + "model execution carefully. ") * 32, +] + + +def build(cc: str) -> Path: + """Build bench/bench_tokenizer via the Makefile; return its path.""" + proc = subprocess.run(["make", "-C", "bench", f"CC={cc}", "bench_tokenizer"], + cwd=ROOT, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + raise SystemExit(proc.returncode) + return ROOT / "bench" / "bench_tokenizer" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--runs", type=int, default=1000) + ap.add_argument("--cc", default=os.environ.get("CC", "cc")) + ap.add_argument("--text", action="append", + help="text to benchmark; may be repeated") + args = ap.parse_args() + + binary = build(args.cc) + vocab = Path(args.model_dir) / "vocab.json" + texts = args.text if args.text else DEFAULT_TEXTS + return subprocess.run([str(binary), str(vocab), str(args.runs), *texts], + cwd=ROOT).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) |
