aboutsummaryrefslogtreecommitdiff
path: root/bench/bench_qwen_tokenizer.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:45:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitbc8bebe78358e3a37bad0867b2d94955cac01d47 (patch)
treed5a49a34b95b45cec0917a2a07707159825fb912 /bench/bench_qwen_tokenizer.py
parent6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff)
downloadffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz
ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/bench_qwen_tokenizer.py')
-rwxr-xr-xbench/bench_qwen_tokenizer.py59
1 files changed, 59 insertions, 0 deletions
diff --git a/bench/bench_qwen_tokenizer.py b/bench/bench_qwen_tokenizer.py
new file mode 100755
index 0000000..a69921d
--- /dev/null
+++ b/bench/bench_qwen_tokenizer.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Benchmark Qwen tokenizer encode paths.
+
+Builds the C tokenizer bench via the Makefile (`make -C bench bench_tokenizer`)
+and runs it against a Qwen model's vocab.json with a curated text set.
+
+ bench/bench_qwen_tokenizer.py --model-dir DIR
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+DEFAULT_TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: dense retrieval systems need stable tokenization.",
+ "unicode: cafe, café, Istanbul, İstanbul, emoji \U0001f50d",
+ "mixed whitespace:\n one\t two three\n\nfour",
+ ("long document: " + "embedding inference should batch tokenization and "
+ "model execution carefully. ") * 32,
+]
+
+
+def build(cc: str) -> Path:
+ """Build bench/bench_tokenizer via the Makefile; return its path."""
+ proc = subprocess.run(["make", "-C", "bench", f"CC={cc}", "bench_tokenizer"],
+ cwd=ROOT, text=True, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, check=False)
+ if proc.returncode != 0:
+ sys.stderr.write(proc.stdout)
+ raise SystemExit(proc.returncode)
+ return ROOT / "bench" / "bench_tokenizer"
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--runs", type=int, default=1000)
+ ap.add_argument("--cc", default=os.environ.get("CC", "cc"))
+ ap.add_argument("--text", action="append",
+ help="text to benchmark; may be repeated")
+ args = ap.parse_args()
+
+ binary = build(args.cc)
+ vocab = Path(args.model_dir) / "vocab.json"
+ texts = args.text if args.text else DEFAULT_TEXTS
+ return subprocess.run([str(binary), str(vocab), str(args.runs), *texts],
+ cwd=ROOT).returncode
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())