diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:45:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | bc8bebe78358e3a37bad0867b2d94955cac01d47 (patch) | |
| tree | d5a49a34b95b45cec0917a2a07707159825fb912 /bench/bench_late_rerank.py | |
| parent | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff) | |
| download | ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip | |
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/bench_late_rerank.py')
| -rwxr-xr-x | bench/bench_late_rerank.py | 52 |
1 files changed, 52 insertions, 0 deletions
diff --git a/bench/bench_late_rerank.py b/bench/bench_late_rerank.py new file mode 100755 index 0000000..99bb817 --- /dev/null +++ b/bench/bench_late_rerank.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Benchmark late-interaction MaxSim reranking shapes. + +Builds the C bench (bench/bench_late_rerank.c) via the Makefile and runs it +against a late model directory (e.g. pplx-embed-v1-late-0.6b). python3 only. + + bench/bench_late_rerank.py --model-dir DIR +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def build(cc: str, backend: str) -> Path: + """Build the late-rerank bench via the Makefile; return its path.""" + target = "bench_late_rerank_mlx" if backend == "mlx" else "bench_late_rerank" + proc = subprocess.run(["make", "-C", "bench", f"CC={cc}", target], cwd=ROOT, + text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + raise SystemExit(proc.returncode) + return ROOT / "bench" / target + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--backend", choices=("cpu", "mlx"), default="cpu") + ap.add_argument("--candidates", type=int, default=32) + ap.add_argument("--doc-repeat", type=int, default=3) + ap.add_argument("--runs", type=int, default=200) + ap.add_argument("--cc", default=os.environ.get("CC", "cc")) + args = ap.parse_args() + + binary = build(args.cc, args.backend) + return subprocess.run( + [str(binary), args.model_dir, str(args.candidates), + str(args.doc_repeat), str(args.runs)], + cwd=ROOT, + ).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) |
