aboutsummaryrefslogtreecommitdiff
path: root/bench/bench_late_api.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:45:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitbc8bebe78358e3a37bad0867b2d94955cac01d47 (patch)
treed5a49a34b95b45cec0917a2a07707159825fb912 /bench/bench_late_api.py
parent6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff)
downloadffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz
ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/bench_late_api.py')
-rwxr-xr-xbench/bench_late_api.py202
1 files changed, 202 insertions, 0 deletions
diff --git a/bench/bench_late_api.py b/bench/bench_late_api.py
new file mode 100755
index 0000000..7a6073f
--- /dev/null
+++ b/bench/bench_late_api.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Benchmark /v1/rerank and optionally compare end-to-end with PyLate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import time
+import urllib.error
+import urllib.request
+from typing import Any
+
+
+BASE_DOCUMENTS = [
+ "Scientists explore the universe driven by curiosity.",
+ "Children learn through curious exploration.",
+ "Historical discoveries began with careful questions.",
+ "Redis stores data structures in memory for fast access.",
+ "SQLite keeps relational data in a local database file.",
+ "Embedding models map text into semantic vector spaces.",
+ "Paris is the capital of France and a major European destination.",
+ "Neural search compares query and document token representations.",
+]
+QUERY = "What motivates scientific discovery?"
+
+
+def percentile(values: list[float], p: float) -> float:
+ values = sorted(values)
+ rank = (len(values) - 1) * p / 100
+ lo = int(rank)
+ hi = min(lo + 1, len(values) - 1)
+ frac = rank - lo
+ return values[lo] * (1 - frac) + values[hi] * frac
+
+
+def make_documents(candidates: int, repeat: int) -> list[str]:
+ return [
+ " ".join(
+ BASE_DOCUMENTS[(candidate + offset) % len(BASE_DOCUMENTS)]
+ for offset in range(repeat)
+ )
+ for candidate in range(candidates)
+ ]
+
+
+def post_json(
+ base_url: str,
+ payload: dict[str, Any],
+ api_key: str | None,
+ timeout: float,
+) -> dict[str, Any]:
+ headers = {"Content-Type": "application/json"}
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+ req = urllib.request.Request(
+ base_url.rstrip("/") + "/v1/rerank",
+ data=json.dumps(payload).encode(),
+ headers=headers,
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
+ return json.loads(resp.read())
+ except urllib.error.HTTPError as exc:
+ text = exc.read().decode(errors="replace")
+ raise RuntimeError(f"HTTP {exc.code}: {text}") from exc
+
+
+def summarize(name: str, values: list[float], candidates: int) -> None:
+ mean = statistics.fmean(values)
+ print(
+ f"{name}: mean_ms={mean:.2f} p50_ms={percentile(values, 50):.2f} "
+ f"p95_ms={percentile(values, 95):.2f} "
+ f"candidates/s={candidates * 1000 / mean:.2f}"
+ )
+
+
+def synchronize(device: str) -> None:
+ if device == "mps":
+ import torch
+
+ torch.mps.synchronize()
+ elif device.startswith("cuda"):
+ import torch
+
+ torch.cuda.synchronize()
+
+
+def pylate_run(model: Any, query: str, documents: list[str], device: str) -> list[float]:
+ from pylate.scores import colbert_scores
+
+ query_vectors = model.encode(
+ [query],
+ is_query=True,
+ convert_to_tensor=True,
+ show_progress_bar=False,
+ )[0]
+ document_vectors = model.encode(
+ documents,
+ is_query=False,
+ convert_to_tensor=True,
+ show_progress_bar=False,
+ )
+ scores = [
+ float(
+ colbert_scores(
+ query_vectors.unsqueeze(0),
+ document.unsqueeze(0),
+ )[0, 0]
+ )
+ for document in document_vectors
+ ]
+ synchronize(device)
+ return scores
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--base-url", default="http://127.0.0.1:8000")
+ ap.add_argument("--api-key")
+ ap.add_argument("--model", default="pplx-embed-v1-late-0.6b")
+ ap.add_argument("--model-dir")
+ ap.add_argument("--candidates", type=int, default=32)
+ ap.add_argument("--document-repeat", type=int, default=2)
+ ap.add_argument("--runs", type=int, default=5)
+ ap.add_argument("--warmups", type=int, default=1)
+ ap.add_argument("--timeout", type=float, default=600)
+ ap.add_argument("--compare-pylate", action="store_true")
+ ap.add_argument("--pylate-device", default="mps")
+ args = ap.parse_args()
+ if min(args.candidates, args.document_repeat, args.runs) <= 0:
+ raise SystemExit("candidates, document-repeat, and runs must be positive")
+ if args.warmups < 0:
+ raise SystemExit("warmups must be non-negative")
+ if args.compare_pylate and not args.model_dir:
+ raise SystemExit("--model-dir is required with --compare-pylate")
+
+ documents = make_documents(args.candidates, args.document_repeat)
+ payload = {
+ "model": args.model,
+ "query": QUERY,
+ "documents": documents,
+ }
+ for _ in range(args.warmups):
+ post_json(args.base_url, payload, args.api_key, args.timeout)
+ server_times = []
+ server_response: dict[str, Any] = {}
+ for _ in range(args.runs):
+ start = time.perf_counter()
+ server_response = post_json(
+ args.base_url, payload, args.api_key, args.timeout
+ )
+ server_times.append((time.perf_counter() - start) * 1000)
+ summarize("pplx-embed-server", server_times, args.candidates)
+
+ if args.compare_pylate:
+ try:
+ from pylate import models
+ except ImportError as exc:
+ raise SystemExit(
+ "PyLate is missing; run with `uv run --with pylate "
+ "scripts/bench_late_api.py ...`"
+ ) from exc
+
+ model = models.ColBERT(
+ model_name_or_path=args.model_dir,
+ trust_remote_code=True,
+ device=args.pylate_device,
+ )
+ for _ in range(args.warmups):
+ pylate_run(model, QUERY, documents, args.pylate_device)
+ pylate_times = []
+ pylate_scores: list[float] = []
+ for _ in range(args.runs):
+ start = time.perf_counter()
+ pylate_scores = pylate_run(
+ model, QUERY, documents, args.pylate_device
+ )
+ pylate_times.append((time.perf_counter() - start) * 1000)
+ summarize(f"PyLate/{args.pylate_device}", pylate_times, args.candidates)
+
+ server_scores = [0.0] * args.candidates
+ for result in server_response["results"]:
+ server_scores[result["index"]] = float(result["relevance_score"])
+ worst = max(abs(a - b) for a, b in zip(server_scores, pylate_scores))
+ server_order = sorted(
+ range(args.candidates), key=server_scores.__getitem__, reverse=True
+ )
+ pylate_order = sorted(
+ range(args.candidates), key=pylate_scores.__getitem__, reverse=True
+ )
+ print(
+ f"parity: worst_score_diff={worst:.8g} "
+ f"top1_match={server_order[0] == pylate_order[0]}"
+ )
+ if worst > 1e-4 or server_order[0] != pylate_order[0]:
+ raise RuntimeError("server and PyLate score parity failed")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())