diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:45:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | bc8bebe78358e3a37bad0867b2d94955cac01d47 (patch) | |
| tree | d5a49a34b95b45cec0917a2a07707159825fb912 /bench/bench_contextual_api.py | |
| parent | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff) | |
| download | ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip | |
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/bench_contextual_api.py')
| -rwxr-xr-x | bench/bench_contextual_api.py | 209 |
1 files changed, 209 insertions, 0 deletions
diff --git a/bench/bench_contextual_api.py b/bench/bench_contextual_api.py new file mode 100755 index 0000000..2c0a558 --- /dev/null +++ b/bench/bench_contextual_api.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Benchmark contextual span inference through the local HTTP API.""" + +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import json +import subprocess +import statistics +import time +import urllib.error +import urllib.request + + +CHUNK_TEXT = ( + "Contextual embedding inference processes complete documents before pooling " + "individual chunk spans. This benchmark keeps the input deterministic while " + "stressing long bidirectional attention and compact chunk result handling. " +) + + +def percentile(values: list[float], p: float) -> float: + xs = sorted(values) + if len(xs) == 1: + return xs[0] + rank = (len(xs) - 1) * (p / 100.0) + lo = int(rank) + hi = min(lo + 1, len(xs) - 1) + frac = rank - lo + return xs[lo] * (1.0 - frac) + xs[hi] * frac + + +def parse_server_timing(value: str | None) -> dict[str, float]: + out: dict[str, float] = {} + if not value: + return out + for part in value.split(","): + fields = [x.strip() for x in part.strip().split(";")] + if not fields or not fields[0]: + continue + name = fields[0] + for field in fields[1:]: + if field.startswith("dur="): + try: + out[name] = float(field[4:]) + except ValueError: + pass + return out + + +def chunk_count(args: argparse.Namespace, doc_index: int) -> int: + return args.chunks * (doc_index + 1) if args.ragged else args.chunks + + +def total_chunks_per_request(args: argparse.Namespace) -> int: + return sum(chunk_count(args, di) for di in range(args.documents)) + + +def make_payload(args: argparse.Namespace, request_id: int) -> dict: + docs = [] + for di in range(args.documents): + chunks = [ + f"request {request_id} document {di} chunk {ci}: " + + CHUNK_TEXT * args.repeats_per_chunk + for ci in range(chunk_count(args, di)) + ] + docs.append(chunks) + return { + "model": args.model, + "dimensions": args.dimensions, + "input": docs, + } + + +def post_json(args: argparse.Namespace, request_id: int) -> tuple[float, int, dict[str, float]]: + payload = make_payload(args, request_id) + body = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + if args.api_key: + headers["Authorization"] = f"Bearer {args.api_key}" + req = urllib.request.Request( + args.base_url.rstrip("/") + "/v1/contextualizedembeddings", + data=body, + headers=headers, + ) + t0 = time.perf_counter() + try: + with urllib.request.urlopen(req, timeout=args.timeout) as resp: + timing = parse_server_timing(resp.headers.get("Server-Timing")) + data = json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + text = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {e.code}: {text}") from e + rows = data["data"] + if len(rows) != args.documents: + raise RuntimeError("unexpected contextual document count") + for di, row in enumerate(rows): + expected_chunks = chunk_count(args, di) + if row["index"] != di or len(row["data"]) != expected_chunks: + raise RuntimeError("unexpected contextual response shape") + for ci, chunk in enumerate(row["data"]): + if chunk["index"] != ci: + raise RuntimeError("unexpected contextual chunk index") + return (time.perf_counter() - t0) * 1000.0, int(data["usage"]["prompt_tokens"]), timing + + +def rss_mb(pid: int | None) -> float | None: + if not pid: + return None + try: + out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)], text=True) + kb = int(out.strip()) + return kb / 1024.0 + except Exception: + return None + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--base-url", default="http://127.0.0.1:8000") + ap.add_argument("--api-key") + ap.add_argument("--model", default="pplx-embed-context-v1-0.6b") + ap.add_argument("--dimensions", type=int, default=128) + ap.add_argument("--documents", type=int, default=1) + ap.add_argument("--chunks", type=int, default=8) + ap.add_argument("--repeats-per-chunk", type=int, default=8) + ap.add_argument("--runs", type=int, default=7) + ap.add_argument("--warmups", type=int, default=2) + ap.add_argument("--concurrency", type=int, default=1) + ap.add_argument("--timeout", type=float, default=300.0) + ap.add_argument("--server-pid", type=int) + ap.add_argument("--ragged", action="store_true", + help="scale chunk count by document index to benchmark padded contextual batches") + args = ap.parse_args() + if min(args.dimensions, args.documents, args.chunks, args.repeats_per_chunk, + args.runs, args.concurrency) <= 0: + raise SystemExit( + "dimensions, documents, chunks, repeats-per-chunk, runs, and " + "concurrency must be positive" + ) + if args.warmups < 0: + raise SystemExit("warmups must be >= 0") + + for i in range(args.warmups): + post_json(args, -i - 1) + + latencies = [] + timing_rows: list[dict[str, float]] = [] + wall_minus_worker: list[float] = [] + total_tokens = 0 + t0 = time.perf_counter() + with cf.ThreadPoolExecutor(max_workers=args.concurrency) as ex: + futures = [ex.submit(post_json, args, i) for i in range(args.runs)] + for future in cf.as_completed(futures): + elapsed, tokens, timing = future.result() + latencies.append(elapsed) + if timing: + timing_rows.append(timing) + if "worker" in timing: + wall_minus_worker.append(elapsed - timing["worker"]) + total_tokens += tokens + wall = time.perf_counter() - t0 + p50 = percentile(latencies, 50) + chunks_per_request = total_chunks_per_request(args) + print( + f"server={args.base_url} model={args.model} documents={args.documents} " + f"chunks/request={chunks_per_request} ragged={args.ragged} " + f"runs={args.runs} concurrency={args.concurrency}" + ) + print( + "latency_ms " + f"mean={statistics.fmean(latencies):.2f} " + f"p50={p50:.2f} " + f"p95={percentile(latencies, 95):.2f} " + f"max={max(latencies):.2f}" + ) + total_docs = args.runs * args.documents + total_chunks = args.runs * chunks_per_request + print( + f"wall={wall:.3f}s req/s={args.runs / wall:.2f} " + f"documents/s={total_docs / wall:.2f} chunks/s={total_chunks / wall:.2f} " + f"tokens/s={total_tokens / wall:.2f}" + ) + mem = rss_mb(args.server_pid) + if mem is not None: + print(f"rss_mb={mem:.1f}") + if timing_rows: + for name in ("queue", "parse", "tokenize", "infer", "encode", "worker"): + values = [row[name] for row in timing_rows if name in row] + if values: + print( + f"server_timing_{name}_ms " + f"mean={statistics.fmean(values):.3f} " + f"p50={percentile(values, 50):.3f} " + f"p95={percentile(values, 95):.3f}" + ) + if wall_minus_worker: + print( + "wall_minus_worker_ms " + f"mean={statistics.fmean(wall_minus_worker):.3f} " + f"p50={percentile(wall_minus_worker, 50):.3f} " + f"p95={percentile(wall_minus_worker, 95):.3f}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
