diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:45:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | bc8bebe78358e3a37bad0867b2d94955cac01d47 (patch) | |
| tree | d5a49a34b95b45cec0917a2a07707159825fb912 /bench | |
| parent | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff) | |
| download | ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip | |
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench')
| -rw-r--r-- | bench/Makefile | 73 | ||||
| -rw-r--r-- | bench/bench.h | 2 | ||||
| -rwxr-xr-x | bench/bench_contextual_api.py | 209 | ||||
| -rw-r--r-- | bench/bench_kernels.c | 27 | ||||
| -rwxr-xr-x | bench/bench_late_api.py | 202 | ||||
| -rw-r--r-- | bench/bench_late_rerank.c | 293 | ||||
| -rwxr-xr-x | bench/bench_late_rerank.py | 52 | ||||
| -rwxr-xr-x | bench/bench_matrix.py | 439 | ||||
| -rw-r--r-- | bench/bench_model.c | 40 | ||||
| -rwxr-xr-x | bench/bench_qwen_tokenizer.py | 59 | ||||
| -rwxr-xr-x | bench/bench_server_api.py | 274 | ||||
| -rw-r--r-- | bench/bench_tokenizer.c | 63 | ||||
| -rwxr-xr-x | bench/benchstat.py | 127 |
13 files changed, 1792 insertions, 68 deletions
diff --git a/bench/Makefile b/bench/Makefile new file mode 100644 index 0000000..9eeaa95 --- /dev/null +++ b/bench/Makefile @@ -0,0 +1,73 @@ +ROOT := .. +include ../build.mk + +CFLAGS += -std=c11 + +BLAS_LIBS := $(HOST_BLAS_LDFLAGS) + +# Benchmarks link the blas libffwd archive from the per-config build tree and +# include its headers. +LIBFFWD_DIR := ../libffwd +LIBFFWD_A := $(ROOT)/build/$(MODE)/blas/libffwd.a +SERVER_DIR := ../tools/server + +RESULTS := results +# Commit, a -dirty marker when the tree has uncommitted changes, and a timestamp, +# so repeated runs at the same commit do not overwrite each other. +STAMP ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S) + +BENCHES := bench_kernels bench_tokenizer bench_server_utils bench_model \ + bench_late_rerank + +.PHONY: all bench bench-model bench-tokenizer bench-server-utils clean \ + libffwd-blas + +all: $(BENCHES) + +# The archive is built with a backend-specific flag, so a phony target (not a +# file rule) drives the sub-make; the benchmarks list it as an order prerequisite +# and link the resulting $(LIBFFWD_A). +libffwd-blas: + $(MAKE) -C $(LIBFFWD_DIR) BACKEND=blas + +$(eval $(call c_program_deps,bench_kernels,bench_kernels.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)) + +$(eval $(call c_program_deps,bench_tokenizer,bench_tokenizer.c,libffwd-blas,$(LIBFFWD_A),-I$(LIBFFWD_DIR),-lm)) + +# The server utilities (base64, sbuf, util) depend on no library, so the bench +# compiles those sources directly here and keeps the build self-contained. +SERVER_UTIL_SRCS := $(SERVER_DIR)/base64.c $(SERVER_DIR)/sbuf.c $(SERVER_DIR)/util.c +$(eval $(call c_program_deps,bench_server_utils,bench_server_utils.c $(SERVER_UTIL_SRCS),bench.h,,-I$(SERVER_DIR),-lm)) + +$(eval $(call c_program_deps,bench_model,bench_model.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)) + +$(eval $(call c_program_deps,bench_late_rerank,bench_late_rerank.c,libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)) + +# MLX rerank benchmark: the blas engine archive for the CPU reference paths plus +# mlx.c for the device API. The archive calls cblas, so the bench links +# Accelerate/OpenBLAS itself ($(HOST_BLAS_LDFLAGS)) on top of Metal/MLX. macOS only. +MLX_DRIVER_CFLAGS := -pthread -DCHECK_MLX -I$(MLXC_PREFIX)/include -I$(LIBFFWD_DIR) $(HOST_BLAS_CFLAGS) +MLX_DRIVER_LDLIBS := $(MLX_BACKEND_LDFLAGS) $(HOST_BLAS_LDFLAGS) -lm +$(eval $(call c_program_deps,bench_late_rerank_mlx,bench_late_rerank.c $(LIBFFWD_DIR)/mlx.c,libffwd-blas,$(LIBFFWD_A),$(MLX_DRIVER_CFLAGS),$(MLX_DRIVER_LDLIBS))) + +bench: bench_kernels + @mkdir -p $(RESULTS) + ./bench_kernels --json $(RESULTS)/kernels-$(STAMP).json + @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json" + +bench-server-utils: bench_server_utils + @mkdir -p $(RESULTS) + ./bench_server_utils --json $(RESULTS)/server-utils-$(STAMP).json + @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json" + +bench-model: bench_model + @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; } + @mkdir -p $(RESULTS) + ./bench_model "$(MODEL_DIR)" --json $(RESULTS)/model-$(STAMP).json + @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json" + +bench-tokenizer: bench_tokenizer + @echo "run: ./bench_tokenizer <model_dir-or-vocab.json> <runs> TEXT..." + +clean: + $(RM) -r $(BENCHES) bench_late_rerank_mlx *.dSYM diff --git a/bench/bench.h b/bench/bench.h index 1d2b8f2..8be685c 100644 --- a/bench/bench.h +++ b/bench/bench.h @@ -8,7 +8,7 @@ * float *x = make_input(); // setup, not measured * bench_begin(b); * for (long i = 0; i < b->n; i++) - * bench_sink += embed_dot_f32(x, x, 1024); + * bench_sink += ffwd_dot_f32(x, x, 1024); * free(x); * } * diff --git a/bench/bench_contextual_api.py b/bench/bench_contextual_api.py new file mode 100755 index 0000000..2c0a558 --- /dev/null +++ b/bench/bench_contextual_api.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Benchmark contextual span inference through the local HTTP API.""" + +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import json +import subprocess +import statistics +import time +import urllib.error +import urllib.request + + +CHUNK_TEXT = ( + "Contextual embedding inference processes complete documents before pooling " + "individual chunk spans. This benchmark keeps the input deterministic while " + "stressing long bidirectional attention and compact chunk result handling. " +) + + +def percentile(values: list[float], p: float) -> float: + xs = sorted(values) + if len(xs) == 1: + return xs[0] + rank = (len(xs) - 1) * (p / 100.0) + lo = int(rank) + hi = min(lo + 1, len(xs) - 1) + frac = rank - lo + return xs[lo] * (1.0 - frac) + xs[hi] * frac + + +def parse_server_timing(value: str | None) -> dict[str, float]: + out: dict[str, float] = {} + if not value: + return out + for part in value.split(","): + fields = [x.strip() for x in part.strip().split(";")] + if not fields or not fields[0]: + continue + name = fields[0] + for field in fields[1:]: + if field.startswith("dur="): + try: + out[name] = float(field[4:]) + except ValueError: + pass + return out + + +def chunk_count(args: argparse.Namespace, doc_index: int) -> int: + return args.chunks * (doc_index + 1) if args.ragged else args.chunks + + +def total_chunks_per_request(args: argparse.Namespace) -> int: + return sum(chunk_count(args, di) for di in range(args.documents)) + + +def make_payload(args: argparse.Namespace, request_id: int) -> dict: + docs = [] + for di in range(args.documents): + chunks = [ + f"request {request_id} document {di} chunk {ci}: " + + CHUNK_TEXT * args.repeats_per_chunk + for ci in range(chunk_count(args, di)) + ] + docs.append(chunks) + return { + "model": args.model, + "dimensions": args.dimensions, + "input": docs, + } + + +def post_json(args: argparse.Namespace, request_id: int) -> tuple[float, int, dict[str, float]]: + payload = make_payload(args, request_id) + body = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + if args.api_key: + headers["Authorization"] = f"Bearer {args.api_key}" + req = urllib.request.Request( + args.base_url.rstrip("/") + "/v1/contextualizedembeddings", + data=body, + headers=headers, + ) + t0 = time.perf_counter() + try: + with urllib.request.urlopen(req, timeout=args.timeout) as resp: + timing = parse_server_timing(resp.headers.get("Server-Timing")) + data = json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + text = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {e.code}: {text}") from e + rows = data["data"] + if len(rows) != args.documents: + raise RuntimeError("unexpected contextual document count") + for di, row in enumerate(rows): + expected_chunks = chunk_count(args, di) + if row["index"] != di or len(row["data"]) != expected_chunks: + raise RuntimeError("unexpected contextual response shape") + for ci, chunk in enumerate(row["data"]): + if chunk["index"] != ci: + raise RuntimeError("unexpected contextual chunk index") + return (time.perf_counter() - t0) * 1000.0, int(data["usage"]["prompt_tokens"]), timing + + +def rss_mb(pid: int | None) -> float | None: + if not pid: + return None + try: + out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)], text=True) + kb = int(out.strip()) + return kb / 1024.0 + except Exception: + return None + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--base-url", default="http://127.0.0.1:8000") + ap.add_argument("--api-key") + ap.add_argument("--model", default="pplx-embed-context-v1-0.6b") + ap.add_argument("--dimensions", type=int, default=128) + ap.add_argument("--documents", type=int, default=1) + ap.add_argument("--chunks", type=int, default=8) + ap.add_argument("--repeats-per-chunk", type=int, default=8) + ap.add_argument("--runs", type=int, default=7) + ap.add_argument("--warmups", type=int, default=2) + ap.add_argument("--concurrency", type=int, default=1) + ap.add_argument("--timeout", type=float, default=300.0) + ap.add_argument("--server-pid", type=int) + ap.add_argument("--ragged", action="store_true", + help="scale chunk count by document index to benchmark padded contextual batches") + args = ap.parse_args() + if min(args.dimensions, args.documents, args.chunks, args.repeats_per_chunk, + args.runs, args.concurrency) <= 0: + raise SystemExit( + "dimensions, documents, chunks, repeats-per-chunk, runs, and " + "concurrency must be positive" + ) + if args.warmups < 0: + raise SystemExit("warmups must be >= 0") + + for i in range(args.warmups): + post_json(args, -i - 1) + + latencies = [] + timing_rows: list[dict[str, float]] = [] + wall_minus_worker: list[float] = [] + total_tokens = 0 + t0 = time.perf_counter() + with cf.ThreadPoolExecutor(max_workers=args.concurrency) as ex: + futures = [ex.submit(post_json, args, i) for i in range(args.runs)] + for future in cf.as_completed(futures): + elapsed, tokens, timing = future.result() + latencies.append(elapsed) + if timing: + timing_rows.append(timing) + if "worker" in timing: + wall_minus_worker.append(elapsed - timing["worker"]) + total_tokens += tokens + wall = time.perf_counter() - t0 + p50 = percentile(latencies, 50) + chunks_per_request = total_chunks_per_request(args) + print( + f"server={args.base_url} model={args.model} documents={args.documents} " + f"chunks/request={chunks_per_request} ragged={args.ragged} " + f"runs={args.runs} concurrency={args.concurrency}" + ) + print( + "latency_ms " + f"mean={statistics.fmean(latencies):.2f} " + f"p50={p50:.2f} " + f"p95={percentile(latencies, 95):.2f} " + f"max={max(latencies):.2f}" + ) + total_docs = args.runs * args.documents + total_chunks = args.runs * chunks_per_request + print( + f"wall={wall:.3f}s req/s={args.runs / wall:.2f} " + f"documents/s={total_docs / wall:.2f} chunks/s={total_chunks / wall:.2f} " + f"tokens/s={total_tokens / wall:.2f}" + ) + mem = rss_mb(args.server_pid) + if mem is not None: + print(f"rss_mb={mem:.1f}") + if timing_rows: + for name in ("queue", "parse", "tokenize", "infer", "encode", "worker"): + values = [row[name] for row in timing_rows if name in row] + if values: + print( + f"server_timing_{name}_ms " + f"mean={statistics.fmean(values):.3f} " + f"p50={percentile(values, 50):.3f} " + f"p95={percentile(values, 95):.3f}" + ) + if wall_minus_worker: + print( + "wall_minus_worker_ms " + f"mean={statistics.fmean(wall_minus_worker):.3f} " + f"p50={percentile(wall_minus_worker, 50):.3f} " + f"p95={percentile(wall_minus_worker, 95):.3f}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c index 2b2fe47..55f859b 100644 --- a/bench/bench_kernels.c +++ b/bench/bench_kernels.c @@ -4,16 +4,15 @@ * 8 KV heads x head_dim 128, intermediate 3072) so a slowdown here predicts * a slowdown in inference. Single-threaded by default for stable numbers; * --threads N opts into the pool. Build and record via `make bench`, - * compare records with tools/benchstat.py. + * compare records with bench/benchstat.py. */ #include "bench.h" #include "kernels.h" +#include "threadpool.h" #include <stdint.h> -int embed_verbose = 0; /* defined by embed.c in full builds */ - enum { HIDDEN = 1024, Q_DIM = 2048, /* 16 heads * 128 */ @@ -71,7 +70,7 @@ static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) { float *y = alloc_f32((size_t)seq * out, 3); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_linear_nobias(y, x, w, seq, in, out); + linear_nobias(y, x, w, seq, in, out); bench_sink += y[0]; free(y); free(w); @@ -91,7 +90,7 @@ static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) { float *y = alloc_f32((size_t)seq * out, 6); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_linear_nobias_bf16(y, x, w, seq, in, out); + linear_nobias_bf16(y, x, w, seq, in, out); bench_sink += y[0]; free(y); free(w); @@ -111,7 +110,7 @@ static void bm_qkv_bf16(bench_state_t *b) { float *v = alloc_f32((size_t)SEQ * KV_DIM, 13); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM); + linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM); bench_sink += q[0] + k[0] + v[0]; free(v); free(k); @@ -128,7 +127,7 @@ static void bm_bf16_widen(bench_state_t *b) { float *dst = alloc_f32(n, 15); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_bf16_to_f32_buf(dst, src, n); + bf16_to_f32_buf(dst, src, n); bench_sink += dst[0]; free(dst); free(src); @@ -143,7 +142,7 @@ static void bm_attn_scores(bench_state_t *b) { float *s = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 18); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ); + matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ); bench_sink += s[0]; free(s); free(km); @@ -154,7 +153,7 @@ static void bm_softmax(bench_state_t *b) { float *x = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 19); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_softmax(x, ATTN_SEQ, ATTN_SEQ); + softmax(x, ATTN_SEQ, ATTN_SEQ); bench_sink += x[0]; free(x); } @@ -165,7 +164,7 @@ static void bm_rope(bench_state_t *b) { float *sinv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 22); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM); + apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM); bench_sink += x[0]; free(sinv); free(cosv); @@ -180,7 +179,7 @@ static void bm_rms_norm(bench_state_t *b) { float *y = alloc_f32((size_t)SEQ * HIDDEN, 25); bench_begin(b); for (long i = 0; i < b->n; i++) - embed_rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f); + rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f); bench_sink += y[0]; free(y); free(w); @@ -197,7 +196,7 @@ static void bm_silu_mul(bench_state_t *b) { bench_begin(b); for (long i = 0; i < b->n; i++) { memcpy(gate, gate0, n * sizeof(float)); - embed_silu_mul_inplace(gate, up, (int)n); + silu_mul_inplace(gate, up, (int)n); } bench_sink += gate[0]; free(up); @@ -210,7 +209,7 @@ static void bm_dot(bench_state_t *b) { float *y = alloc_f32(HIDDEN, 29); bench_begin(b); for (long i = 0; i < b->n; i++) - bench_sink += embed_dot_f32(x, y, HIDDEN); + bench_sink += dot_f32(x, y, HIDDEN); free(y); free(x); } @@ -240,7 +239,7 @@ int main(int argc, char **argv) { } if (threads < 1) threads = 1; - embed_set_threads(threads); + tp_set_threads(threads); char meta[128]; #ifdef USE_BLAS diff --git a/bench/bench_late_api.py b/bench/bench_late_api.py new file mode 100755 index 0000000..7a6073f --- /dev/null +++ b/bench/bench_late_api.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Benchmark /v1/rerank and optionally compare end-to-end with PyLate.""" + +from __future__ import annotations + +import argparse +import json +import statistics +import time +import urllib.error +import urllib.request +from typing import Any + + +BASE_DOCUMENTS = [ + "Scientists explore the universe driven by curiosity.", + "Children learn through curious exploration.", + "Historical discoveries began with careful questions.", + "Redis stores data structures in memory for fast access.", + "SQLite keeps relational data in a local database file.", + "Embedding models map text into semantic vector spaces.", + "Paris is the capital of France and a major European destination.", + "Neural search compares query and document token representations.", +] +QUERY = "What motivates scientific discovery?" + + +def percentile(values: list[float], p: float) -> float: + values = sorted(values) + rank = (len(values) - 1) * p / 100 + lo = int(rank) + hi = min(lo + 1, len(values) - 1) + frac = rank - lo + return values[lo] * (1 - frac) + values[hi] * frac + + +def make_documents(candidates: int, repeat: int) -> list[str]: + return [ + " ".join( + BASE_DOCUMENTS[(candidate + offset) % len(BASE_DOCUMENTS)] + for offset in range(repeat) + ) + for candidate in range(candidates) + ] + + +def post_json( + base_url: str, + payload: dict[str, Any], + api_key: str | None, + timeout: float, +) -> dict[str, Any]: + headers = {"Content-Type": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + req = urllib.request.Request( + base_url.rstrip("/") + "/v1/rerank", + data=json.dumps(payload).encode(), + headers=headers, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as exc: + text = exc.read().decode(errors="replace") + raise RuntimeError(f"HTTP {exc.code}: {text}") from exc + + +def summarize(name: str, values: list[float], candidates: int) -> None: + mean = statistics.fmean(values) + print( + f"{name}: mean_ms={mean:.2f} p50_ms={percentile(values, 50):.2f} " + f"p95_ms={percentile(values, 95):.2f} " + f"candidates/s={candidates * 1000 / mean:.2f}" + ) + + +def synchronize(device: str) -> None: + if device == "mps": + import torch + + torch.mps.synchronize() + elif device.startswith("cuda"): + import torch + + torch.cuda.synchronize() + + +def pylate_run(model: Any, query: str, documents: list[str], device: str) -> list[float]: + from pylate.scores import colbert_scores + + query_vectors = model.encode( + [query], + is_query=True, + convert_to_tensor=True, + show_progress_bar=False, + )[0] + document_vectors = model.encode( + documents, + is_query=False, + convert_to_tensor=True, + show_progress_bar=False, + ) + scores = [ + float( + colbert_scores( + query_vectors.unsqueeze(0), + document.unsqueeze(0), + )[0, 0] + ) + for document in document_vectors + ] + synchronize(device) + return scores + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--base-url", default="http://127.0.0.1:8000") + ap.add_argument("--api-key") + ap.add_argument("--model", default="pplx-embed-v1-late-0.6b") + ap.add_argument("--model-dir") + ap.add_argument("--candidates", type=int, default=32) + ap.add_argument("--document-repeat", type=int, default=2) + ap.add_argument("--runs", type=int, default=5) + ap.add_argument("--warmups", type=int, default=1) + ap.add_argument("--timeout", type=float, default=600) + ap.add_argument("--compare-pylate", action="store_true") + ap.add_argument("--pylate-device", default="mps") + args = ap.parse_args() + if min(args.candidates, args.document_repeat, args.runs) <= 0: + raise SystemExit("candidates, document-repeat, and runs must be positive") + if args.warmups < 0: + raise SystemExit("warmups must be non-negative") + if args.compare_pylate and not args.model_dir: + raise SystemExit("--model-dir is required with --compare-pylate") + + documents = make_documents(args.candidates, args.document_repeat) + payload = { + "model": args.model, + "query": QUERY, + "documents": documents, + } + for _ in range(args.warmups): + post_json(args.base_url, payload, args.api_key, args.timeout) + server_times = [] + server_response: dict[str, Any] = {} + for _ in range(args.runs): + start = time.perf_counter() + server_response = post_json( + args.base_url, payload, args.api_key, args.timeout + ) + server_times.append((time.perf_counter() - start) * 1000) + summarize("pplx-embed-server", server_times, args.candidates) + + if args.compare_pylate: + try: + from pylate import models + except ImportError as exc: + raise SystemExit( + "PyLate is missing; run with `uv run --with pylate " + "scripts/bench_late_api.py ...`" + ) from exc + + model = models.ColBERT( + model_name_or_path=args.model_dir, + trust_remote_code=True, + device=args.pylate_device, + ) + for _ in range(args.warmups): + pylate_run(model, QUERY, documents, args.pylate_device) + pylate_times = [] + pylate_scores: list[float] = [] + for _ in range(args.runs): + start = time.perf_counter() + pylate_scores = pylate_run( + model, QUERY, documents, args.pylate_device + ) + pylate_times.append((time.perf_counter() - start) * 1000) + summarize(f"PyLate/{args.pylate_device}", pylate_times, args.candidates) + + server_scores = [0.0] * args.candidates + for result in server_response["results"]: + server_scores[result["index"]] = float(result["relevance_score"]) + worst = max(abs(a - b) for a, b in zip(server_scores, pylate_scores)) + server_order = sorted( + range(args.candidates), key=server_scores.__getitem__, reverse=True + ) + pylate_order = sorted( + range(args.candidates), key=pylate_scores.__getitem__, reverse=True + ) + print( + f"parity: worst_score_diff={worst:.8g} " + f"top1_match={server_order[0] == pylate_order[0]}" + ) + if worst > 1e-4 or server_order[0] != pylate_order[0]: + raise RuntimeError("server and PyLate score parity failed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/bench_late_rerank.c b/bench/bench_late_rerank.c new file mode 100644 index 0000000..f6e7734 --- /dev/null +++ b/bench/bench_late_rerank.c @@ -0,0 +1,293 @@ +/* bench/bench_late_rerank.c - late-interaction MaxSim rerank benchmark. + * Build via `make bench-late-rerank`; run with: + * ./bench_late_rerank MODEL_DIR CANDIDATES DOC_REPEAT RUNS + * Encodes a query and N candidate documents to token vectors, then times the + * packed MaxSim rerank. check it against bench_late_rerank.py, which builds and + * runs this binary. */ +#include "internal.h" +#ifdef CHECK_MLX +# include "mlx.h" +#endif +#include "bpe.h" + +#include <float.h> +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#define QUERY_LENGTH 32 +#define MASK_TOKEN_ID 151642 +#define QUERY_PREFIX_ID 151669 +#define DOC_PREFIX_ID 151670 + +typedef struct { + int *ids; + int n_ids; + float *vecs; + int n_vecs; +#ifdef CHECK_MLX + ffwd_mlx_late_vectors_t *dev; +#endif +} late_item_t; + +#ifdef CHECK_MLX +typedef ffwd_mlx_late_ctx_t test_model_t; +typedef void test_workspace_t; +static test_model_t *test_model_load(const char *dir) { return ffwd_mlx_late_load(dir); } +static void test_model_free(test_model_t *m) { ffwd_mlx_late_free(m); } +static test_workspace_t *test_workspace_new(test_model_t *m) { + (void)m; + return (test_workspace_t *)1; +} +static void test_workspace_free(test_workspace_t *ws) { (void)ws; } +static const ffwd_config_t *test_config(test_model_t *m) { return ffwd_mlx_late_config(m); } +static int test_token_dim(test_model_t *m) { return ffwd_mlx_late_token_dim(m); } +static int test_encode( + test_model_t *m, test_workspace_t *ws, const int *ids, int n_ids, int normalize, float *out) { + (void)ws; + return ffwd_mlx_late_encode_tokens(m, ids, n_ids, normalize, out); +} +#else +typedef ffwd_late_model_t test_model_t; +typedef ffwd_late_workspace_t test_workspace_t; +static test_model_t *test_model_load(const char *dir) { return ffwd_late_model_load(dir); } +static void test_model_free(test_model_t *m) { ffwd_late_model_free(m); } +static test_workspace_t *test_workspace_new(test_model_t *m) { return ffwd_late_workspace_new(m); } +static void test_workspace_free(test_workspace_t *ws) { ffwd_late_workspace_free(ws); } +static const ffwd_config_t *test_config(test_model_t *m) { return ffwd_late_model_config(m); } +static int test_token_dim(test_model_t *m) { return ffwd_late_model_token_dim(m); } +static int test_encode( + test_model_t *m, test_workspace_t *ws, const int *ids, int n_ids, int normalize, float *out) { + return ffwd_late_model_encode_tokens(m, ws, ids, n_ids, normalize, out); +} +#endif + +static double now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0; +} + +static int encode_with_prefix_id( + tok_bpe_t *tok, int prefix_id, const char *text, int expand_query, late_item_t *out) { + out->ids = tok_bpe_encode(tok, text, &out->n_ids); + if (!out->ids || out->n_ids <= 0) + return -1; + + int target_len = out->n_ids + 1; + if (expand_query && target_len < QUERY_LENGTH) + target_len = QUERY_LENGTH; + + int *tmp = (int *)realloc(out->ids, (size_t)target_len * sizeof(int)); + if (!tmp) + return -1; + out->ids = tmp; + + memmove(out->ids + 2, out->ids + 1, (size_t)(out->n_ids - 1) * sizeof(int)); + out->ids[1] = prefix_id; + out->n_ids++; + + if (expand_query && out->n_ids < QUERY_LENGTH) { + for (int i = out->n_ids; i < QUERY_LENGTH; i++) + out->ids[i] = MASK_TOKEN_ID; + out->n_ids = QUERY_LENGTH; + } + return 0; +} + +static int encode_late_item(test_model_t *model, test_workspace_t *ws, late_item_t *item, int dim) { + item->n_vecs = item->n_ids; + item->vecs = (float *)malloc((size_t)item->n_vecs * dim * sizeof(float)); + if (!item->vecs) + return -1; + if (test_encode(model, ws, item->ids, item->n_ids, 1, item->vecs) != 0) + return -1; +#ifdef CHECK_MLX + item->dev = ffwd_mlx_late_encode_tokens_device(model, item->ids, item->n_ids, 1); + if (!item->dev) + return -1; +#endif + return 0; +} + +static void free_item(late_item_t *item) { +#ifdef CHECK_MLX + ffwd_mlx_late_vectors_free(item->dev); +#endif + free(item->ids); + free(item->vecs); +} + +static char *make_doc_text(int index, int repeat) { + static const char *base[] = {"Scientists explore the universe driven by curiosity", + "Children learn through curious exploration", + "Historical discoveries began with careful questions", + "Redis stores data structures in memory for fast access", + "SQLite keeps relational data in a local database file", + "Embedding models map text into semantic vector spaces", + "Paris is a capital city and a major European destination", + "Neural search systems compare query and document representations"}; + enum { NBASE = 8 }; + size_t cap = 64; + for (int i = 0; i < repeat; i++) + cap += strlen(base[(index + i) % NBASE]) + 2; + char *out = (char *)malloc(cap); + if (!out) + return NULL; + out[0] = '\0'; + for (int i = 0; i < repeat; i++) { + if (i) + strcat(out, ". "); + strcat(out, base[(index + i) % NBASE]); + } + return out; +} + +int main(int argc, char **argv) { + if (argc != 5) { + fprintf(stderr, "usage: %s MODEL_DIR CANDIDATES DOC_REPEAT RUNS\n", argv[0]); + return 2; + } + const char *model_dir = argv[1]; + int candidates = atoi(argv[2]); + int doc_repeat = atoi(argv[3]); + int runs = atoi(argv[4]); + if (candidates <= 0 || doc_repeat <= 0 || runs <= 0) + return 2; + + char vocab_path[4096]; + snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); + tok_bpe_t *tok = tok_bpe_load(vocab_path); + test_model_t *model = test_model_load(model_dir); + test_workspace_t *ws = test_workspace_new(model); + if (!tok || !model || !ws) { + fprintf(stderr, "failed to initialize late benchmark\n"); + return 1; + } + + int dim = test_token_dim(model); + const ffwd_config_t *cfg = test_config(model); + if (dim <= 0 || !cfg) { + fprintf(stderr, "bad late model config\n"); + return 1; + } + + late_item_t q = {0}; + late_item_t *docs = (late_item_t *)calloc((size_t)candidates, sizeof(*docs)); + int *offsets = (int *)calloc((size_t)candidates + 1, sizeof(*offsets)); + float *scores = (float *)malloc((size_t)candidates * sizeof(float)); + float *device_scores = (float *)malloc((size_t)candidates * sizeof(float)); + if (!docs || !offsets || !scores || !device_scores) + return 1; + + if (encode_with_prefix_id(tok, QUERY_PREFIX_ID, "What motivates scientific discovery?", 1, &q) != + 0 || + encode_late_item(model, ws, &q, dim) != 0) + return 1; + + int total_doc_tokens = 0; + offsets[0] = 0; + for (int i = 0; i < candidates; i++) { + char *text = make_doc_text(i, doc_repeat); + if (!text) + return 1; + int rc = encode_with_prefix_id(tok, DOC_PREFIX_ID, text, 0, &docs[i]); + free(text); + if (rc != 0 || encode_late_item(model, ws, &docs[i], dim) != 0) + return 1; + total_doc_tokens += docs[i].n_vecs; + offsets[i + 1] = total_doc_tokens; + } + + float *packed = (float *)malloc((size_t)total_doc_tokens * dim * sizeof(float)); + if (!packed) + return 1; + int pos = 0; + for (int i = 0; i < candidates; i++) { + memcpy(packed + (size_t)pos * dim, docs[i].vecs, + (size_t)docs[i].n_vecs * dim * sizeof(float)); + pos += docs[i].n_vecs; + } + + if (ffwd_late_maxsim_batch(q.vecs, q.n_vecs, packed, offsets, candidates, dim, scores) != 0) + return 1; + +#ifdef CHECK_MLX + const ffwd_mlx_late_vectors_t **dev_docs = + (const ffwd_mlx_late_vectors_t **)malloc((size_t)candidates * sizeof(*dev_docs)); + if (!dev_docs) + return 1; + for (int i = 0; i < candidates; i++) + dev_docs[i] = docs[i].dev; + ffwd_mlx_late_vectors_t *packed_dev = + ffwd_mlx_late_vectors_concat(model, dev_docs, candidates); + free(dev_docs); + if (!packed_dev) + return 1; + if (ffwd_mlx_late_maxsim_batch_device(model, q.dev, packed_dev, offsets, candidates, + device_scores) != 0) + return 1; + float worst_diff = 0.0f; + for (int i = 0; i < candidates; i++) { + float d = fabsf(scores[i] - device_scores[i]); + if (d > worst_diff) + worst_diff = d; + if (d > 3e-4f) { + fprintf(stderr, "device score mismatch at %d: %.9g %.9g\n", i, device_scores[i], + scores[i]); + return 1; + } + } +#endif + + double start = now_ms(); + for (int r = 0; r < runs; r++) { + if (ffwd_late_maxsim_batch(q.vecs, q.n_vecs, packed, offsets, candidates, dim, scores) != + 0) + return 1; + } + double cpu_ms = now_ms() - start; + + printf("late_rerank backend=%s candidates=%d query_tokens=%d " + "doc_tokens_total=%d doc_tokens_avg=%.1f dim=%d runs=%d\n", +#ifdef CHECK_MLX + "mlx", +#else + "cpu", +#endif + candidates, q.n_vecs, total_doc_tokens, (double)total_doc_tokens / (double)candidates, dim, + runs); + printf("cpu_packed_maxsim_ms %.6f candidates/s %.0f token_pairs/s %.0f\n", cpu_ms / (double)runs, + ((double)runs * candidates) / (cpu_ms / 1000.0), + ((double)runs * q.n_vecs * total_doc_tokens) / (cpu_ms / 1000.0)); + +#ifdef CHECK_MLX + start = now_ms(); + for (int r = 0; r < runs; r++) { + if (ffwd_mlx_late_maxsim_batch_device(model, q.dev, packed_dev, offsets, candidates, + device_scores) != 0) + return 1; + } + double mlx_ms = now_ms() - start; + printf("mlx_device_maxsim_ms %.6f candidates/s %.0f token_pairs/s %.0f " + "max_diff %.8g\n", + mlx_ms / (double)runs, ((double)runs * candidates) / (mlx_ms / 1000.0), + ((double)runs * q.n_vecs * total_doc_tokens) / (mlx_ms / 1000.0), worst_diff); + ffwd_mlx_late_vectors_free(packed_dev); +#endif + + for (int i = 0; i < candidates; i++) + free_item(&docs[i]); + free_item(&q); + free(packed); + free(device_scores); + free(scores); + free(offsets); + free(docs); + test_workspace_free(ws); + test_model_free(model); + tok_bpe_free(tok); + return 0; +} diff --git a/bench/bench_late_rerank.py b/bench/bench_late_rerank.py new file mode 100755 index 0000000..99bb817 --- /dev/null +++ b/bench/bench_late_rerank.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Benchmark late-interaction MaxSim reranking shapes. + +Builds the C bench (bench/bench_late_rerank.c) via the Makefile and runs it +against a late model directory (e.g. pplx-embed-v1-late-0.6b). python3 only. + + bench/bench_late_rerank.py --model-dir DIR +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def build(cc: str, backend: str) -> Path: + """Build the late-rerank bench via the Makefile; return its path.""" + target = "bench_late_rerank_mlx" if backend == "mlx" else "bench_late_rerank" + proc = subprocess.run(["make", "-C", "bench", f"CC={cc}", target], cwd=ROOT, + text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + raise SystemExit(proc.returncode) + return ROOT / "bench" / target + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--backend", choices=("cpu", "mlx"), default="cpu") + ap.add_argument("--candidates", type=int, default=32) + ap.add_argument("--doc-repeat", type=int, default=3) + ap.add_argument("--runs", type=int, default=200) + ap.add_argument("--cc", default=os.environ.get("CC", "cc")) + args = ap.parse_args() + + binary = build(args.cc, args.backend) + return subprocess.run( + [str(binary), args.model_dir, str(args.candidates), + str(args.doc_repeat), str(args.runs)], + cwd=ROOT, + ).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/bench_matrix.py b/bench/bench_matrix.py new file mode 100755 index 0000000..b302100 --- /dev/null +++ b/bench/bench_matrix.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +"""Batch-aware benchmark matrix for the stdin backends.""" + +import argparse +import datetime as dt +import json +import os +import platform +import shlex +import statistics +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +try: + import psutil +except Exception: # pragma: no cover - optional dependency + psutil = None + + +# A case is a target token count (an integer). The input is synthesized to about +# that many common words (~1 token each); the tokenizer reports the actual count +# per run, so the case is an explicit, approximate length target. +_FILLER = ( + "embedding models convert text into dense vectors that capture semantic " + "meaning for search clustering classification and retrieval augmented " + "generation across many production systems languages and domains where " + "latency throughput memory bandwidth and matrix multiplication all matter" +).split() + + +def make_text(n_tokens): + n = max(1, int(n_tokens)) + return " ".join(_FILLER[i % len(_FILLER)] for i in range(n)) + + +def parse_csv(value, cast=str): + items = [] + for raw in value.split(","): + raw = raw.strip() + if raw: + items.append(cast(raw)) + return items + + +def percentile(values, p): + if not values: + return None + xs = sorted(values) + if len(xs) == 1: + return xs[0] + rank = (len(xs) - 1) * (p / 100.0) + lo = int(rank) + hi = min(lo + 1, len(xs) - 1) + frac = rank - lo + return xs[lo] * (1.0 - frac) + xs[hi] * frac + + +def summarize(values): + return { + "min_ms": min(values), + "mean_ms": statistics.fmean(values), + "p50_ms": percentile(values, 50), + "p90_ms": percentile(values, 90), + "p95_ms": percentile(values, 95), + "p99_ms": percentile(values, 99), + "max_ms": max(values), + } + + +def make_batch_texts(n_tokens, batch_size): + return [make_text(n_tokens) for _ in range(batch_size)] + + +class StdinRunner: + def __init__(self, binary, model_dir, backend, batch_size, threads, + binary_args=None, backend_args=None): + self.binary = binary + self.model_dir = model_dir + self.backend = backend + self.batch_size = batch_size + self.threads = threads + self.binary_args = list(binary_args or []) + self.backend_args = list(backend_args or []) + self.proc = None + self.ps = None + self.load_ms = None + + def start(self): + cmd = [self.binary, "-d", self.model_dir, + *self.binary_args, *self.backend_args, + "--stream", "-b", str(self.batch_size)] + if self.backend == "cpu" and self.threads: + cmd.extend(["-t", str(self.threads)]) + + t0 = time.perf_counter() + self.proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + if psutil: + self.ps = psutil.Process(self.proc.pid) + + warmup = make_batch_texts(16, self.batch_size) + self.send_batch(warmup) + self.load_ms = (time.perf_counter() - t0) * 1000.0 + + def rss_mb(self): + if self.ps: + try: + return self.ps.memory_info().rss / (1024 * 1024) + except Exception: + pass + if not self.proc: + return None + try: + proc = subprocess.run( + ["ps", "-o", "rss=", "-p", str(self.proc.pid)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + value = proc.stdout.strip() if proc.returncode == 0 else "" + return (float(value) / 1024.0) if value else None + except Exception: + return None + + def send_batch(self, texts): + if len(texts) != self.batch_size: + raise ValueError("send_batch requires exactly batch_size texts") + if not self.proc or not self.proc.stdin or not self.proc.stdout: + raise RuntimeError("stdin is not running") + + t0 = time.perf_counter() + self.proc.stdin.write("\n".join(texts) + "\n") + self.proc.stdin.flush() + + rows = [] + for _ in texts: + line = self.proc.stdout.readline() + if line == "": + stderr = self.proc.stderr.read() if self.proc.stderr else "" + raise RuntimeError(f"stdin exited early\n{stderr}") + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin returned error: {row}") + rows.append(row) + + wall_ms = (time.perf_counter() - t0) * 1000.0 + engine_ms = float(rows[0]["ms"]) + return rows, engine_ms, wall_ms + + def stop(self): + if not self.proc: + return + try: + if self.proc.stdin: + self.proc.stdin.close() + self.proc.wait(timeout=5) + finally: + self.proc = None + + +def read_model_config(model_dir): + with open(os.path.join(model_dir, "config.json"), "r", encoding="utf-8") as f: + cfg = json.load(f) + return { + "hidden_size": cfg.get("hidden_size"), + "num_hidden_layers": cfg.get("num_hidden_layers"), + "num_attention_heads": cfg.get("num_attention_heads"), + "num_key_value_heads": cfg.get("num_key_value_heads"), + "intermediate_size": cfg.get("intermediate_size"), + "vocab_size": cfg.get("vocab_size"), + } + + +def run_case(stdin_runner, case_name, runs, warmups): + texts = make_batch_texts(case_name, stdin_runner.batch_size) + + for _ in range(warmups): + stdin_runner.send_batch(texts) + + engine_times = [] + wall_times = [] + tokens_per_text = None + dim = None + workspace_bytes = None + for _ in range(runs): + rows, engine_ms, wall_ms = stdin_runner.send_batch(texts) + engine_times.append(engine_ms) + wall_times.append(wall_ms) + tokens_per_text = [int(row["tokens"]) for row in rows] + dim = int(rows[0]["dim"]) + row_ws = [int(row.get("workspace_bytes", 0)) for row in rows] + if row_ws: + workspace_bytes = max(workspace_bytes or 0, max(row_ws)) + + tokens_per_batch = sum(tokens_per_text) + stats = summarize(engine_times) + p50 = stats["p50_ms"] + return { + "case": case_name, + "batch_size": stdin_runner.batch_size, + "tokens_per_text": tokens_per_text, + "tokens_per_batch": tokens_per_batch, + "dim": dim, + "workspace_bytes": workspace_bytes, + "engine_ms": engine_times, + "wall_ms": wall_times, + **stats, + "wall_p50_ms": percentile(wall_times, 50), + "texts_per_sec_p50": (1000.0 * stdin_runner.batch_size / p50) if p50 else None, + "tokens_per_sec_p50": (1000.0 * tokens_per_batch / p50) if p50 else None, + } + + +def format_num(value, digits=1): + if value is None: + return "-" + if isinstance(value, int): + return str(value) + return f"{value:.{digits}f}" + + +def print_table(results): + headers = [ + "backend", "B", "case", "tokens", "p50", "p90", "p95", + "txt/s", "tok/s", "RSS MB", "WS MB", + ] + rows = [] + for r in results: + token_desc = ( + str(r["tokens_per_text"][0]) + if len(set(r["tokens_per_text"])) == 1 + else f"{min(r['tokens_per_text'])}-{max(r['tokens_per_text'])}" + ) + rows.append([ + r["backend"], + str(r["batch_size"]), + str(r["case"]), + token_desc, + format_num(r["p50_ms"]), + format_num(r["p90_ms"]), + format_num(r["p95_ms"]), + format_num(r["texts_per_sec_p50"]), + format_num(r["tokens_per_sec_p50"], 0), + format_num(r.get("rss_mb"), 0), + format_num((r["workspace_bytes"] / (1024 * 1024)) + if r.get("workspace_bytes") else None, 1), + ]) + + widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(cell)) + fmt = " ".join(f"{{:<{w}}}" for w in widths) + print(fmt.format(*headers)) + print(fmt.format(*["-" * w for w in widths])) + for row in rows: + print(fmt.format(*row)) + + +def markdown_table(results): + headers = [ + "Backend", "Batch", "Case", "Tokens", "P50 ms", "P90 ms", + "P95 ms", "Texts/s", "Tokens/s", "RSS MB", "Workspace MB", + ] + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join(["---"] * len(headers)) + " |", + ] + for r in results: + token_desc = ( + str(r["tokens_per_text"][0]) + if len(set(r["tokens_per_text"])) == 1 + else f"{min(r['tokens_per_text'])}-{max(r['tokens_per_text'])}" + ) + cells = [ + r["backend"], + str(r["batch_size"]), + str(r["case"]), + token_desc, + format_num(r["p50_ms"]), + format_num(r["p90_ms"]), + format_num(r["p95_ms"]), + format_num(r["texts_per_sec_p50"]), + format_num(r["tokens_per_sec_p50"], 0), + format_num(r.get("rss_mb"), 0), + format_num((r["workspace_bytes"] / (1024 * 1024)) + if r.get("workspace_bytes") else None, 1), + ] + lines.append("| " + " | ".join(cells) + " |") + return "\n".join(lines) + "\n" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--binary", default=str(ROOT / "tools" / "cli" / "ffwd-cli")) + ap.add_argument("--backends", default="cpu", + help="Comma-separated list: cpu,mlx,cuda") + ap.add_argument("--batch-sizes", default="1,2,4,8") + ap.add_argument("--cases", default="48,80,140,256,384,512", + help="target token counts, e.g. 48,256,512") + ap.add_argument("--runs", type=int, default=7) + ap.add_argument("--warmups", type=int, default=2) + ap.add_argument("--threads", type=int, default=0, + help="CPU thread count passed to -t; 0 uses binary default") + ap.add_argument("--binary-args", default="", + help="extra arguments passed to every backend binary") + ap.add_argument("--cpu-binary-args", default="", + help="extra arguments passed only to CPU backend runs") + ap.add_argument("--mlx-binary-args", default="", + help="extra arguments passed only to MLX backend runs") + ap.add_argument("--cuda-binary-args", default="", + help="extra arguments passed only to CUDA backend runs") + ap.add_argument("--json-out") + ap.add_argument("--markdown-out") + ap.add_argument("--continue-on-error", action="store_true") + args = ap.parse_args() + + backends = parse_csv(args.backends) + batch_sizes = parse_csv(args.batch_sizes, int) + cases = parse_csv(args.cases, int) + common_binary_args = shlex.split(args.binary_args) + backend_binary_args = { + "cpu": shlex.split(args.cpu_binary_args), + "mlx": shlex.split(args.mlx_binary_args), + "cuda": shlex.split(args.cuda_binary_args), + } + + if args.runs <= 0 or args.warmups < 0: + raise SystemExit("--runs must be > 0 and --warmups must be >= 0") + for case in cases: + if case <= 0: + raise SystemExit(f"case token count must be a positive integer, got: {case}") + + report = { + "created_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "binary": os.path.abspath(args.binary), + "model_dir": os.path.abspath(args.model_dir), + "model": read_model_config(args.model_dir), + "host": { + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + "python": platform.python_version(), + }, + "config": { + "backends": backends, + "batch_sizes": batch_sizes, + "cases": cases, + "runs": args.runs, + "warmups": args.warmups, + "threads": args.threads, + }, + "results": [], + "errors": [], + } + + for backend in backends: + for batch_size in batch_sizes: + print(f"\n== backend={backend} batch={batch_size} ==") + stdin_runner = StdinRunner( + args.binary, args.model_dir, backend, batch_size, args.threads, + common_binary_args, backend_binary_args.get(backend, [])) + try: + stdin_runner.start() + rss_after_warmup = stdin_runner.rss_mb() + print( + f"load+first_batch={stdin_runner.load_ms:.1f} ms" + + (f", rss={rss_after_warmup:.0f} MB" if rss_after_warmup else "") + ) + for case in cases: + r = run_case(stdin_runner, case, args.runs, args.warmups) + r.update({ + "backend": backend, + "load_ms": stdin_runner.load_ms, + "rss_mb": stdin_runner.rss_mb() or rss_after_warmup, + }) + report["results"].append(r) + print( + f" {case:<7} tokens={r['tokens_per_batch']:>4} " + f"p50={r['p50_ms']:.1f} ms " + f"p95={r['p95_ms']:.1f} ms " + f"txt/s={r['texts_per_sec_p50']:.1f}" + + (f" ws={r['workspace_bytes'] / (1024 * 1024):.1f} MB" + if r.get("workspace_bytes") else "") + ) + except Exception as exc: + err = {"backend": backend, "batch_size": batch_size, "error": str(exc)} + report["errors"].append(err) + print(f"error: {backend} batch={batch_size}: {exc}", file=sys.stderr) + if not args.continue_on_error: + stdin_runner.stop() + raise + finally: + stdin_runner.stop() + + print("\nSummary") + print_table(report["results"]) + + if args.json_out: + with open(args.json_out, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + f.write("\n") + print(f"\nWrote {args.json_out}") + + if args.markdown_out: + with open(args.markdown_out, "w", encoding="utf-8") as f: + f.write("# ffwd benchmark matrix\n\n") + f.write(f"- Created: `{report['created_at']}`\n") + f.write(f"- Binary: `{report['binary']}`\n") + f.write(f"- Model: `{report['model_dir']}`\n") + f.write(f"- Host: `{report['host']['platform']}`\n\n") + f.write(markdown_table(report["results"])) + if report["errors"]: + f.write("\n## Errors\n\n") + for err in report["errors"]: + f.write(f"- `{err['backend']}` batch `{err['batch_size']}`: {err['error']}\n") + print(f"Wrote {args.markdown_out}") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + raise + except Exception as exc: + print(f"fatal: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/bench/bench_model.c b/bench/bench_model.c index b2d4769..7cb6458 100644 --- a/bench/bench_model.c +++ b/bench/bench_model.c @@ -10,16 +10,16 @@ */ #include "bench.h" -#include "embed.h" -#include "kernels.h" +#include "internal.h" +#include "threadpool.h" -static embed_model_t *g_model; -static embed_workspace_t *g_ws; +static ffwd_model_t *g_model; +static ffwd_workspace_t *g_ws; static int g_hidden; enum { MAX_BATCH = 8 }; -static void bm_embed(bench_state_t *b, int batch, int len) { +static void bm_ffwd(bench_state_t *b, int batch, int len) { int *ids = (int *)malloc((size_t)batch * len * sizeof(int)); float *out = (float *)malloc((size_t)batch * g_hidden * sizeof(float)); if (!ids || !out) @@ -29,14 +29,14 @@ static void bm_embed(bench_state_t *b, int batch, int len) { s = s * 1664525u + 1013904223u; ids[i] = (int)(s % 1000u); /* ordinary vocab rows */ } - embed_input_t inputs[MAX_BATCH]; + ffwd_input_t inputs[MAX_BATCH]; for (int i = 0; i < batch; i++) { inputs[i].ids = ids + (size_t)i * len; inputs[i].n_tokens = len; } bench_begin(b); for (long i = 0; i < b->n; i++) { - if (embed_model_encode_batch(g_model, g_ws, inputs, batch, out) != 0) + if (ffwd_model_encode_batch(g_model, g_ws, inputs, batch, out) != 0) exit(2); bench_sink += out[0]; } @@ -44,14 +44,14 @@ static void bm_embed(bench_state_t *b, int batch, int len) { free(ids); } -static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); } -static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); } -static void bm_embed_b1_len256(bench_state_t *b) { bm_embed(b, 1, 256); } +static void bm_ffwd_b1_len32(bench_state_t *b) { bm_ffwd(b, 1, 32); } +static void bm_ffwd_b8_len32(bench_state_t *b) { bm_ffwd(b, 8, 32); } +static void bm_ffwd_b1_len256(bench_state_t *b) { bm_ffwd(b, 1, 256); } static const bench_case_t CASES[] = { - {"embed/b1_len32", bm_embed_b1_len32}, - {"embed/b8_len32", bm_embed_b8_len32}, - {"embed/b1_len256", bm_embed_b1_len256}, + {"ffwd/b1_len32", bm_ffwd_b1_len32}, + {"ffwd/b8_len32", bm_ffwd_b8_len32}, + {"ffwd/b1_len256", bm_ffwd_b1_len256}, }; int main(int argc, char **argv) { @@ -67,19 +67,19 @@ int main(int argc, char **argv) { } if (threads < 1) threads = 1; - embed_set_threads(threads); + tp_set_threads(threads); - g_model = embed_model_load(argv[1]); + g_model = ffwd_model_load(argv[1]); if (!g_model) { fprintf(stderr, "failed to load model from %s\n", argv[1]); return 1; } - g_ws = embed_workspace_new(g_model); + g_ws = ffwd_workspace_new(g_model); if (!g_ws) { - embed_model_free(g_model); + ffwd_model_free(g_model); return 1; } - g_hidden = embed_model_config(g_model)->hidden_size; + g_hidden = ffwd_model_config(g_model)->hidden_size; char meta[160]; snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d", threads, g_hidden); @@ -89,7 +89,7 @@ int main(int argc, char **argv) { bench_parse_args(&opts, argc - 1, argv + 1); int rc = bench_main(&opts, CASES, (int)(sizeof(CASES) / sizeof(CASES[0]))); - embed_workspace_free(g_ws); - embed_model_free(g_model); + ffwd_workspace_free(g_ws); + ffwd_model_free(g_model); return rc; } diff --git a/bench/bench_qwen_tokenizer.py b/bench/bench_qwen_tokenizer.py new file mode 100755 index 0000000..a69921d --- /dev/null +++ b/bench/bench_qwen_tokenizer.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Benchmark Qwen tokenizer encode paths. + +Builds the C tokenizer bench via the Makefile (`make -C bench bench_tokenizer`) +and runs it against a Qwen model's vocab.json with a curated text set. + + bench/bench_qwen_tokenizer.py --model-dir DIR +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +DEFAULT_TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: dense retrieval systems need stable tokenization.", + "unicode: cafe, café, Istanbul, İstanbul, emoji \U0001f50d", + "mixed whitespace:\n one\t two three\n\nfour", + ("long document: " + "embedding inference should batch tokenization and " + "model execution carefully. ") * 32, +] + + +def build(cc: str) -> Path: + """Build bench/bench_tokenizer via the Makefile; return its path.""" + proc = subprocess.run(["make", "-C", "bench", f"CC={cc}", "bench_tokenizer"], + cwd=ROOT, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + raise SystemExit(proc.returncode) + return ROOT / "bench" / "bench_tokenizer" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--runs", type=int, default=1000) + ap.add_argument("--cc", default=os.environ.get("CC", "cc")) + ap.add_argument("--text", action="append", + help="text to benchmark; may be repeated") + args = ap.parse_args() + + binary = build(args.cc) + vocab = Path(args.model_dir) / "vocab.json" + texts = args.text if args.text else DEFAULT_TEXTS + return subprocess.run([str(binary), str(vocab), str(args.runs), *texts], + cwd=ROOT).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/bench_server_api.py b/bench/bench_server_api.py new file mode 100755 index 0000000..76fbba1 --- /dev/null +++ b/bench/bench_server_api.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Concurrent benchmark for the local ffwd HTTP API.""" + +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures as cf +import http.client +import json +import math +import os +import statistics +import subprocess +import threading +import time +import urllib.error +import urllib.request +import urllib.parse +from typing import Any + + +TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: Istanbul is a major city in Turkey.", + "A small embedding request for throughput testing.", + "Redis is an in-memory data structure server used for caching.", + "SQLite stores relational data in a local file.", + "Machine learning systems often use embeddings for semantic retrieval.", +] + +_thread_local = threading.local() + + +def percentile(values: list[float], p: float) -> float: + xs = sorted(values) + if len(xs) == 1: + return xs[0] + rank = (len(xs) - 1) * (p / 100.0) + lo = int(rank) + hi = min(lo + 1, len(xs) - 1) + frac = rank - lo + return xs[lo] * (1.0 - frac) + xs[hi] * frac + + +def decode_int8(s: str) -> list[float]: + return [float(b - 256 if b > 127 else b) for b in base64.b64decode(s)] + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def make_input(request_id: int, texts_per_request: int) -> list[str]: + out = [] + for i in range(texts_per_request): + out.append(TEXTS[(request_id + i) % len(TEXTS)]) + return out + + +def keepalive_connection(args: argparse.Namespace) -> tuple[http.client.HTTPConnection, str]: + url = urllib.parse.urlparse(args.base_url) + if url.scheme and url.scheme != "http": + raise RuntimeError("--keepalive supports http URLs only") + host = url.hostname or "127.0.0.1" + port = url.port or 80 + prefix = url.path.rstrip("/") + key = (host, port, prefix) + cached = getattr(_thread_local, "conn", None) + cached_key = getattr(_thread_local, "key", None) + if cached is None or cached_key != key: + if cached is not None: + cached.close() + cached = http.client.HTTPConnection(host, port, timeout=args.timeout) + _thread_local.conn = cached + _thread_local.key = key + return cached, prefix + + +def parse_server_timing(value: str | None) -> dict[str, float]: + out: dict[str, float] = {} + if not value: + return out + for part in value.split(","): + fields = [x.strip() for x in part.strip().split(";")] + if not fields or not fields[0]: + continue + name = fields[0] + for field in fields[1:]: + if field.startswith("dur="): + try: + out[name] = float(field[4:]) + except ValueError: + pass + return out + + +def post_embeddings(args: argparse.Namespace, request_id: int, + texts: list[str] | None = None) -> tuple[float, int, int, dict[str, float]]: + if texts is None: + texts = make_input(request_id, args.texts_per_request) + payload = { + "model": args.model, + "dimensions": args.dimensions, + "input": texts, + } + body = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + if args.api_key: + headers["Authorization"] = f"Bearer {args.api_key}" + t0 = time.perf_counter() + if args.keepalive: + conn, prefix = keepalive_connection(args) + try: + conn.request("POST", prefix + "/v1/embeddings", body, headers) + resp = conn.getresponse() + raw = resp.read() + if resp.status != 200: + raise RuntimeError( + f"HTTP {resp.status}: {raw.decode('utf-8', errors='replace')}" + ) + if (resp.getheader("Connection") or "").lower() == "close": + raise RuntimeError("server closed keep-alive benchmark connection") + timing = parse_server_timing(resp.getheader("Server-Timing")) + data = json.loads(raw.decode("utf-8")) + except Exception: + conn.close() + _thread_local.conn = None + raise + else: + req = urllib.request.Request( + args.base_url.rstrip("/") + "/v1/embeddings", + data=body, headers=headers) + try: + with urllib.request.urlopen(req, timeout=args.timeout) as resp: + timing = parse_server_timing(resp.headers.get("Server-Timing")) + data = json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + text = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {e.code}: {text}") from e + elapsed_ms = (time.perf_counter() - t0) * 1000.0 + + rows = data["data"] + if len(rows) != len(texts): + raise RuntimeError(f"expected {len(texts)} embeddings, got {len(rows)}") + for i, row in enumerate(rows): + if row["index"] != i: + raise RuntimeError(f"bad index at request {request_id}: {row['index']} != {i}") + raw = base64.b64decode(row["embedding"]) + if len(raw) != args.dimensions: + raise RuntimeError(f"bad decoded length: {len(raw)} != {args.dimensions}") + + return elapsed_ms, len(texts), int(data["usage"]["prompt_tokens"]), timing + + +def verify_paris_ranking(args: argparse.Namespace) -> bool: + texts = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + ] + body = json.dumps({ + "model": args.model, + "dimensions": args.dimensions, + "input": texts, + }).encode("utf-8") + headers = {"Content-Type": "application/json"} + if args.api_key: + headers["Authorization"] = f"Bearer {args.api_key}" + req = urllib.request.Request(args.base_url.rstrip("/") + "/v1/embeddings", + data=body, headers=headers) + with urllib.request.urlopen(req, timeout=args.timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + rows = data["data"] + if len(rows) != len(texts): + raise RuntimeError(f"ranking probe expected {len(texts)} embeddings, got {len(rows)}") + vecs = [decode_int8(row["embedding"]) for row in data["data"]] + return cosine(vecs[0], vecs[1]) > cosine(vecs[0], vecs[2]) + + +def rss_mb(pid: int | None) -> float | None: + if not pid: + return None + try: + out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)], text=True) + kb = int(out.strip()) + return kb / 1024.0 + except Exception: + return None + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--base-url", default="http://127.0.0.1:8000") + ap.add_argument("--api-key") + ap.add_argument("--model", default="pplx-embed-v1-0.6b") + ap.add_argument("--dimensions", type=int, default=128) + ap.add_argument("--requests", type=int, default=100) + ap.add_argument("--concurrency", type=int, default=16) + ap.add_argument("--texts-per-request", type=int, default=1) + ap.add_argument("--timeout", type=float, default=120.0) + ap.add_argument("--server-pid", type=int) + ap.add_argument("--verify-ranking", action="store_true") + ap.add_argument("--keepalive", action="store_true", + help="reuse one HTTP/1.1 connection per benchmark worker thread") + args = ap.parse_args() + + if args.requests <= 0 or args.concurrency <= 0 or args.texts_per_request <= 0: + raise SystemExit("requests, concurrency, and texts-per-request must be positive") + + t0 = time.perf_counter() + latencies: list[float] = [] + timing_rows: list[dict[str, float]] = [] + wall_minus_worker: list[float] = [] + total_texts = 0 + total_tokens = 0 + with cf.ThreadPoolExecutor(max_workers=args.concurrency) as ex: + futs = [ex.submit(post_embeddings, args, i) for i in range(args.requests)] + for fut in cf.as_completed(futs): + ms, texts, tokens, timing = fut.result() + latencies.append(ms) + if timing: + timing_rows.append(timing) + if "worker" in timing: + wall_minus_worker.append(ms - timing["worker"]) + total_texts += texts + total_tokens += tokens + wall = time.perf_counter() - t0 + ranking_ok = verify_paris_ranking(args) if args.verify_ranking else True + + print(f"server: {args.base_url} model={args.model} dim={args.dimensions}") + mode = "keepalive" if args.keepalive else "native-http" + print(f"requests={args.requests} concurrency={args.concurrency} texts/request={args.texts_per_request} mode={mode}") + print(f"wall={wall:.3f}s req/s={args.requests / wall:.2f} texts/s={total_texts / wall:.2f} tokens/s={total_tokens / wall:.2f}") + print( + "latency_ms " + f"mean={statistics.fmean(latencies):.2f} " + f"p50={percentile(latencies, 50):.2f} " + f"p95={percentile(latencies, 95):.2f} " + f"p99={percentile(latencies, 99):.2f} " + f"max={max(latencies):.2f}" + ) + mem = rss_mb(args.server_pid) + if mem is not None: + print(f"rss_mb={mem:.1f}") + if timing_rows: + for name in ("queue", "parse", "tokenize", "infer", "encode", "worker"): + values = [row[name] for row in timing_rows if name in row] + if values: + print( + f"server_timing_{name}_ms " + f"mean={statistics.fmean(values):.3f} " + f"p50={percentile(values, 50):.3f} " + f"p95={percentile(values, 95):.3f}" + ) + if wall_minus_worker: + print( + "wall_minus_worker_ms " + f"mean={statistics.fmean(wall_minus_worker):.3f} " + f"p50={percentile(wall_minus_worker, 50):.3f} " + f"p95={percentile(wall_minus_worker, 95):.3f}" + ) + if args.verify_ranking: + print(f"ranking={'ok' if ranking_ok else 'FAIL'}") + return 0 if ranking_ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index 8175ffe..9bb8822 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -1,9 +1,9 @@ /* bench/bench_tokenizer.c - tokenizer encode-path benchmark. * Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */ -#include "tokenizer_bpe.h" -#include "tokenizer_sentencepiece.h" -#include "tokenizer_wordpiece.h" +#include "bpe.h" +#include "sentencepiece.h" +#include "wordpiece.h" #include <stdio.h> #include <stdlib.h> @@ -11,8 +11,6 @@ #include <unistd.h> #include <time.h> -int embed_verbose = 0; - static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); @@ -50,8 +48,8 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) { snprintf(p, sizeof(p), "%s/vocab.txt", path); if (path_exists(p)) { bt->kind = TOK_WORDPIECE; - bt->tok = wordpiece_tokenizer_load(path); - bt->ws = wordpiece_workspace_new(); + bt->tok = tok_wp_load(path); + bt->ws = tok_wp_workspace_new(); return bt->tok && bt->ws ? 0 : -1; } snprintf(p, sizeof(p), "%s/sentencepiece.bpe.model", path); @@ -62,8 +60,8 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) { } if (path_exists(p)) { bt->kind = TOK_SENTENCEPIECE; - bt->tok = sentencepiece_tokenizer_load(path); - bt->ws = sentencepiece_workspace_new(); + bt->tok = tok_spm_load(path); + bt->ws = tok_spm_workspace_new(); return bt->tok && bt->ws ? 0 : -1; } bt->kind = TOK_BPE; @@ -71,64 +69,63 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) { snprintf(p, sizeof(p), "%s/vocab.json", path); if (path_exists(p)) vocab = p; - bt->tok = embed_tokenizer_load(vocab); - bt->ws = embed_tokenizer_workspace_new(); + bt->tok = tok_bpe_load(vocab); + bt->ws = tok_bpe_workspace_new(); return bt->tok && bt->ws ? 0 : -1; } static int *tok_encode(bench_tok_t *bt, const char *text, int *n) { switch (bt->kind) { case TOK_WORDPIECE: - return wordpiece_tokenizer_encode((wordpiece_tokenizer_t *)bt->tok, text, n); + return tok_wp_encode((tok_wp_t *)bt->tok, text, n); case TOK_SENTENCEPIECE: - return sentencepiece_tokenizer_encode((sentencepiece_tokenizer_t *)bt->tok, text, n); + return tok_spm_encode((tok_spm_t *)bt->tok, text, n); default: - return embed_tokenizer_encode((embed_tokenizer_t *)bt->tok, text, n); + return tok_bpe_encode((tok_bpe_t *)bt->tok, text, n); } } static int *tok_encode_ws(bench_tok_t *bt, const char *text, int *n) { switch (bt->kind) { case TOK_WORDPIECE: - return wordpiece_tokenizer_encode_with_workspace((wordpiece_tokenizer_t *)bt->tok, - (wordpiece_workspace_t *)bt->ws, text, n); + return tok_wp_encode_with_workspace((tok_wp_t *)bt->tok, + (tok_wp_workspace_t *)bt->ws, text, n); case TOK_SENTENCEPIECE: - return sentencepiece_tokenizer_encode_with_workspace( - (sentencepiece_tokenizer_t *)bt->tok, (sentencepiece_workspace_t *)bt->ws, text, n); + return tok_spm_encode_with_workspace( + (tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws, text, n); default: - return embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)bt->tok, - (embed_tokenizer_workspace_t *)bt->ws, text, n); + return tok_bpe_encode_with_workspace((tok_bpe_t *)bt->tok, + (tok_bpe_workspace_t *)bt->ws, text, n); } } static int tok_encode_into(bench_tok_t *bt, const char *text, int *out, int cap, int *n) { switch (bt->kind) { case TOK_WORDPIECE: - return wordpiece_tokenizer_encode_into((wordpiece_tokenizer_t *)bt->tok, - (wordpiece_workspace_t *)bt->ws, text, out, cap, n); + return tok_wp_encode_into((tok_wp_t *)bt->tok, + (tok_wp_workspace_t *)bt->ws, text, out, cap, n); case TOK_SENTENCEPIECE: - return sentencepiece_tokenizer_encode_into((sentencepiece_tokenizer_t *)bt->tok, - (sentencepiece_workspace_t *)bt->ws, text, out, - cap, n); + return tok_spm_encode_into((tok_spm_t *)bt->tok, + (tok_spm_workspace_t *)bt->ws, text, out, cap, n); default: - return embed_tokenizer_encode_into((embed_tokenizer_t *)bt->tok, - (embed_tokenizer_workspace_t *)bt->ws, text, out, cap, n); + return tok_bpe_encode_into((tok_bpe_t *)bt->tok, + (tok_bpe_workspace_t *)bt->ws, text, out, cap, n); } } static void free_tokenizer(bench_tok_t *bt) { switch (bt->kind) { case TOK_WORDPIECE: - wordpiece_workspace_free((wordpiece_workspace_t *)bt->ws); - wordpiece_tokenizer_free((wordpiece_tokenizer_t *)bt->tok); + tok_wp_workspace_free((tok_wp_workspace_t *)bt->ws); + tok_wp_free((tok_wp_t *)bt->tok); break; case TOK_SENTENCEPIECE: - sentencepiece_workspace_free((sentencepiece_workspace_t *)bt->ws); - sentencepiece_tokenizer_free((sentencepiece_tokenizer_t *)bt->tok); + tok_spm_workspace_free((tok_spm_workspace_t *)bt->ws); + tok_spm_free((tok_spm_t *)bt->tok); break; default: - embed_tokenizer_workspace_free((embed_tokenizer_workspace_t *)bt->ws); - embed_tokenizer_free((embed_tokenizer_t *)bt->tok); + tok_bpe_workspace_free((tok_bpe_workspace_t *)bt->ws); + tok_bpe_free((tok_bpe_t *)bt->tok); break; } } diff --git a/bench/benchstat.py b/bench/benchstat.py new file mode 100755 index 0000000..7f8b695 --- /dev/null +++ b/bench/benchstat.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Compare two benchmark records produced by make bench or make bench-model +(JSON written by bench/bench.h --json). + +usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0] + +Per benchmark: median ns/op of both sample sets, the delta, and a +Mann-Whitney U significance test. A delta only counts when it is BOTH +significant at --alpha AND at least --threshold percent: back-to-back +runs of an unchanged binary drift a couple of percent with machine state +(thermals, core scheduling), so smaller shifts are below the +reproducibility floor even when statistically clean. Everything else +prints as "~" - treat it as noise, not improvement or regression. +Exits 1 if any benchmark regresses past both bars. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path + + +def load(path: str) -> dict: + with open(path) as f: + data = json.load(f) + return {b["name"]: b["ns_per_op"] for b in data["benchmarks"]} + + +def median(v: list[float]) -> float: + s = sorted(v) + n = len(s) + return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2]) + + +def mann_whitney_p(a: list[float], b: list[float]) -> float: + """Two-sided p-value via the normal approximation with tie correction. + Fine for the sample counts bench.h produces (default 10 per side).""" + n1, n2 = len(a), len(b) + pooled = sorted((x, 0) for x in a) + sorted((x, 1) for x in b) + pooled.sort(key=lambda t: t[0]) + + ranks: list[float] = [0.0] * len(pooled) + tie_term = 0.0 + i = 0 + while i < len(pooled): + j = i + while j < len(pooled) and pooled[j][0] == pooled[i][0]: + j += 1 + rank = (i + j + 1) / 2.0 # average rank for the tie group (1-based) + for k in range(i, j): + ranks[k] = rank + t = j - i + tie_term += t * t * t - t + i = j + + r1 = sum(r for r, (_, grp) in zip(ranks, pooled) if grp == 0) + u1 = r1 - n1 * (n1 + 1) / 2.0 + mu = n1 * n2 / 2.0 + n = n1 + n2 + sigma_sq = n1 * n2 / 12.0 * ((n + 1) - tie_term / (n * (n - 1))) + if sigma_sq <= 0: + return 1.0 # all values identical + z = (u1 - mu - (0.5 if u1 > mu else -0.5 if u1 < mu else 0.0)) \ + / math.sqrt(sigma_sq) + return math.erfc(abs(z) / math.sqrt(2.0)) + + +def fmt_ns(ns: float) -> str: + if ns < 1e3: + return f"{ns:.1f}ns" + if ns < 1e6: + return f"{ns / 1e3:.2f}µs" + if ns < 1e9: + return f"{ns / 1e6:.2f}ms" + return f"{ns / 1e9:.3f}s" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("old") + ap.add_argument("new") + ap.add_argument("--alpha", type=float, default=0.05) + ap.add_argument("--threshold", type=float, default=3.0, + help="minimum |delta| percent to report (default 3)") + args = ap.parse_args() + + old, new = load(args.old), load(args.new) + common = [k for k in old if k in new] + if not common: + print("no common benchmarks", file=sys.stderr) + return 2 + + width = max(len(k) for k in common) + print(f"{'benchmark':<{width}} {'old':>10} {'new':>10} delta") + regressions = [] + for name in common: + mo, mn = median(old[name]), median(new[name]) + p = mann_whitney_p(old[name], new[name]) + delta = (mn - mo) / mo * 100.0 + if p < args.alpha and abs(delta) >= args.threshold: + mark = f"{delta:+.1f}% (p={p:.3f})" + if delta > 0: + regressions.append((name, delta)) + else: + mark = "~" + print(f"{name:<{width}} {fmt_ns(mo):>10} {fmt_ns(mn):>10} {mark}") + + for k in old: + if k not in new: + print(f"only in {Path(args.old).name}: {k}") + for k in new: + if k not in old: + print(f"only in {Path(args.new).name}: {k}") + + if regressions: + worst = max(regressions, key=lambda r: r[1]) + print(f"\n{len(regressions)} significant regression(s); " + f"worst: {worst[0]} {worst[1]:+.1f}%", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
