aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bench/Makefile73
-rw-r--r--bench/bench.h2
-rwxr-xr-xbench/bench_contextual_api.py209
-rw-r--r--bench/bench_kernels.c27
-rwxr-xr-xbench/bench_late_api.py202
-rw-r--r--bench/bench_late_rerank.c293
-rwxr-xr-xbench/bench_late_rerank.py52
-rwxr-xr-xbench/bench_matrix.py439
-rw-r--r--bench/bench_model.c40
-rwxr-xr-xbench/bench_qwen_tokenizer.py59
-rwxr-xr-xbench/bench_server_api.py274
-rw-r--r--bench/bench_tokenizer.c63
-rwxr-xr-xbench/benchstat.py (renamed from tools/benchstat.py)0
-rw-r--r--requirements-reference.txt5
14 files changed, 1670 insertions, 68 deletions
diff --git a/bench/Makefile b/bench/Makefile
new file mode 100644
index 0000000..9eeaa95
--- /dev/null
+++ b/bench/Makefile
@@ -0,0 +1,73 @@
+ROOT := ..
+include ../build.mk
+
+CFLAGS += -std=c11
+
+BLAS_LIBS := $(HOST_BLAS_LDFLAGS)
+
+# Benchmarks link the blas libffwd archive from the per-config build tree and
+# include its headers.
+LIBFFWD_DIR := ../libffwd
+LIBFFWD_A := $(ROOT)/build/$(MODE)/blas/libffwd.a
+SERVER_DIR := ../tools/server
+
+RESULTS := results
+# Commit, a -dirty marker when the tree has uncommitted changes, and a timestamp,
+# so repeated runs at the same commit do not overwrite each other.
+STAMP ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S)
+
+BENCHES := bench_kernels bench_tokenizer bench_server_utils bench_model \
+ bench_late_rerank
+
+.PHONY: all bench bench-model bench-tokenizer bench-server-utils clean \
+ libffwd-blas
+
+all: $(BENCHES)
+
+# The archive is built with a backend-specific flag, so a phony target (not a
+# file rule) drives the sub-make; the benchmarks list it as an order prerequisite
+# and link the resulting $(LIBFFWD_A).
+libffwd-blas:
+ $(MAKE) -C $(LIBFFWD_DIR) BACKEND=blas
+
+$(eval $(call c_program_deps,bench_kernels,bench_kernels.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm))
+
+$(eval $(call c_program_deps,bench_tokenizer,bench_tokenizer.c,libffwd-blas,$(LIBFFWD_A),-I$(LIBFFWD_DIR),-lm))
+
+# The server utilities (base64, sbuf, util) depend on no library, so the bench
+# compiles those sources directly here and keeps the build self-contained.
+SERVER_UTIL_SRCS := $(SERVER_DIR)/base64.c $(SERVER_DIR)/sbuf.c $(SERVER_DIR)/util.c
+$(eval $(call c_program_deps,bench_server_utils,bench_server_utils.c $(SERVER_UTIL_SRCS),bench.h,,-I$(SERVER_DIR),-lm))
+
+$(eval $(call c_program_deps,bench_model,bench_model.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm))
+
+$(eval $(call c_program_deps,bench_late_rerank,bench_late_rerank.c,libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm))
+
+# MLX rerank benchmark: the blas engine archive for the CPU reference paths plus
+# mlx.c for the device API. The archive calls cblas, so the bench links
+# Accelerate/OpenBLAS itself ($(HOST_BLAS_LDFLAGS)) on top of Metal/MLX. macOS only.
+MLX_DRIVER_CFLAGS := -pthread -DCHECK_MLX -I$(MLXC_PREFIX)/include -I$(LIBFFWD_DIR) $(HOST_BLAS_CFLAGS)
+MLX_DRIVER_LDLIBS := $(MLX_BACKEND_LDFLAGS) $(HOST_BLAS_LDFLAGS) -lm
+$(eval $(call c_program_deps,bench_late_rerank_mlx,bench_late_rerank.c $(LIBFFWD_DIR)/mlx.c,libffwd-blas,$(LIBFFWD_A),$(MLX_DRIVER_CFLAGS),$(MLX_DRIVER_LDLIBS)))
+
+bench: bench_kernels
+ @mkdir -p $(RESULTS)
+ ./bench_kernels --json $(RESULTS)/kernels-$(STAMP).json
+ @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json"
+
+bench-server-utils: bench_server_utils
+ @mkdir -p $(RESULTS)
+ ./bench_server_utils --json $(RESULTS)/server-utils-$(STAMP).json
+ @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json"
+
+bench-model: bench_model
+ @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; }
+ @mkdir -p $(RESULTS)
+ ./bench_model "$(MODEL_DIR)" --json $(RESULTS)/model-$(STAMP).json
+ @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json"
+
+bench-tokenizer: bench_tokenizer
+ @echo "run: ./bench_tokenizer <model_dir-or-vocab.json> <runs> TEXT..."
+
+clean:
+ $(RM) -r $(BENCHES) bench_late_rerank_mlx *.dSYM
diff --git a/bench/bench.h b/bench/bench.h
index 1d2b8f2..8be685c 100644
--- a/bench/bench.h
+++ b/bench/bench.h
@@ -8,7 +8,7 @@
* float *x = make_input(); // setup, not measured
* bench_begin(b);
* for (long i = 0; i < b->n; i++)
- * bench_sink += embed_dot_f32(x, x, 1024);
+ * bench_sink += ffwd_dot_f32(x, x, 1024);
* free(x);
* }
*
diff --git a/bench/bench_contextual_api.py b/bench/bench_contextual_api.py
new file mode 100755
index 0000000..2c0a558
--- /dev/null
+++ b/bench/bench_contextual_api.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Benchmark contextual span inference through the local HTTP API."""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures as cf
+import json
+import subprocess
+import statistics
+import time
+import urllib.error
+import urllib.request
+
+
+CHUNK_TEXT = (
+ "Contextual embedding inference processes complete documents before pooling "
+ "individual chunk spans. This benchmark keeps the input deterministic while "
+ "stressing long bidirectional attention and compact chunk result handling. "
+)
+
+
+def percentile(values: list[float], p: float) -> float:
+ xs = sorted(values)
+ if len(xs) == 1:
+ return xs[0]
+ rank = (len(xs) - 1) * (p / 100.0)
+ lo = int(rank)
+ hi = min(lo + 1, len(xs) - 1)
+ frac = rank - lo
+ return xs[lo] * (1.0 - frac) + xs[hi] * frac
+
+
+def parse_server_timing(value: str | None) -> dict[str, float]:
+ out: dict[str, float] = {}
+ if not value:
+ return out
+ for part in value.split(","):
+ fields = [x.strip() for x in part.strip().split(";")]
+ if not fields or not fields[0]:
+ continue
+ name = fields[0]
+ for field in fields[1:]:
+ if field.startswith("dur="):
+ try:
+ out[name] = float(field[4:])
+ except ValueError:
+ pass
+ return out
+
+
+def chunk_count(args: argparse.Namespace, doc_index: int) -> int:
+ return args.chunks * (doc_index + 1) if args.ragged else args.chunks
+
+
+def total_chunks_per_request(args: argparse.Namespace) -> int:
+ return sum(chunk_count(args, di) for di in range(args.documents))
+
+
+def make_payload(args: argparse.Namespace, request_id: int) -> dict:
+ docs = []
+ for di in range(args.documents):
+ chunks = [
+ f"request {request_id} document {di} chunk {ci}: "
+ + CHUNK_TEXT * args.repeats_per_chunk
+ for ci in range(chunk_count(args, di))
+ ]
+ docs.append(chunks)
+ return {
+ "model": args.model,
+ "dimensions": args.dimensions,
+ "input": docs,
+ }
+
+
+def post_json(args: argparse.Namespace, request_id: int) -> tuple[float, int, dict[str, float]]:
+ payload = make_payload(args, request_id)
+ body = json.dumps(payload).encode("utf-8")
+ headers = {"Content-Type": "application/json"}
+ if args.api_key:
+ headers["Authorization"] = f"Bearer {args.api_key}"
+ req = urllib.request.Request(
+ args.base_url.rstrip("/") + "/v1/contextualizedembeddings",
+ data=body,
+ headers=headers,
+ )
+ t0 = time.perf_counter()
+ try:
+ with urllib.request.urlopen(req, timeout=args.timeout) as resp:
+ timing = parse_server_timing(resp.headers.get("Server-Timing"))
+ data = json.loads(resp.read().decode("utf-8"))
+ except urllib.error.HTTPError as e:
+ text = e.read().decode("utf-8", errors="replace")
+ raise RuntimeError(f"HTTP {e.code}: {text}") from e
+ rows = data["data"]
+ if len(rows) != args.documents:
+ raise RuntimeError("unexpected contextual document count")
+ for di, row in enumerate(rows):
+ expected_chunks = chunk_count(args, di)
+ if row["index"] != di or len(row["data"]) != expected_chunks:
+ raise RuntimeError("unexpected contextual response shape")
+ for ci, chunk in enumerate(row["data"]):
+ if chunk["index"] != ci:
+ raise RuntimeError("unexpected contextual chunk index")
+ return (time.perf_counter() - t0) * 1000.0, int(data["usage"]["prompt_tokens"]), timing
+
+
+def rss_mb(pid: int | None) -> float | None:
+ if not pid:
+ return None
+ try:
+ out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)], text=True)
+ kb = int(out.strip())
+ return kb / 1024.0
+ except Exception:
+ return None
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--base-url", default="http://127.0.0.1:8000")
+ ap.add_argument("--api-key")
+ ap.add_argument("--model", default="pplx-embed-context-v1-0.6b")
+ ap.add_argument("--dimensions", type=int, default=128)
+ ap.add_argument("--documents", type=int, default=1)
+ ap.add_argument("--chunks", type=int, default=8)
+ ap.add_argument("--repeats-per-chunk", type=int, default=8)
+ ap.add_argument("--runs", type=int, default=7)
+ ap.add_argument("--warmups", type=int, default=2)
+ ap.add_argument("--concurrency", type=int, default=1)
+ ap.add_argument("--timeout", type=float, default=300.0)
+ ap.add_argument("--server-pid", type=int)
+ ap.add_argument("--ragged", action="store_true",
+ help="scale chunk count by document index to benchmark padded contextual batches")
+ args = ap.parse_args()
+ if min(args.dimensions, args.documents, args.chunks, args.repeats_per_chunk,
+ args.runs, args.concurrency) <= 0:
+ raise SystemExit(
+ "dimensions, documents, chunks, repeats-per-chunk, runs, and "
+ "concurrency must be positive"
+ )
+ if args.warmups < 0:
+ raise SystemExit("warmups must be >= 0")
+
+ for i in range(args.warmups):
+ post_json(args, -i - 1)
+
+ latencies = []
+ timing_rows: list[dict[str, float]] = []
+ wall_minus_worker: list[float] = []
+ total_tokens = 0
+ t0 = time.perf_counter()
+ with cf.ThreadPoolExecutor(max_workers=args.concurrency) as ex:
+ futures = [ex.submit(post_json, args, i) for i in range(args.runs)]
+ for future in cf.as_completed(futures):
+ elapsed, tokens, timing = future.result()
+ latencies.append(elapsed)
+ if timing:
+ timing_rows.append(timing)
+ if "worker" in timing:
+ wall_minus_worker.append(elapsed - timing["worker"])
+ total_tokens += tokens
+ wall = time.perf_counter() - t0
+ p50 = percentile(latencies, 50)
+ chunks_per_request = total_chunks_per_request(args)
+ print(
+ f"server={args.base_url} model={args.model} documents={args.documents} "
+ f"chunks/request={chunks_per_request} ragged={args.ragged} "
+ f"runs={args.runs} concurrency={args.concurrency}"
+ )
+ print(
+ "latency_ms "
+ f"mean={statistics.fmean(latencies):.2f} "
+ f"p50={p50:.2f} "
+ f"p95={percentile(latencies, 95):.2f} "
+ f"max={max(latencies):.2f}"
+ )
+ total_docs = args.runs * args.documents
+ total_chunks = args.runs * chunks_per_request
+ print(
+ f"wall={wall:.3f}s req/s={args.runs / wall:.2f} "
+ f"documents/s={total_docs / wall:.2f} chunks/s={total_chunks / wall:.2f} "
+ f"tokens/s={total_tokens / wall:.2f}"
+ )
+ mem = rss_mb(args.server_pid)
+ if mem is not None:
+ print(f"rss_mb={mem:.1f}")
+ if timing_rows:
+ for name in ("queue", "parse", "tokenize", "infer", "encode", "worker"):
+ values = [row[name] for row in timing_rows if name in row]
+ if values:
+ print(
+ f"server_timing_{name}_ms "
+ f"mean={statistics.fmean(values):.3f} "
+ f"p50={percentile(values, 50):.3f} "
+ f"p95={percentile(values, 95):.3f}"
+ )
+ if wall_minus_worker:
+ print(
+ "wall_minus_worker_ms "
+ f"mean={statistics.fmean(wall_minus_worker):.3f} "
+ f"p50={percentile(wall_minus_worker, 50):.3f} "
+ f"p95={percentile(wall_minus_worker, 95):.3f}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c
index 2b2fe47..55f859b 100644
--- a/bench/bench_kernels.c
+++ b/bench/bench_kernels.c
@@ -4,16 +4,15 @@
* 8 KV heads x head_dim 128, intermediate 3072) so a slowdown here predicts
* a slowdown in inference. Single-threaded by default for stable numbers;
* --threads N opts into the pool. Build and record via `make bench`,
- * compare records with tools/benchstat.py.
+ * compare records with bench/benchstat.py.
*/
#include "bench.h"
#include "kernels.h"
+#include "threadpool.h"
#include <stdint.h>
-int embed_verbose = 0; /* defined by embed.c in full builds */
-
enum {
HIDDEN = 1024,
Q_DIM = 2048, /* 16 heads * 128 */
@@ -71,7 +70,7 @@ static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) {
float *y = alloc_f32((size_t)seq * out, 3);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_linear_nobias(y, x, w, seq, in, out);
+ linear_nobias(y, x, w, seq, in, out);
bench_sink += y[0];
free(y);
free(w);
@@ -91,7 +90,7 @@ static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) {
float *y = alloc_f32((size_t)seq * out, 6);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_linear_nobias_bf16(y, x, w, seq, in, out);
+ linear_nobias_bf16(y, x, w, seq, in, out);
bench_sink += y[0];
free(y);
free(w);
@@ -111,7 +110,7 @@ static void bm_qkv_bf16(bench_state_t *b) {
float *v = alloc_f32((size_t)SEQ * KV_DIM, 13);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM);
+ linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM);
bench_sink += q[0] + k[0] + v[0];
free(v);
free(k);
@@ -128,7 +127,7 @@ static void bm_bf16_widen(bench_state_t *b) {
float *dst = alloc_f32(n, 15);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_bf16_to_f32_buf(dst, src, n);
+ bf16_to_f32_buf(dst, src, n);
bench_sink += dst[0];
free(dst);
free(src);
@@ -143,7 +142,7 @@ static void bm_attn_scores(bench_state_t *b) {
float *s = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 18);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ);
+ matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ);
bench_sink += s[0];
free(s);
free(km);
@@ -154,7 +153,7 @@ static void bm_softmax(bench_state_t *b) {
float *x = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 19);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_softmax(x, ATTN_SEQ, ATTN_SEQ);
+ softmax(x, ATTN_SEQ, ATTN_SEQ);
bench_sink += x[0];
free(x);
}
@@ -165,7 +164,7 @@ static void bm_rope(bench_state_t *b) {
float *sinv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 22);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM);
+ apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM);
bench_sink += x[0];
free(sinv);
free(cosv);
@@ -180,7 +179,7 @@ static void bm_rms_norm(bench_state_t *b) {
float *y = alloc_f32((size_t)SEQ * HIDDEN, 25);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- embed_rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f);
+ rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f);
bench_sink += y[0];
free(y);
free(w);
@@ -197,7 +196,7 @@ static void bm_silu_mul(bench_state_t *b) {
bench_begin(b);
for (long i = 0; i < b->n; i++) {
memcpy(gate, gate0, n * sizeof(float));
- embed_silu_mul_inplace(gate, up, (int)n);
+ silu_mul_inplace(gate, up, (int)n);
}
bench_sink += gate[0];
free(up);
@@ -210,7 +209,7 @@ static void bm_dot(bench_state_t *b) {
float *y = alloc_f32(HIDDEN, 29);
bench_begin(b);
for (long i = 0; i < b->n; i++)
- bench_sink += embed_dot_f32(x, y, HIDDEN);
+ bench_sink += dot_f32(x, y, HIDDEN);
free(y);
free(x);
}
@@ -240,7 +239,7 @@ int main(int argc, char **argv) {
}
if (threads < 1)
threads = 1;
- embed_set_threads(threads);
+ tp_set_threads(threads);
char meta[128];
#ifdef USE_BLAS
diff --git a/bench/bench_late_api.py b/bench/bench_late_api.py
new file mode 100755
index 0000000..7a6073f
--- /dev/null
+++ b/bench/bench_late_api.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Benchmark /v1/rerank and optionally compare end-to-end with PyLate."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import time
+import urllib.error
+import urllib.request
+from typing import Any
+
+
+BASE_DOCUMENTS = [
+ "Scientists explore the universe driven by curiosity.",
+ "Children learn through curious exploration.",
+ "Historical discoveries began with careful questions.",
+ "Redis stores data structures in memory for fast access.",
+ "SQLite keeps relational data in a local database file.",
+ "Embedding models map text into semantic vector spaces.",
+ "Paris is the capital of France and a major European destination.",
+ "Neural search compares query and document token representations.",
+]
+QUERY = "What motivates scientific discovery?"
+
+
+def percentile(values: list[float], p: float) -> float:
+ values = sorted(values)
+ rank = (len(values) - 1) * p / 100
+ lo = int(rank)
+ hi = min(lo + 1, len(values) - 1)
+ frac = rank - lo
+ return values[lo] * (1 - frac) + values[hi] * frac
+
+
+def make_documents(candidates: int, repeat: int) -> list[str]:
+ return [
+ " ".join(
+ BASE_DOCUMENTS[(candidate + offset) % len(BASE_DOCUMENTS)]
+ for offset in range(repeat)
+ )
+ for candidate in range(candidates)
+ ]
+
+
+def post_json(
+ base_url: str,
+ payload: dict[str, Any],
+ api_key: str | None,
+ timeout: float,
+) -> dict[str, Any]:
+ headers = {"Content-Type": "application/json"}
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+ req = urllib.request.Request(
+ base_url.rstrip("/") + "/v1/rerank",
+ data=json.dumps(payload).encode(),
+ headers=headers,
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
+ return json.loads(resp.read())
+ except urllib.error.HTTPError as exc:
+ text = exc.read().decode(errors="replace")
+ raise RuntimeError(f"HTTP {exc.code}: {text}") from exc
+
+
+def summarize(name: str, values: list[float], candidates: int) -> None:
+ mean = statistics.fmean(values)
+ print(
+ f"{name}: mean_ms={mean:.2f} p50_ms={percentile(values, 50):.2f} "
+ f"p95_ms={percentile(values, 95):.2f} "
+ f"candidates/s={candidates * 1000 / mean:.2f}"
+ )
+
+
+def synchronize(device: str) -> None:
+ if device == "mps":
+ import torch
+
+ torch.mps.synchronize()
+ elif device.startswith("cuda"):
+ import torch
+
+ torch.cuda.synchronize()
+
+
+def pylate_run(model: Any, query: str, documents: list[str], device: str) -> list[float]:
+ from pylate.scores import colbert_scores
+
+ query_vectors = model.encode(
+ [query],
+ is_query=True,
+ convert_to_tensor=True,
+ show_progress_bar=False,
+ )[0]
+ document_vectors = model.encode(
+ documents,
+ is_query=False,
+ convert_to_tensor=True,
+ show_progress_bar=False,
+ )
+ scores = [
+ float(
+ colbert_scores(
+ query_vectors.unsqueeze(0),
+ document.unsqueeze(0),
+ )[0, 0]
+ )
+ for document in document_vectors
+ ]
+ synchronize(device)
+ return scores
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--base-url", default="http://127.0.0.1:8000")
+ ap.add_argument("--api-key")
+ ap.add_argument("--model", default="pplx-embed-v1-late-0.6b")
+ ap.add_argument("--model-dir")
+ ap.add_argument("--candidates", type=int, default=32)
+ ap.add_argument("--document-repeat", type=int, default=2)
+ ap.add_argument("--runs", type=int, default=5)
+ ap.add_argument("--warmups", type=int, default=1)
+ ap.add_argument("--timeout", type=float, default=600)
+ ap.add_argument("--compare-pylate", action="store_true")
+ ap.add_argument("--pylate-device", default="mps")
+ args = ap.parse_args()
+ if min(args.candidates, args.document_repeat, args.runs) <= 0:
+ raise SystemExit("candidates, document-repeat, and runs must be positive")
+ if args.warmups < 0:
+ raise SystemExit("warmups must be non-negative")
+ if args.compare_pylate and not args.model_dir:
+ raise SystemExit("--model-dir is required with --compare-pylate")
+
+ documents = make_documents(args.candidates, args.document_repeat)
+ payload = {
+ "model": args.model,
+ "query": QUERY,
+ "documents": documents,
+ }
+ for _ in range(args.warmups):
+ post_json(args.base_url, payload, args.api_key, args.timeout)
+ server_times = []
+ server_response: dict[str, Any] = {}
+ for _ in range(args.runs):
+ start = time.perf_counter()
+ server_response = post_json(
+ args.base_url, payload, args.api_key, args.timeout
+ )
+ server_times.append((time.perf_counter() - start) * 1000)
+ summarize("pplx-embed-server", server_times, args.candidates)
+
+ if args.compare_pylate:
+ try:
+ from pylate import models
+ except ImportError as exc:
+ raise SystemExit(
+ "PyLate is missing; run with `uv run --with pylate "
+ "scripts/bench_late_api.py ...`"
+ ) from exc
+
+ model = models.ColBERT(
+ model_name_or_path=args.model_dir,
+ trust_remote_code=True,
+ device=args.pylate_device,
+ )
+ for _ in range(args.warmups):
+ pylate_run(model, QUERY, documents, args.pylate_device)
+ pylate_times = []
+ pylate_scores: list[float] = []
+ for _ in range(args.runs):
+ start = time.perf_counter()
+ pylate_scores = pylate_run(
+ model, QUERY, documents, args.pylate_device
+ )
+ pylate_times.append((time.perf_counter() - start) * 1000)
+ summarize(f"PyLate/{args.pylate_device}", pylate_times, args.candidates)
+
+ server_scores = [0.0] * args.candidates
+ for result in server_response["results"]:
+ server_scores[result["index"]] = float(result["relevance_score"])
+ worst = max(abs(a - b) for a, b in zip(server_scores, pylate_scores))
+ server_order = sorted(
+ range(args.candidates), key=server_scores.__getitem__, reverse=True
+ )
+ pylate_order = sorted(
+ range(args.candidates), key=pylate_scores.__getitem__, reverse=True
+ )
+ print(
+ f"parity: worst_score_diff={worst:.8g} "
+ f"top1_match={server_order[0] == pylate_order[0]}"
+ )
+ if worst > 1e-4 or server_order[0] != pylate_order[0]:
+ raise RuntimeError("server and PyLate score parity failed")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/bench/bench_late_rerank.c b/bench/bench_late_rerank.c
new file mode 100644
index 0000000..f6e7734
--- /dev/null
+++ b/bench/bench_late_rerank.c
@@ -0,0 +1,293 @@
+/* bench/bench_late_rerank.c - late-interaction MaxSim rerank benchmark.
+ * Build via `make bench-late-rerank`; run with:
+ * ./bench_late_rerank MODEL_DIR CANDIDATES DOC_REPEAT RUNS
+ * Encodes a query and N candidate documents to token vectors, then times the
+ * packed MaxSim rerank. check it against bench_late_rerank.py, which builds and
+ * runs this binary. */
+#include "internal.h"
+#ifdef CHECK_MLX
+# include "mlx.h"
+#endif
+#include "bpe.h"
+
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#define QUERY_LENGTH 32
+#define MASK_TOKEN_ID 151642
+#define QUERY_PREFIX_ID 151669
+#define DOC_PREFIX_ID 151670
+
+typedef struct {
+ int *ids;
+ int n_ids;
+ float *vecs;
+ int n_vecs;
+#ifdef CHECK_MLX
+ ffwd_mlx_late_vectors_t *dev;
+#endif
+} late_item_t;
+
+#ifdef CHECK_MLX
+typedef ffwd_mlx_late_ctx_t test_model_t;
+typedef void test_workspace_t;
+static test_model_t *test_model_load(const char *dir) { return ffwd_mlx_late_load(dir); }
+static void test_model_free(test_model_t *m) { ffwd_mlx_late_free(m); }
+static test_workspace_t *test_workspace_new(test_model_t *m) {
+ (void)m;
+ return (test_workspace_t *)1;
+}
+static void test_workspace_free(test_workspace_t *ws) { (void)ws; }
+static const ffwd_config_t *test_config(test_model_t *m) { return ffwd_mlx_late_config(m); }
+static int test_token_dim(test_model_t *m) { return ffwd_mlx_late_token_dim(m); }
+static int test_encode(
+ test_model_t *m, test_workspace_t *ws, const int *ids, int n_ids, int normalize, float *out) {
+ (void)ws;
+ return ffwd_mlx_late_encode_tokens(m, ids, n_ids, normalize, out);
+}
+#else
+typedef ffwd_late_model_t test_model_t;
+typedef ffwd_late_workspace_t test_workspace_t;
+static test_model_t *test_model_load(const char *dir) { return ffwd_late_model_load(dir); }
+static void test_model_free(test_model_t *m) { ffwd_late_model_free(m); }
+static test_workspace_t *test_workspace_new(test_model_t *m) { return ffwd_late_workspace_new(m); }
+static void test_workspace_free(test_workspace_t *ws) { ffwd_late_workspace_free(ws); }
+static const ffwd_config_t *test_config(test_model_t *m) { return ffwd_late_model_config(m); }
+static int test_token_dim(test_model_t *m) { return ffwd_late_model_token_dim(m); }
+static int test_encode(
+ test_model_t *m, test_workspace_t *ws, const int *ids, int n_ids, int normalize, float *out) {
+ return ffwd_late_model_encode_tokens(m, ws, ids, n_ids, normalize, out);
+}
+#endif
+
+static double now_ms(void) {
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0;
+}
+
+static int encode_with_prefix_id(
+ tok_bpe_t *tok, int prefix_id, const char *text, int expand_query, late_item_t *out) {
+ out->ids = tok_bpe_encode(tok, text, &out->n_ids);
+ if (!out->ids || out->n_ids <= 0)
+ return -1;
+
+ int target_len = out->n_ids + 1;
+ if (expand_query && target_len < QUERY_LENGTH)
+ target_len = QUERY_LENGTH;
+
+ int *tmp = (int *)realloc(out->ids, (size_t)target_len * sizeof(int));
+ if (!tmp)
+ return -1;
+ out->ids = tmp;
+
+ memmove(out->ids + 2, out->ids + 1, (size_t)(out->n_ids - 1) * sizeof(int));
+ out->ids[1] = prefix_id;
+ out->n_ids++;
+
+ if (expand_query && out->n_ids < QUERY_LENGTH) {
+ for (int i = out->n_ids; i < QUERY_LENGTH; i++)
+ out->ids[i] = MASK_TOKEN_ID;
+ out->n_ids = QUERY_LENGTH;
+ }
+ return 0;
+}
+
+static int encode_late_item(test_model_t *model, test_workspace_t *ws, late_item_t *item, int dim) {
+ item->n_vecs = item->n_ids;
+ item->vecs = (float *)malloc((size_t)item->n_vecs * dim * sizeof(float));
+ if (!item->vecs)
+ return -1;
+ if (test_encode(model, ws, item->ids, item->n_ids, 1, item->vecs) != 0)
+ return -1;
+#ifdef CHECK_MLX
+ item->dev = ffwd_mlx_late_encode_tokens_device(model, item->ids, item->n_ids, 1);
+ if (!item->dev)
+ return -1;
+#endif
+ return 0;
+}
+
+static void free_item(late_item_t *item) {
+#ifdef CHECK_MLX
+ ffwd_mlx_late_vectors_free(item->dev);
+#endif
+ free(item->ids);
+ free(item->vecs);
+}
+
+static char *make_doc_text(int index, int repeat) {
+ static const char *base[] = {"Scientists explore the universe driven by curiosity",
+ "Children learn through curious exploration",
+ "Historical discoveries began with careful questions",
+ "Redis stores data structures in memory for fast access",
+ "SQLite keeps relational data in a local database file",
+ "Embedding models map text into semantic vector spaces",
+ "Paris is a capital city and a major European destination",
+ "Neural search systems compare query and document representations"};
+ enum { NBASE = 8 };
+ size_t cap = 64;
+ for (int i = 0; i < repeat; i++)
+ cap += strlen(base[(index + i) % NBASE]) + 2;
+ char *out = (char *)malloc(cap);
+ if (!out)
+ return NULL;
+ out[0] = '\0';
+ for (int i = 0; i < repeat; i++) {
+ if (i)
+ strcat(out, ". ");
+ strcat(out, base[(index + i) % NBASE]);
+ }
+ return out;
+}
+
+int main(int argc, char **argv) {
+ if (argc != 5) {
+ fprintf(stderr, "usage: %s MODEL_DIR CANDIDATES DOC_REPEAT RUNS\n", argv[0]);
+ return 2;
+ }
+ const char *model_dir = argv[1];
+ int candidates = atoi(argv[2]);
+ int doc_repeat = atoi(argv[3]);
+ int runs = atoi(argv[4]);
+ if (candidates <= 0 || doc_repeat <= 0 || runs <= 0)
+ return 2;
+
+ char vocab_path[4096];
+ snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
+ tok_bpe_t *tok = tok_bpe_load(vocab_path);
+ test_model_t *model = test_model_load(model_dir);
+ test_workspace_t *ws = test_workspace_new(model);
+ if (!tok || !model || !ws) {
+ fprintf(stderr, "failed to initialize late benchmark\n");
+ return 1;
+ }
+
+ int dim = test_token_dim(model);
+ const ffwd_config_t *cfg = test_config(model);
+ if (dim <= 0 || !cfg) {
+ fprintf(stderr, "bad late model config\n");
+ return 1;
+ }
+
+ late_item_t q = {0};
+ late_item_t *docs = (late_item_t *)calloc((size_t)candidates, sizeof(*docs));
+ int *offsets = (int *)calloc((size_t)candidates + 1, sizeof(*offsets));
+ float *scores = (float *)malloc((size_t)candidates * sizeof(float));
+ float *device_scores = (float *)malloc((size_t)candidates * sizeof(float));
+ if (!docs || !offsets || !scores || !device_scores)
+ return 1;
+
+ if (encode_with_prefix_id(tok, QUERY_PREFIX_ID, "What motivates scientific discovery?", 1, &q) !=
+ 0 ||
+ encode_late_item(model, ws, &q, dim) != 0)
+ return 1;
+
+ int total_doc_tokens = 0;
+ offsets[0] = 0;
+ for (int i = 0; i < candidates; i++) {
+ char *text = make_doc_text(i, doc_repeat);
+ if (!text)
+ return 1;
+ int rc = encode_with_prefix_id(tok, DOC_PREFIX_ID, text, 0, &docs[i]);
+ free(text);
+ if (rc != 0 || encode_late_item(model, ws, &docs[i], dim) != 0)
+ return 1;
+ total_doc_tokens += docs[i].n_vecs;
+ offsets[i + 1] = total_doc_tokens;
+ }
+
+ float *packed = (float *)malloc((size_t)total_doc_tokens * dim * sizeof(float));
+ if (!packed)
+ return 1;
+ int pos = 0;
+ for (int i = 0; i < candidates; i++) {
+ memcpy(packed + (size_t)pos * dim, docs[i].vecs,
+ (size_t)docs[i].n_vecs * dim * sizeof(float));
+ pos += docs[i].n_vecs;
+ }
+
+ if (ffwd_late_maxsim_batch(q.vecs, q.n_vecs, packed, offsets, candidates, dim, scores) != 0)
+ return 1;
+
+#ifdef CHECK_MLX
+ const ffwd_mlx_late_vectors_t **dev_docs =
+ (const ffwd_mlx_late_vectors_t **)malloc((size_t)candidates * sizeof(*dev_docs));
+ if (!dev_docs)
+ return 1;
+ for (int i = 0; i < candidates; i++)
+ dev_docs[i] = docs[i].dev;
+ ffwd_mlx_late_vectors_t *packed_dev =
+ ffwd_mlx_late_vectors_concat(model, dev_docs, candidates);
+ free(dev_docs);
+ if (!packed_dev)
+ return 1;
+ if (ffwd_mlx_late_maxsim_batch_device(model, q.dev, packed_dev, offsets, candidates,
+ device_scores) != 0)
+ return 1;
+ float worst_diff = 0.0f;
+ for (int i = 0; i < candidates; i++) {
+ float d = fabsf(scores[i] - device_scores[i]);
+ if (d > worst_diff)
+ worst_diff = d;
+ if (d > 3e-4f) {
+ fprintf(stderr, "device score mismatch at %d: %.9g %.9g\n", i, device_scores[i],
+ scores[i]);
+ return 1;
+ }
+ }
+#endif
+
+ double start = now_ms();
+ for (int r = 0; r < runs; r++) {
+ if (ffwd_late_maxsim_batch(q.vecs, q.n_vecs, packed, offsets, candidates, dim, scores) !=
+ 0)
+ return 1;
+ }
+ double cpu_ms = now_ms() - start;
+
+ printf("late_rerank backend=%s candidates=%d query_tokens=%d "
+ "doc_tokens_total=%d doc_tokens_avg=%.1f dim=%d runs=%d\n",
+#ifdef CHECK_MLX
+ "mlx",
+#else
+ "cpu",
+#endif
+ candidates, q.n_vecs, total_doc_tokens, (double)total_doc_tokens / (double)candidates, dim,
+ runs);
+ printf("cpu_packed_maxsim_ms %.6f candidates/s %.0f token_pairs/s %.0f\n", cpu_ms / (double)runs,
+ ((double)runs * candidates) / (cpu_ms / 1000.0),
+ ((double)runs * q.n_vecs * total_doc_tokens) / (cpu_ms / 1000.0));
+
+#ifdef CHECK_MLX
+ start = now_ms();
+ for (int r = 0; r < runs; r++) {
+ if (ffwd_mlx_late_maxsim_batch_device(model, q.dev, packed_dev, offsets, candidates,
+ device_scores) != 0)
+ return 1;
+ }
+ double mlx_ms = now_ms() - start;
+ printf("mlx_device_maxsim_ms %.6f candidates/s %.0f token_pairs/s %.0f "
+ "max_diff %.8g\n",
+ mlx_ms / (double)runs, ((double)runs * candidates) / (mlx_ms / 1000.0),
+ ((double)runs * q.n_vecs * total_doc_tokens) / (mlx_ms / 1000.0), worst_diff);
+ ffwd_mlx_late_vectors_free(packed_dev);
+#endif
+
+ for (int i = 0; i < candidates; i++)
+ free_item(&docs[i]);
+ free_item(&q);
+ free(packed);
+ free(device_scores);
+ free(scores);
+ free(offsets);
+ free(docs);
+ test_workspace_free(ws);
+ test_model_free(model);
+ tok_bpe_free(tok);
+ return 0;
+}
diff --git a/bench/bench_late_rerank.py b/bench/bench_late_rerank.py
new file mode 100755
index 0000000..99bb817
--- /dev/null
+++ b/bench/bench_late_rerank.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""Benchmark late-interaction MaxSim reranking shapes.
+
+Builds the C bench (bench/bench_late_rerank.c) via the Makefile and runs it
+against a late model directory (e.g. pplx-embed-v1-late-0.6b). python3 only.
+
+ bench/bench_late_rerank.py --model-dir DIR
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def build(cc: str, backend: str) -> Path:
+ """Build the late-rerank bench via the Makefile; return its path."""
+ target = "bench_late_rerank_mlx" if backend == "mlx" else "bench_late_rerank"
+ proc = subprocess.run(["make", "-C", "bench", f"CC={cc}", target], cwd=ROOT,
+ text=True, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, check=False)
+ if proc.returncode != 0:
+ sys.stderr.write(proc.stdout)
+ raise SystemExit(proc.returncode)
+ return ROOT / "bench" / target
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--backend", choices=("cpu", "mlx"), default="cpu")
+ ap.add_argument("--candidates", type=int, default=32)
+ ap.add_argument("--doc-repeat", type=int, default=3)
+ ap.add_argument("--runs", type=int, default=200)
+ ap.add_argument("--cc", default=os.environ.get("CC", "cc"))
+ args = ap.parse_args()
+
+ binary = build(args.cc, args.backend)
+ return subprocess.run(
+ [str(binary), args.model_dir, str(args.candidates),
+ str(args.doc_repeat), str(args.runs)],
+ cwd=ROOT,
+ ).returncode
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/bench/bench_matrix.py b/bench/bench_matrix.py
new file mode 100755
index 0000000..b302100
--- /dev/null
+++ b/bench/bench_matrix.py
@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+"""Batch-aware benchmark matrix for the stdin backends."""
+
+import argparse
+import datetime as dt
+import json
+import os
+import platform
+import shlex
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+try:
+ import psutil
+except Exception: # pragma: no cover - optional dependency
+ psutil = None
+
+
+# A case is a target token count (an integer). The input is synthesized to about
+# that many common words (~1 token each); the tokenizer reports the actual count
+# per run, so the case is an explicit, approximate length target.
+_FILLER = (
+ "embedding models convert text into dense vectors that capture semantic "
+ "meaning for search clustering classification and retrieval augmented "
+ "generation across many production systems languages and domains where "
+ "latency throughput memory bandwidth and matrix multiplication all matter"
+).split()
+
+
+def make_text(n_tokens):
+ n = max(1, int(n_tokens))
+ return " ".join(_FILLER[i % len(_FILLER)] for i in range(n))
+
+
+def parse_csv(value, cast=str):
+ items = []
+ for raw in value.split(","):
+ raw = raw.strip()
+ if raw:
+ items.append(cast(raw))
+ return items
+
+
+def percentile(values, p):
+ if not values:
+ return None
+ xs = sorted(values)
+ if len(xs) == 1:
+ return xs[0]
+ rank = (len(xs) - 1) * (p / 100.0)
+ lo = int(rank)
+ hi = min(lo + 1, len(xs) - 1)
+ frac = rank - lo
+ return xs[lo] * (1.0 - frac) + xs[hi] * frac
+
+
+def summarize(values):
+ return {
+ "min_ms": min(values),
+ "mean_ms": statistics.fmean(values),
+ "p50_ms": percentile(values, 50),
+ "p90_ms": percentile(values, 90),
+ "p95_ms": percentile(values, 95),
+ "p99_ms": percentile(values, 99),
+ "max_ms": max(values),
+ }
+
+
+def make_batch_texts(n_tokens, batch_size):
+ return [make_text(n_tokens) for _ in range(batch_size)]
+
+
+class StdinRunner:
+ def __init__(self, binary, model_dir, backend, batch_size, threads,
+ binary_args=None, backend_args=None):
+ self.binary = binary
+ self.model_dir = model_dir
+ self.backend = backend
+ self.batch_size = batch_size
+ self.threads = threads
+ self.binary_args = list(binary_args or [])
+ self.backend_args = list(backend_args or [])
+ self.proc = None
+ self.ps = None
+ self.load_ms = None
+
+ def start(self):
+ cmd = [self.binary, "-d", self.model_dir,
+ *self.binary_args, *self.backend_args,
+ "--stream", "-b", str(self.batch_size)]
+ if self.backend == "cpu" and self.threads:
+ cmd.extend(["-t", str(self.threads)])
+
+ t0 = time.perf_counter()
+ self.proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ bufsize=1,
+ )
+ if psutil:
+ self.ps = psutil.Process(self.proc.pid)
+
+ warmup = make_batch_texts(16, self.batch_size)
+ self.send_batch(warmup)
+ self.load_ms = (time.perf_counter() - t0) * 1000.0
+
+ def rss_mb(self):
+ if self.ps:
+ try:
+ return self.ps.memory_info().rss / (1024 * 1024)
+ except Exception:
+ pass
+ if not self.proc:
+ return None
+ try:
+ proc = subprocess.run(
+ ["ps", "-o", "rss=", "-p", str(self.proc.pid)],
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ check=False,
+ )
+ value = proc.stdout.strip() if proc.returncode == 0 else ""
+ return (float(value) / 1024.0) if value else None
+ except Exception:
+ return None
+
+ def send_batch(self, texts):
+ if len(texts) != self.batch_size:
+ raise ValueError("send_batch requires exactly batch_size texts")
+ if not self.proc or not self.proc.stdin or not self.proc.stdout:
+ raise RuntimeError("stdin is not running")
+
+ t0 = time.perf_counter()
+ self.proc.stdin.write("\n".join(texts) + "\n")
+ self.proc.stdin.flush()
+
+ rows = []
+ for _ in texts:
+ line = self.proc.stdout.readline()
+ if line == "":
+ stderr = self.proc.stderr.read() if self.proc.stderr else ""
+ raise RuntimeError(f"stdin exited early\n{stderr}")
+ row = json.loads(line)
+ if "error" in row:
+ raise RuntimeError(f"stdin returned error: {row}")
+ rows.append(row)
+
+ wall_ms = (time.perf_counter() - t0) * 1000.0
+ engine_ms = float(rows[0]["ms"])
+ return rows, engine_ms, wall_ms
+
+ def stop(self):
+ if not self.proc:
+ return
+ try:
+ if self.proc.stdin:
+ self.proc.stdin.close()
+ self.proc.wait(timeout=5)
+ finally:
+ self.proc = None
+
+
+def read_model_config(model_dir):
+ with open(os.path.join(model_dir, "config.json"), "r", encoding="utf-8") as f:
+ cfg = json.load(f)
+ return {
+ "hidden_size": cfg.get("hidden_size"),
+ "num_hidden_layers": cfg.get("num_hidden_layers"),
+ "num_attention_heads": cfg.get("num_attention_heads"),
+ "num_key_value_heads": cfg.get("num_key_value_heads"),
+ "intermediate_size": cfg.get("intermediate_size"),
+ "vocab_size": cfg.get("vocab_size"),
+ }
+
+
+def run_case(stdin_runner, case_name, runs, warmups):
+ texts = make_batch_texts(case_name, stdin_runner.batch_size)
+
+ for _ in range(warmups):
+ stdin_runner.send_batch(texts)
+
+ engine_times = []
+ wall_times = []
+ tokens_per_text = None
+ dim = None
+ workspace_bytes = None
+ for _ in range(runs):
+ rows, engine_ms, wall_ms = stdin_runner.send_batch(texts)
+ engine_times.append(engine_ms)
+ wall_times.append(wall_ms)
+ tokens_per_text = [int(row["tokens"]) for row in rows]
+ dim = int(rows[0]["dim"])
+ row_ws = [int(row.get("workspace_bytes", 0)) for row in rows]
+ if row_ws:
+ workspace_bytes = max(workspace_bytes or 0, max(row_ws))
+
+ tokens_per_batch = sum(tokens_per_text)
+ stats = summarize(engine_times)
+ p50 = stats["p50_ms"]
+ return {
+ "case": case_name,
+ "batch_size": stdin_runner.batch_size,
+ "tokens_per_text": tokens_per_text,
+ "tokens_per_batch": tokens_per_batch,
+ "dim": dim,
+ "workspace_bytes": workspace_bytes,
+ "engine_ms": engine_times,
+ "wall_ms": wall_times,
+ **stats,
+ "wall_p50_ms": percentile(wall_times, 50),
+ "texts_per_sec_p50": (1000.0 * stdin_runner.batch_size / p50) if p50 else None,
+ "tokens_per_sec_p50": (1000.0 * tokens_per_batch / p50) if p50 else None,
+ }
+
+
+def format_num(value, digits=1):
+ if value is None:
+ return "-"
+ if isinstance(value, int):
+ return str(value)
+ return f"{value:.{digits}f}"
+
+
+def print_table(results):
+ headers = [
+ "backend", "B", "case", "tokens", "p50", "p90", "p95",
+ "txt/s", "tok/s", "RSS MB", "WS MB",
+ ]
+ rows = []
+ for r in results:
+ token_desc = (
+ str(r["tokens_per_text"][0])
+ if len(set(r["tokens_per_text"])) == 1
+ else f"{min(r['tokens_per_text'])}-{max(r['tokens_per_text'])}"
+ )
+ rows.append([
+ r["backend"],
+ str(r["batch_size"]),
+ str(r["case"]),
+ token_desc,
+ format_num(r["p50_ms"]),
+ format_num(r["p90_ms"]),
+ format_num(r["p95_ms"]),
+ format_num(r["texts_per_sec_p50"]),
+ format_num(r["tokens_per_sec_p50"], 0),
+ format_num(r.get("rss_mb"), 0),
+ format_num((r["workspace_bytes"] / (1024 * 1024))
+ if r.get("workspace_bytes") else None, 1),
+ ])
+
+ widths = [len(h) for h in headers]
+ for row in rows:
+ for i, cell in enumerate(row):
+ widths[i] = max(widths[i], len(cell))
+ fmt = " ".join(f"{{:<{w}}}" for w in widths)
+ print(fmt.format(*headers))
+ print(fmt.format(*["-" * w for w in widths]))
+ for row in rows:
+ print(fmt.format(*row))
+
+
+def markdown_table(results):
+ headers = [
+ "Backend", "Batch", "Case", "Tokens", "P50 ms", "P90 ms",
+ "P95 ms", "Texts/s", "Tokens/s", "RSS MB", "Workspace MB",
+ ]
+ lines = [
+ "| " + " | ".join(headers) + " |",
+ "| " + " | ".join(["---"] * len(headers)) + " |",
+ ]
+ for r in results:
+ token_desc = (
+ str(r["tokens_per_text"][0])
+ if len(set(r["tokens_per_text"])) == 1
+ else f"{min(r['tokens_per_text'])}-{max(r['tokens_per_text'])}"
+ )
+ cells = [
+ r["backend"],
+ str(r["batch_size"]),
+ str(r["case"]),
+ token_desc,
+ format_num(r["p50_ms"]),
+ format_num(r["p90_ms"]),
+ format_num(r["p95_ms"]),
+ format_num(r["texts_per_sec_p50"]),
+ format_num(r["tokens_per_sec_p50"], 0),
+ format_num(r.get("rss_mb"), 0),
+ format_num((r["workspace_bytes"] / (1024 * 1024))
+ if r.get("workspace_bytes") else None, 1),
+ ]
+ lines.append("| " + " | ".join(cells) + " |")
+ return "\n".join(lines) + "\n"
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--binary", default=str(ROOT / "tools" / "cli" / "ffwd-cli"))
+ ap.add_argument("--backends", default="cpu",
+ help="Comma-separated list: cpu,mlx,cuda")
+ ap.add_argument("--batch-sizes", default="1,2,4,8")
+ ap.add_argument("--cases", default="48,80,140,256,384,512",
+ help="target token counts, e.g. 48,256,512")
+ ap.add_argument("--runs", type=int, default=7)
+ ap.add_argument("--warmups", type=int, default=2)
+ ap.add_argument("--threads", type=int, default=0,
+ help="CPU thread count passed to -t; 0 uses binary default")
+ ap.add_argument("--binary-args", default="",
+ help="extra arguments passed to every backend binary")
+ ap.add_argument("--cpu-binary-args", default="",
+ help="extra arguments passed only to CPU backend runs")
+ ap.add_argument("--mlx-binary-args", default="",
+ help="extra arguments passed only to MLX backend runs")
+ ap.add_argument("--cuda-binary-args", default="",
+ help="extra arguments passed only to CUDA backend runs")
+ ap.add_argument("--json-out")
+ ap.add_argument("--markdown-out")
+ ap.add_argument("--continue-on-error", action="store_true")
+ args = ap.parse_args()
+
+ backends = parse_csv(args.backends)
+ batch_sizes = parse_csv(args.batch_sizes, int)
+ cases = parse_csv(args.cases, int)
+ common_binary_args = shlex.split(args.binary_args)
+ backend_binary_args = {
+ "cpu": shlex.split(args.cpu_binary_args),
+ "mlx": shlex.split(args.mlx_binary_args),
+ "cuda": shlex.split(args.cuda_binary_args),
+ }
+
+ if args.runs <= 0 or args.warmups < 0:
+ raise SystemExit("--runs must be > 0 and --warmups must be >= 0")
+ for case in cases:
+ if case <= 0:
+ raise SystemExit(f"case token count must be a positive integer, got: {case}")
+
+ report = {
+ "created_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+ "binary": os.path.abspath(args.binary),
+ "model_dir": os.path.abspath(args.model_dir),
+ "model": read_model_config(args.model_dir),
+ "host": {
+ "platform": platform.platform(),
+ "machine": platform.machine(),
+ "processor": platform.processor(),
+ "python": platform.python_version(),
+ },
+ "config": {
+ "backends": backends,
+ "batch_sizes": batch_sizes,
+ "cases": cases,
+ "runs": args.runs,
+ "warmups": args.warmups,
+ "threads": args.threads,
+ },
+ "results": [],
+ "errors": [],
+ }
+
+ for backend in backends:
+ for batch_size in batch_sizes:
+ print(f"\n== backend={backend} batch={batch_size} ==")
+ stdin_runner = StdinRunner(
+ args.binary, args.model_dir, backend, batch_size, args.threads,
+ common_binary_args, backend_binary_args.get(backend, []))
+ try:
+ stdin_runner.start()
+ rss_after_warmup = stdin_runner.rss_mb()
+ print(
+ f"load+first_batch={stdin_runner.load_ms:.1f} ms"
+ + (f", rss={rss_after_warmup:.0f} MB" if rss_after_warmup else "")
+ )
+ for case in cases:
+ r = run_case(stdin_runner, case, args.runs, args.warmups)
+ r.update({
+ "backend": backend,
+ "load_ms": stdin_runner.load_ms,
+ "rss_mb": stdin_runner.rss_mb() or rss_after_warmup,
+ })
+ report["results"].append(r)
+ print(
+ f" {case:<7} tokens={r['tokens_per_batch']:>4} "
+ f"p50={r['p50_ms']:.1f} ms "
+ f"p95={r['p95_ms']:.1f} ms "
+ f"txt/s={r['texts_per_sec_p50']:.1f}"
+ + (f" ws={r['workspace_bytes'] / (1024 * 1024):.1f} MB"
+ if r.get("workspace_bytes") else "")
+ )
+ except Exception as exc:
+ err = {"backend": backend, "batch_size": batch_size, "error": str(exc)}
+ report["errors"].append(err)
+ print(f"error: {backend} batch={batch_size}: {exc}", file=sys.stderr)
+ if not args.continue_on_error:
+ stdin_runner.stop()
+ raise
+ finally:
+ stdin_runner.stop()
+
+ print("\nSummary")
+ print_table(report["results"])
+
+ if args.json_out:
+ with open(args.json_out, "w", encoding="utf-8") as f:
+ json.dump(report, f, indent=2)
+ f.write("\n")
+ print(f"\nWrote {args.json_out}")
+
+ if args.markdown_out:
+ with open(args.markdown_out, "w", encoding="utf-8") as f:
+ f.write("# ffwd benchmark matrix\n\n")
+ f.write(f"- Created: `{report['created_at']}`\n")
+ f.write(f"- Binary: `{report['binary']}`\n")
+ f.write(f"- Model: `{report['model_dir']}`\n")
+ f.write(f"- Host: `{report['host']['platform']}`\n\n")
+ f.write(markdown_table(report["results"]))
+ if report["errors"]:
+ f.write("\n## Errors\n\n")
+ for err in report["errors"]:
+ f.write(f"- `{err['backend']}` batch `{err['batch_size']}`: {err['error']}\n")
+ print(f"Wrote {args.markdown_out}")
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except KeyboardInterrupt:
+ raise
+ except Exception as exc:
+ print(f"fatal: {exc}", file=sys.stderr)
+ sys.exit(1)
diff --git a/bench/bench_model.c b/bench/bench_model.c
index b2d4769..7cb6458 100644
--- a/bench/bench_model.c
+++ b/bench/bench_model.c
@@ -10,16 +10,16 @@
*/
#include "bench.h"
-#include "embed.h"
-#include "kernels.h"
+#include "internal.h"
+#include "threadpool.h"
-static embed_model_t *g_model;
-static embed_workspace_t *g_ws;
+static ffwd_model_t *g_model;
+static ffwd_workspace_t *g_ws;
static int g_hidden;
enum { MAX_BATCH = 8 };
-static void bm_embed(bench_state_t *b, int batch, int len) {
+static void bm_ffwd(bench_state_t *b, int batch, int len) {
int *ids = (int *)malloc((size_t)batch * len * sizeof(int));
float *out = (float *)malloc((size_t)batch * g_hidden * sizeof(float));
if (!ids || !out)
@@ -29,14 +29,14 @@ static void bm_embed(bench_state_t *b, int batch, int len) {
s = s * 1664525u + 1013904223u;
ids[i] = (int)(s % 1000u); /* ordinary vocab rows */
}
- embed_input_t inputs[MAX_BATCH];
+ ffwd_input_t inputs[MAX_BATCH];
for (int i = 0; i < batch; i++) {
inputs[i].ids = ids + (size_t)i * len;
inputs[i].n_tokens = len;
}
bench_begin(b);
for (long i = 0; i < b->n; i++) {
- if (embed_model_encode_batch(g_model, g_ws, inputs, batch, out) != 0)
+ if (ffwd_model_encode_batch(g_model, g_ws, inputs, batch, out) != 0)
exit(2);
bench_sink += out[0];
}
@@ -44,14 +44,14 @@ static void bm_embed(bench_state_t *b, int batch, int len) {
free(ids);
}
-static void bm_embed_b1_len32(bench_state_t *b) { bm_embed(b, 1, 32); }
-static void bm_embed_b8_len32(bench_state_t *b) { bm_embed(b, 8, 32); }
-static void bm_embed_b1_len256(bench_state_t *b) { bm_embed(b, 1, 256); }
+static void bm_ffwd_b1_len32(bench_state_t *b) { bm_ffwd(b, 1, 32); }
+static void bm_ffwd_b8_len32(bench_state_t *b) { bm_ffwd(b, 8, 32); }
+static void bm_ffwd_b1_len256(bench_state_t *b) { bm_ffwd(b, 1, 256); }
static const bench_case_t CASES[] = {
- {"embed/b1_len32", bm_embed_b1_len32},
- {"embed/b8_len32", bm_embed_b8_len32},
- {"embed/b1_len256", bm_embed_b1_len256},
+ {"ffwd/b1_len32", bm_ffwd_b1_len32},
+ {"ffwd/b8_len32", bm_ffwd_b8_len32},
+ {"ffwd/b1_len256", bm_ffwd_b1_len256},
};
int main(int argc, char **argv) {
@@ -67,19 +67,19 @@ int main(int argc, char **argv) {
}
if (threads < 1)
threads = 1;
- embed_set_threads(threads);
+ tp_set_threads(threads);
- g_model = embed_model_load(argv[1]);
+ g_model = ffwd_model_load(argv[1]);
if (!g_model) {
fprintf(stderr, "failed to load model from %s\n", argv[1]);
return 1;
}
- g_ws = embed_workspace_new(g_model);
+ g_ws = ffwd_workspace_new(g_model);
if (!g_ws) {
- embed_model_free(g_model);
+ ffwd_model_free(g_model);
return 1;
}
- g_hidden = embed_model_config(g_model)->hidden_size;
+ g_hidden = ffwd_model_config(g_model)->hidden_size;
char meta[160];
snprintf(meta, sizeof(meta), "suite=model threads=%d hidden=%d", threads, g_hidden);
@@ -89,7 +89,7 @@ int main(int argc, char **argv) {
bench_parse_args(&opts, argc - 1, argv + 1);
int rc = bench_main(&opts, CASES, (int)(sizeof(CASES) / sizeof(CASES[0])));
- embed_workspace_free(g_ws);
- embed_model_free(g_model);
+ ffwd_workspace_free(g_ws);
+ ffwd_model_free(g_model);
return rc;
}
diff --git a/bench/bench_qwen_tokenizer.py b/bench/bench_qwen_tokenizer.py
new file mode 100755
index 0000000..a69921d
--- /dev/null
+++ b/bench/bench_qwen_tokenizer.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Benchmark Qwen tokenizer encode paths.
+
+Builds the C tokenizer bench via the Makefile (`make -C bench bench_tokenizer`)
+and runs it against a Qwen model's vocab.json with a curated text set.
+
+ bench/bench_qwen_tokenizer.py --model-dir DIR
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+DEFAULT_TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: dense retrieval systems need stable tokenization.",
+ "unicode: cafe, café, Istanbul, İstanbul, emoji \U0001f50d",
+ "mixed whitespace:\n one\t two three\n\nfour",
+ ("long document: " + "embedding inference should batch tokenization and "
+ "model execution carefully. ") * 32,
+]
+
+
+def build(cc: str) -> Path:
+ """Build bench/bench_tokenizer via the Makefile; return its path."""
+ proc = subprocess.run(["make", "-C", "bench", f"CC={cc}", "bench_tokenizer"],
+ cwd=ROOT, text=True, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, check=False)
+ if proc.returncode != 0:
+ sys.stderr.write(proc.stdout)
+ raise SystemExit(proc.returncode)
+ return ROOT / "bench" / "bench_tokenizer"
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--runs", type=int, default=1000)
+ ap.add_argument("--cc", default=os.environ.get("CC", "cc"))
+ ap.add_argument("--text", action="append",
+ help="text to benchmark; may be repeated")
+ args = ap.parse_args()
+
+ binary = build(args.cc)
+ vocab = Path(args.model_dir) / "vocab.json"
+ texts = args.text if args.text else DEFAULT_TEXTS
+ return subprocess.run([str(binary), str(vocab), str(args.runs), *texts],
+ cwd=ROOT).returncode
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/bench/bench_server_api.py b/bench/bench_server_api.py
new file mode 100755
index 0000000..76fbba1
--- /dev/null
+++ b/bench/bench_server_api.py
@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""Concurrent benchmark for the local ffwd HTTP API."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import concurrent.futures as cf
+import http.client
+import json
+import math
+import os
+import statistics
+import subprocess
+import threading
+import time
+import urllib.error
+import urllib.request
+import urllib.parse
+from typing import Any
+
+
+TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "document: Istanbul is a major city in Turkey.",
+ "A small embedding request for throughput testing.",
+ "Redis is an in-memory data structure server used for caching.",
+ "SQLite stores relational data in a local file.",
+ "Machine learning systems often use embeddings for semantic retrieval.",
+]
+
+_thread_local = threading.local()
+
+
+def percentile(values: list[float], p: float) -> float:
+ xs = sorted(values)
+ if len(xs) == 1:
+ return xs[0]
+ rank = (len(xs) - 1) * (p / 100.0)
+ lo = int(rank)
+ hi = min(lo + 1, len(xs) - 1)
+ frac = rank - lo
+ return xs[lo] * (1.0 - frac) + xs[hi] * frac
+
+
+def decode_int8(s: str) -> list[float]:
+ return [float(b - 256 if b > 127 else b) for b in base64.b64decode(s)]
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def make_input(request_id: int, texts_per_request: int) -> list[str]:
+ out = []
+ for i in range(texts_per_request):
+ out.append(TEXTS[(request_id + i) % len(TEXTS)])
+ return out
+
+
+def keepalive_connection(args: argparse.Namespace) -> tuple[http.client.HTTPConnection, str]:
+ url = urllib.parse.urlparse(args.base_url)
+ if url.scheme and url.scheme != "http":
+ raise RuntimeError("--keepalive supports http URLs only")
+ host = url.hostname or "127.0.0.1"
+ port = url.port or 80
+ prefix = url.path.rstrip("/")
+ key = (host, port, prefix)
+ cached = getattr(_thread_local, "conn", None)
+ cached_key = getattr(_thread_local, "key", None)
+ if cached is None or cached_key != key:
+ if cached is not None:
+ cached.close()
+ cached = http.client.HTTPConnection(host, port, timeout=args.timeout)
+ _thread_local.conn = cached
+ _thread_local.key = key
+ return cached, prefix
+
+
+def parse_server_timing(value: str | None) -> dict[str, float]:
+ out: dict[str, float] = {}
+ if not value:
+ return out
+ for part in value.split(","):
+ fields = [x.strip() for x in part.strip().split(";")]
+ if not fields or not fields[0]:
+ continue
+ name = fields[0]
+ for field in fields[1:]:
+ if field.startswith("dur="):
+ try:
+ out[name] = float(field[4:])
+ except ValueError:
+ pass
+ return out
+
+
+def post_embeddings(args: argparse.Namespace, request_id: int,
+ texts: list[str] | None = None) -> tuple[float, int, int, dict[str, float]]:
+ if texts is None:
+ texts = make_input(request_id, args.texts_per_request)
+ payload = {
+ "model": args.model,
+ "dimensions": args.dimensions,
+ "input": texts,
+ }
+ body = json.dumps(payload).encode("utf-8")
+ headers = {"Content-Type": "application/json"}
+ if args.api_key:
+ headers["Authorization"] = f"Bearer {args.api_key}"
+ t0 = time.perf_counter()
+ if args.keepalive:
+ conn, prefix = keepalive_connection(args)
+ try:
+ conn.request("POST", prefix + "/v1/embeddings", body, headers)
+ resp = conn.getresponse()
+ raw = resp.read()
+ if resp.status != 200:
+ raise RuntimeError(
+ f"HTTP {resp.status}: {raw.decode('utf-8', errors='replace')}"
+ )
+ if (resp.getheader("Connection") or "").lower() == "close":
+ raise RuntimeError("server closed keep-alive benchmark connection")
+ timing = parse_server_timing(resp.getheader("Server-Timing"))
+ data = json.loads(raw.decode("utf-8"))
+ except Exception:
+ conn.close()
+ _thread_local.conn = None
+ raise
+ else:
+ req = urllib.request.Request(
+ args.base_url.rstrip("/") + "/v1/embeddings",
+ data=body, headers=headers)
+ try:
+ with urllib.request.urlopen(req, timeout=args.timeout) as resp:
+ timing = parse_server_timing(resp.headers.get("Server-Timing"))
+ data = json.loads(resp.read().decode("utf-8"))
+ except urllib.error.HTTPError as e:
+ text = e.read().decode("utf-8", errors="replace")
+ raise RuntimeError(f"HTTP {e.code}: {text}") from e
+ elapsed_ms = (time.perf_counter() - t0) * 1000.0
+
+ rows = data["data"]
+ if len(rows) != len(texts):
+ raise RuntimeError(f"expected {len(texts)} embeddings, got {len(rows)}")
+ for i, row in enumerate(rows):
+ if row["index"] != i:
+ raise RuntimeError(f"bad index at request {request_id}: {row['index']} != {i}")
+ raw = base64.b64decode(row["embedding"])
+ if len(raw) != args.dimensions:
+ raise RuntimeError(f"bad decoded length: {len(raw)} != {args.dimensions}")
+
+ return elapsed_ms, len(texts), int(data["usage"]["prompt_tokens"]), timing
+
+
+def verify_paris_ranking(args: argparse.Namespace) -> bool:
+ texts = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ ]
+ body = json.dumps({
+ "model": args.model,
+ "dimensions": args.dimensions,
+ "input": texts,
+ }).encode("utf-8")
+ headers = {"Content-Type": "application/json"}
+ if args.api_key:
+ headers["Authorization"] = f"Bearer {args.api_key}"
+ req = urllib.request.Request(args.base_url.rstrip("/") + "/v1/embeddings",
+ data=body, headers=headers)
+ with urllib.request.urlopen(req, timeout=args.timeout) as resp:
+ data = json.loads(resp.read().decode("utf-8"))
+ rows = data["data"]
+ if len(rows) != len(texts):
+ raise RuntimeError(f"ranking probe expected {len(texts)} embeddings, got {len(rows)}")
+ vecs = [decode_int8(row["embedding"]) for row in data["data"]]
+ return cosine(vecs[0], vecs[1]) > cosine(vecs[0], vecs[2])
+
+
+def rss_mb(pid: int | None) -> float | None:
+ if not pid:
+ return None
+ try:
+ out = subprocess.check_output(["ps", "-o", "rss=", "-p", str(pid)], text=True)
+ kb = int(out.strip())
+ return kb / 1024.0
+ except Exception:
+ return None
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--base-url", default="http://127.0.0.1:8000")
+ ap.add_argument("--api-key")
+ ap.add_argument("--model", default="pplx-embed-v1-0.6b")
+ ap.add_argument("--dimensions", type=int, default=128)
+ ap.add_argument("--requests", type=int, default=100)
+ ap.add_argument("--concurrency", type=int, default=16)
+ ap.add_argument("--texts-per-request", type=int, default=1)
+ ap.add_argument("--timeout", type=float, default=120.0)
+ ap.add_argument("--server-pid", type=int)
+ ap.add_argument("--verify-ranking", action="store_true")
+ ap.add_argument("--keepalive", action="store_true",
+ help="reuse one HTTP/1.1 connection per benchmark worker thread")
+ args = ap.parse_args()
+
+ if args.requests <= 0 or args.concurrency <= 0 or args.texts_per_request <= 0:
+ raise SystemExit("requests, concurrency, and texts-per-request must be positive")
+
+ t0 = time.perf_counter()
+ latencies: list[float] = []
+ timing_rows: list[dict[str, float]] = []
+ wall_minus_worker: list[float] = []
+ total_texts = 0
+ total_tokens = 0
+ with cf.ThreadPoolExecutor(max_workers=args.concurrency) as ex:
+ futs = [ex.submit(post_embeddings, args, i) for i in range(args.requests)]
+ for fut in cf.as_completed(futs):
+ ms, texts, tokens, timing = fut.result()
+ latencies.append(ms)
+ if timing:
+ timing_rows.append(timing)
+ if "worker" in timing:
+ wall_minus_worker.append(ms - timing["worker"])
+ total_texts += texts
+ total_tokens += tokens
+ wall = time.perf_counter() - t0
+ ranking_ok = verify_paris_ranking(args) if args.verify_ranking else True
+
+ print(f"server: {args.base_url} model={args.model} dim={args.dimensions}")
+ mode = "keepalive" if args.keepalive else "native-http"
+ print(f"requests={args.requests} concurrency={args.concurrency} texts/request={args.texts_per_request} mode={mode}")
+ print(f"wall={wall:.3f}s req/s={args.requests / wall:.2f} texts/s={total_texts / wall:.2f} tokens/s={total_tokens / wall:.2f}")
+ print(
+ "latency_ms "
+ f"mean={statistics.fmean(latencies):.2f} "
+ f"p50={percentile(latencies, 50):.2f} "
+ f"p95={percentile(latencies, 95):.2f} "
+ f"p99={percentile(latencies, 99):.2f} "
+ f"max={max(latencies):.2f}"
+ )
+ mem = rss_mb(args.server_pid)
+ if mem is not None:
+ print(f"rss_mb={mem:.1f}")
+ if timing_rows:
+ for name in ("queue", "parse", "tokenize", "infer", "encode", "worker"):
+ values = [row[name] for row in timing_rows if name in row]
+ if values:
+ print(
+ f"server_timing_{name}_ms "
+ f"mean={statistics.fmean(values):.3f} "
+ f"p50={percentile(values, 50):.3f} "
+ f"p95={percentile(values, 95):.3f}"
+ )
+ if wall_minus_worker:
+ print(
+ "wall_minus_worker_ms "
+ f"mean={statistics.fmean(wall_minus_worker):.3f} "
+ f"p50={percentile(wall_minus_worker, 50):.3f} "
+ f"p95={percentile(wall_minus_worker, 95):.3f}"
+ )
+ if args.verify_ranking:
+ print(f"ranking={'ok' if ranking_ok else 'FAIL'}")
+ return 0 if ranking_ok else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index 8175ffe..9bb8822 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -1,9 +1,9 @@
/* bench/bench_tokenizer.c - tokenizer encode-path benchmark.
* Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */
-#include "tokenizer_bpe.h"
-#include "tokenizer_sentencepiece.h"
-#include "tokenizer_wordpiece.h"
+#include "bpe.h"
+#include "sentencepiece.h"
+#include "wordpiece.h"
#include <stdio.h>
#include <stdlib.h>
@@ -11,8 +11,6 @@
#include <unistd.h>
#include <time.h>
-int embed_verbose = 0;
-
static double now_ms(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -50,8 +48,8 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) {
snprintf(p, sizeof(p), "%s/vocab.txt", path);
if (path_exists(p)) {
bt->kind = TOK_WORDPIECE;
- bt->tok = wordpiece_tokenizer_load(path);
- bt->ws = wordpiece_workspace_new();
+ bt->tok = tok_wp_load(path);
+ bt->ws = tok_wp_workspace_new();
return bt->tok && bt->ws ? 0 : -1;
}
snprintf(p, sizeof(p), "%s/sentencepiece.bpe.model", path);
@@ -62,8 +60,8 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) {
}
if (path_exists(p)) {
bt->kind = TOK_SENTENCEPIECE;
- bt->tok = sentencepiece_tokenizer_load(path);
- bt->ws = sentencepiece_workspace_new();
+ bt->tok = tok_spm_load(path);
+ bt->ws = tok_spm_workspace_new();
return bt->tok && bt->ws ? 0 : -1;
}
bt->kind = TOK_BPE;
@@ -71,64 +69,63 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) {
snprintf(p, sizeof(p), "%s/vocab.json", path);
if (path_exists(p))
vocab = p;
- bt->tok = embed_tokenizer_load(vocab);
- bt->ws = embed_tokenizer_workspace_new();
+ bt->tok = tok_bpe_load(vocab);
+ bt->ws = tok_bpe_workspace_new();
return bt->tok && bt->ws ? 0 : -1;
}
static int *tok_encode(bench_tok_t *bt, const char *text, int *n) {
switch (bt->kind) {
case TOK_WORDPIECE:
- return wordpiece_tokenizer_encode((wordpiece_tokenizer_t *)bt->tok, text, n);
+ return tok_wp_encode((tok_wp_t *)bt->tok, text, n);
case TOK_SENTENCEPIECE:
- return sentencepiece_tokenizer_encode((sentencepiece_tokenizer_t *)bt->tok, text, n);
+ return tok_spm_encode((tok_spm_t *)bt->tok, text, n);
default:
- return embed_tokenizer_encode((embed_tokenizer_t *)bt->tok, text, n);
+ return tok_bpe_encode((tok_bpe_t *)bt->tok, text, n);
}
}
static int *tok_encode_ws(bench_tok_t *bt, const char *text, int *n) {
switch (bt->kind) {
case TOK_WORDPIECE:
- return wordpiece_tokenizer_encode_with_workspace((wordpiece_tokenizer_t *)bt->tok,
- (wordpiece_workspace_t *)bt->ws, text, n);
+ return tok_wp_encode_with_workspace((tok_wp_t *)bt->tok,
+ (tok_wp_workspace_t *)bt->ws, text, n);
case TOK_SENTENCEPIECE:
- return sentencepiece_tokenizer_encode_with_workspace(
- (sentencepiece_tokenizer_t *)bt->tok, (sentencepiece_workspace_t *)bt->ws, text, n);
+ return tok_spm_encode_with_workspace(
+ (tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws, text, n);
default:
- return embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)bt->tok,
- (embed_tokenizer_workspace_t *)bt->ws, text, n);
+ return tok_bpe_encode_with_workspace((tok_bpe_t *)bt->tok,
+ (tok_bpe_workspace_t *)bt->ws, text, n);
}
}
static int tok_encode_into(bench_tok_t *bt, const char *text, int *out, int cap, int *n) {
switch (bt->kind) {
case TOK_WORDPIECE:
- return wordpiece_tokenizer_encode_into((wordpiece_tokenizer_t *)bt->tok,
- (wordpiece_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_wp_encode_into((tok_wp_t *)bt->tok,
+ (tok_wp_workspace_t *)bt->ws, text, out, cap, n);
case TOK_SENTENCEPIECE:
- return sentencepiece_tokenizer_encode_into((sentencepiece_tokenizer_t *)bt->tok,
- (sentencepiece_workspace_t *)bt->ws, text, out,
- cap, n);
+ return tok_spm_encode_into((tok_spm_t *)bt->tok,
+ (tok_spm_workspace_t *)bt->ws, text, out, cap, n);
default:
- return embed_tokenizer_encode_into((embed_tokenizer_t *)bt->tok,
- (embed_tokenizer_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_bpe_encode_into((tok_bpe_t *)bt->tok,
+ (tok_bpe_workspace_t *)bt->ws, text, out, cap, n);
}
}
static void free_tokenizer(bench_tok_t *bt) {
switch (bt->kind) {
case TOK_WORDPIECE:
- wordpiece_workspace_free((wordpiece_workspace_t *)bt->ws);
- wordpiece_tokenizer_free((wordpiece_tokenizer_t *)bt->tok);
+ tok_wp_workspace_free((tok_wp_workspace_t *)bt->ws);
+ tok_wp_free((tok_wp_t *)bt->tok);
break;
case TOK_SENTENCEPIECE:
- sentencepiece_workspace_free((sentencepiece_workspace_t *)bt->ws);
- sentencepiece_tokenizer_free((sentencepiece_tokenizer_t *)bt->tok);
+ tok_spm_workspace_free((tok_spm_workspace_t *)bt->ws);
+ tok_spm_free((tok_spm_t *)bt->tok);
break;
default:
- embed_tokenizer_workspace_free((embed_tokenizer_workspace_t *)bt->ws);
- embed_tokenizer_free((embed_tokenizer_t *)bt->tok);
+ tok_bpe_workspace_free((tok_bpe_workspace_t *)bt->ws);
+ tok_bpe_free((tok_bpe_t *)bt->tok);
break;
}
}
diff --git a/tools/benchstat.py b/bench/benchstat.py
index 7f8b695..7f8b695 100755
--- a/tools/benchstat.py
+++ b/bench/benchstat.py
diff --git a/requirements-reference.txt b/requirements-reference.txt
new file mode 100644
index 0000000..733e740
--- /dev/null
+++ b/requirements-reference.txt
@@ -0,0 +1,5 @@
+numpy==2.4.6
+onnxruntime==1.26.0
+sentence-transformers==5.5.1
+torch==2.12.0
+transformers==5.4.0