diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-03-28 23:38:06 +0100 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-03-28 23:38:06 +0100 |
| commit | 353aa5394a11ef8a9c69e73338b95a3f7eac5855 (patch) | |
| tree | 414a8e39d7e19b2704088f8c7d691281625f1394 | |
| parent | c47e900b94d18cca1bf600e58f10dbcc2e34c1f7 (diff) | |
| download | ffwd-353aa5394a11ef8a9c69e73338b95a3f7eac5855.tar.gz ffwd-353aa5394a11ef8a9c69e73338b95a3f7eac5855.zip | |
Add benchmark script
| -rw-r--r-- | README.md | 46 | ||||
| -rw-r--r-- | bench.py | 500 |
2 files changed, 546 insertions, 0 deletions
@@ -85,3 +85,49 @@ Options: **Daemon mode** (`--daemon`): reads one text per line from stdin, writes one JSON object per line to stdout. EOF or empty line exits. +## Benchmark results + +### pplx-embed-v1-0.6b + +1024-dim, 28 layers, 2.38 GB + +**Inference latency** (median, 5 runs, Apple M4 Pro 12-core): + +| Sentence | Tokens | PyTorch | C/BLAS | Speedup | C/MLX | Speedup | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| short | 2 | 76.2 ms | 53.9 ms | 1.4x | 15.2 ms | 5.0x | +| medium | 13 | 75.7 ms | 50.7 ms | 1.5x | 15.4 ms | 4.9x | +| long | 63 | 97.4 ms | 58.9 ms | 1.7x | 20.0 ms | 4.9x | +| 256tok | 199 | 193.5 ms | 178.2 ms | 1.1x | 53.9 ms | 3.6x | + +**Load/memory**: + +| | PyTorch | C/BLAS | C/MLX | +| :--- | :--- | :--- | :--- | +| **Load time** | 186 ms | 195 ms | 363 ms | +| **RSS** | ~2163 MB | ~1737 MB | ~4603 MB | + +### pplx-embed-v1-4b + +2560-dim, 36 layers, 16.09 GB + +| Sentence | Tokens | PyTorch | C/BLAS | Speedup | C/MLX | Speedup | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| short | 2 | 462 ms | 402 ms | 1.1x | 97 ms | 4.8x | +| medium | 13 | 307 ms | 234 ms | 1.3x | 101 ms | 3.0x | +| long | 63 | 411 ms | 350 ms | 1.2x | 121 ms | 3.4x | +| 256tok | 199 | 875 ms | 919 ms | 1.0x | 376 ms | 2.3x | + +| | PyTorch | C/BLAS | C/MLX | +| :--- | :--- | :--- | :--- | +| **Load time** | 12094 ms | 1202 ms | 6134 ms | +| **RSS** | ~14253 MB | ~13902 MB | ~15483 MB | + +**Summary**: + +* C/MLX is up to **5x** faster than PyTorch on Apple M4 Pro, and C/BLAS is up to **1.7x** faster. +* C/BLAS uses ~20% less memory than PyTorch (1737 vs 2163 MB for 0.6B) because it mmap's weights directly instead of copying them into Python objects. +* MLX RSS is higher (~4.6 GB for 0.6B) because Metal duplicates weight data into GPU-accessible memory. This is the cost of GPU speed. +* Load times: C/BLAS loads fastest (mmap, no conversion). MLX has a one-time Metal kernel compilation cost. PyTorch's 12-second load for 4B is because the first inference faults in the entire mmap'd file. +* Accuracy is identical (cosine sim = 1.000000, max diff < 2e-6) between PyTorch and C implementations. + diff --git a/bench.py b/bench.py new file mode 100644 index 0000000..4cd1554 --- /dev/null +++ b/bench.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +""" +bench.py - Benchmark pplx-embed: C (CPU/BLAS + MLX) vs Python/PyTorch + +Measures: + - Model load time + - Inference latency (varying sentence lengths) + - Memory usage + - Numerical accuracy (cosine similarity between implementations) + +Usage: + .venv/bin/python3 bench.py [--model-dir /path/to/model] [--runs 5] +""" + +import argparse +import json +import math +import os +import subprocess +import sys +import time + +import numpy as np +import psutil +import safetensors.torch as st +import torch + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +SENTENCES = { + "short": "hello world", + "medium": "The quick brown fox jumps over the lazy dog near the riverbank", + "long": ( + "Retrieval-augmented generation is a technique that combines large " + "language models with external knowledge sources to produce more " + "accurate and up-to-date responses. The system first retrieves " + "relevant documents from a corpus using dense vector similarity " + "search, then conditions the language model on both the query and " + "the retrieved passages to generate a final answer." + ), + "256tok": ( + "In the field of natural language processing, embedding models play " + "a crucial role in converting text into dense vector representations " + "that capture semantic meaning. These representations enable a wide " + "range of downstream tasks including semantic search, clustering, " + "classification, and retrieval-augmented generation. Modern embedding " + "models are typically based on transformer architectures that process " + "input text bidirectionally, allowing each token to attend to all " + "other tokens in the sequence. This is in contrast to autoregressive " + "language models which only attend to previous tokens. The resulting " + "embeddings are obtained by pooling the final hidden states across " + "all token positions, usually via mean pooling, followed by L2 " + "normalization to produce unit-length vectors. The cosine similarity " + "between two such vectors then serves as a measure of semantic " + "relatedness. Training these models typically involves contrastive " + "learning objectives where the model learns to produce similar " + "embeddings for semantically related texts and dissimilar embeddings " + "for unrelated texts. The choice of training data, negative sampling " + "strategy, and loss function all significantly impact the quality of " + "the resulting embeddings." + ), +} + + +# --------------------------------------------------------------------------- +# Python/PyTorch reference (manual forward, no HuggingFace overhead) +# --------------------------------------------------------------------------- + +class PytorchEmbed: + """Minimal PyTorch forward pass, matching our C implementation exactly.""" + + def __init__(self, model_dir): + self.model_dir = model_dir + self.config = json.load(open(f"{model_dir}/config.json")) + self.hidden = self.config["hidden_size"] + self.n_layers = self.config["num_hidden_layers"] + self.n_heads = self.config["num_attention_heads"] + self.n_kv_heads = self.config["num_key_value_heads"] + self.head_dim = self.config.get("head_dim", 128) + self.eps = self.config.get("rms_norm_eps", 1e-6) + self.rope_theta = self.config.get("rope_theta", 1e6) + self.weights = None + + def load(self): + """Load weights from safetensors.""" + # Handle single or multi-shard + import glob + shards = sorted(glob.glob(f"{self.model_dir}/model-*.safetensors")) + if shards: + self.weights = {} + for s in shards: + self.weights.update(st.load_file(s)) + else: + self.weights = st.load_file(f"{self.model_dir}/model.safetensors") + + def _rms_norm(self, x, w): + rms = torch.sqrt((x * x).mean(-1, keepdim=True) + self.eps) + return x / rms * w + + def _rope_neox(self, x, cos, sin): + half = x.shape[-1] // 2 + x1, x2 = x[..., :half], x[..., half:] + return torch.cat([ + x1 * cos[..., :half] - x2 * sin[..., :half], + x2 * cos[..., :half] + x1 * sin[..., :half], + ], dim=-1) + + def embed(self, token_ids): + """Run forward pass, return L2-normalized embedding.""" + w = self.weights + seq = len(token_ids) + ids = torch.tensor(token_ids) + hd = self.head_dim + + x = w["embed_tokens.weight"][ids].float() + + # RoPE tables + inv_freq = 1.0 / (self.rope_theta ** (torch.arange(0, hd, 2).float() / hd)) + pos = torch.arange(seq).float() + freqs = torch.outer(pos, inv_freq) + cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1) + sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1) + + scale = 1.0 / math.sqrt(hd) + heads_per_kv = self.n_heads // self.n_kv_heads + + for i in range(self.n_layers): + p = f"layers.{i}" + + xn = self._rms_norm(x, w[f"{p}.input_layernorm.weight"]) + + q = xn @ w[f"{p}.self_attn.q_proj.weight"].T + k = xn @ w[f"{p}.self_attn.k_proj.weight"].T + v = xn @ w[f"{p}.self_attn.v_proj.weight"].T + + q = self._rms_norm(q.view(seq, self.n_heads, hd), w[f"{p}.self_attn.q_norm.weight"]) + k = self._rms_norm(k.view(seq, self.n_kv_heads, hd), w[f"{p}.self_attn.k_norm.weight"]) + + q = self._rope_neox(q, cos.unsqueeze(1), sin.unsqueeze(1)) + k = self._rope_neox(k, cos.unsqueeze(1), sin.unsqueeze(1)) + + q = q.reshape(seq, -1) + k = k.reshape(seq, -1) + + attn_out = torch.zeros_like(q) + for h in range(self.n_heads): + kv_h = h // heads_per_kv + qh = q[:, h*hd:(h+1)*hd] + kh = k[:, kv_h*hd:(kv_h+1)*hd] + vh = v[:, kv_h*hd:(kv_h+1)*hd] + scores = torch.softmax((qh @ kh.T) * scale, dim=-1) + attn_out[:, h*hd:(h+1)*hd] = scores @ vh + + x = x + attn_out @ w[f"{p}.self_attn.o_proj.weight"].T + + xn = self._rms_norm(x, w[f"{p}.post_attention_layernorm.weight"]) + gate = xn @ w[f"{p}.mlp.gate_proj.weight"].T + up = xn @ w[f"{p}.mlp.up_proj.weight"].T + x = x + (torch.sigmoid(gate) * gate * up) @ w[f"{p}.mlp.down_proj.weight"].T + + x = self._rms_norm(x, w["norm.weight"]) + emb = x.mean(0) + emb = emb / emb.norm() + return emb.numpy() + + +# --------------------------------------------------------------------------- +# C backend runner (via --daemon mode) +# --------------------------------------------------------------------------- + +class CEmbed: + """Communicate with the C binary via --daemon JSON-lines protocol.""" + + def __init__(self, binary, model_dir, backend="cpu"): + self.binary = binary + self.model_dir = model_dir + self.backend = backend + self.proc = None + self.load_ms = 0 + + def start(self): + cmd = [self.binary, "-d", self.model_dir, "--daemon"] + if self.backend == "mlx": + cmd.append("--mlx") + t0 = time.perf_counter() + self.proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + # Warm up: send one text to trigger model load + kernel compile + self.proc.stdin.write("warmup\n") + self.proc.stdin.flush() + line = self.proc.stdout.readline() + self.load_ms = (time.perf_counter() - t0) * 1000 + # Parse warmup result to confirm it works + try: + json.loads(line) + except Exception: + stderr = self.proc.stderr.read() + raise RuntimeError(f"C daemon failed to start:\n{stderr}") + + def embed(self, text): + self.proc.stdin.write(text + "\n") + self.proc.stdin.flush() + line = self.proc.stdout.readline() + result = json.loads(line) + if "error" in result: + raise RuntimeError(result["error"]) + return np.array(result["embedding"], dtype=np.float32), result["ms"] + + def stop(self): + if self.proc: + self.proc.stdin.close() + self.proc.wait(timeout=5) + self.proc = None + + def get_memory_mb(self): + if not self.proc: + return 0 + try: + p = psutil.Process(self.proc.pid) + return p.memory_info().rss / (1024 * 1024) + except Exception: + return 0 + + +# --------------------------------------------------------------------------- +# Tokenizer helper (use the same BPE as C) +# --------------------------------------------------------------------------- + +def load_tokenizer(model_dir): + from transformers import AutoTokenizer + return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) + + +# --------------------------------------------------------------------------- +# Benchmark runner +# --------------------------------------------------------------------------- + +def cosine_sim(a, b): + return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))) + + +def measure_pytorch(model, tokenizer, texts, n_runs): + """Measure PyTorch inference times and return embeddings.""" + results = {} + for name, text in texts.items(): + ids = tokenizer.encode(text, add_special_tokens=False) + n_tok = len(ids) + + # Warmup + model.embed(ids) + + times = [] + emb = None + for _ in range(n_runs): + t0 = time.perf_counter() + emb = model.embed(ids) + times.append((time.perf_counter() - t0) * 1000) + + results[name] = { + "tokens": n_tok, + "times_ms": times, + "median_ms": float(np.median(times)), + "embedding": emb, + } + return results + + +def measure_c(engine, texts, tokenizer, n_runs): + """Measure C backend inference times and return embeddings.""" + results = {} + for name, text in texts.items(): + ids = tokenizer.encode(text, add_special_tokens=False) + n_tok = len(ids) + + # Warmup (second call, first was during start) + engine.embed(text) + + times = [] + emb = None + for _ in range(n_runs): + emb, ms = engine.embed(text) + times.append(ms) + + results[name] = { + "tokens": n_tok, + "times_ms": times, + "median_ms": float(np.median(times)), + "embedding": emb, + } + return results + + +def print_table(headers, rows, col_widths=None): + if col_widths is None: + col_widths = [max(len(str(h)), max(len(str(r[i])) for r in rows)) + for i, h in enumerate(headers)] + fmt = " ".join(f"{{:<{w}}}" for w in col_widths) + print(fmt.format(*headers)) + print(fmt.format(*["-" * w for w in col_widths])) + for row in rows: + print(fmt.format(*row)) + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark pplx-embed") + parser.add_argument("--model-dir", required=True, help="Model directory") + parser.add_argument("--binary", default="./pplx_embed", help="C binary path") + parser.add_argument("--runs", type=int, default=5, help="Runs per measurement") + args = parser.parse_args() + + model_dir = args.model_dir + config = json.load(open(f"{model_dir}/config.json")) + dim = config["hidden_size"] + n_layers = config["num_hidden_layers"] + model_name = os.path.basename(os.path.dirname(model_dir)) or os.path.basename(model_dir) + + print(f"Benchmark: {model_name}") + print(f" hidden={dim}, layers={n_layers}, runs={args.runs}") + print() + + tokenizer = load_tokenizer(model_dir) + + # Token counts + print("Sentences:") + for name, text in SENTENCES.items(): + ids = tokenizer.encode(text, add_special_tokens=False) + preview = text[:60] + "..." if len(text) > 60 else text + print(f" {name:8s} {len(ids):3d} tokens \"{preview}\"") + print() + + # --- Model size --- + import glob + shard_files = sorted(glob.glob(f"{model_dir}/model*.safetensors")) + model_bytes = sum(os.path.getsize(f) for f in shard_files) + print(f"Model size on disk: {model_bytes / 1e9:.2f} GB ({len(shard_files)} shard(s))") + print() + + # --- PyTorch --- + print("=" * 60) + print("PyTorch (manual forward, float32, CPU)") + print("=" * 60) + + py_model = PytorchEmbed(model_dir) + proc = psutil.Process() + + t0 = time.perf_counter() + py_model.load() + py_load_ms = (time.perf_counter() - t0) * 1000 + + # Force actual loading by running one inference (mmap is lazy) + warmup_ids = tokenizer.encode("warmup", add_special_tokens=False) + mem_before = proc.memory_info().rss + _ = py_model.embed(warmup_ids) + mem_after = proc.memory_info().rss + py_first_ms = (time.perf_counter() - t0) * 1000 # total including first inference + py_mem_mb = (mem_after - mem_before) / (1024 * 1024) + # Also measure total RSS for a fair comparison + py_total_rss = proc.memory_info().rss / (1024 * 1024) + + print(f" Load time: {py_load_ms:.0f} ms (mmap, lazy)") + print(f" First inference (load+embed): {py_first_ms:.0f} ms") + print(f" RSS after first inference: ~{py_total_rss:.0f} MB") + print() + + py_results = measure_pytorch(py_model, tokenizer, SENTENCES, args.runs) + + rows = [] + for name in SENTENCES: + r = py_results[name] + rows.append([name, str(r["tokens"]), f"{r['median_ms']:.1f}"]) + print_table(["sentence", "tokens", "median ms"], rows, [10, 8, 12]) + print() + + # Free pytorch weights + py_model.weights = None + torch.cuda.empty_cache() if torch.cuda.is_available() else None + import gc; gc.collect() + + # --- C/BLAS --- + print("=" * 60) + print("C/BLAS (Accelerate, multithreaded)") + print("=" * 60) + + c_cpu = CEmbed(args.binary, model_dir, "cpu") + c_cpu.start() + cpu_mem = c_cpu.get_memory_mb() + print(f" Load time: {c_cpu.load_ms:.0f} ms (includes warmup)") + print(f" RSS: ~{cpu_mem:.0f} MB") + print() + + cpu_results = measure_c(c_cpu, SENTENCES, tokenizer, args.runs) + c_cpu.stop() + + rows = [] + for name in SENTENCES: + r = cpu_results[name] + rows.append([name, str(r["tokens"]), f"{r['median_ms']:.1f}"]) + print_table(["sentence", "tokens", "median ms"], rows, [10, 8, 12]) + print() + + # --- C/MLX --- + print("=" * 60) + print("C/MLX (Metal GPU)") + print("=" * 60) + + c_mlx = CEmbed(args.binary, model_dir, "mlx") + c_mlx.start() + mlx_mem = c_mlx.get_memory_mb() + print(f" Load time: {c_mlx.load_ms:.0f} ms (includes warmup + kernel compile)") + print(f" RSS: ~{mlx_mem:.0f} MB") + print() + + mlx_results = measure_c(c_mlx, SENTENCES, tokenizer, args.runs) + c_mlx.stop() + + rows = [] + for name in SENTENCES: + r = mlx_results[name] + rows.append([name, str(r["tokens"]), f"{r['median_ms']:.1f}"]) + print_table(["sentence", "tokens", "median ms"], rows, [10, 8, 12]) + print() + + # --- Accuracy --- + print("=" * 60) + print("Accuracy (cosine similarity vs PyTorch reference)") + print("=" * 60) + print() + + rows = [] + for name in SENTENCES: + py_emb = py_results[name]["embedding"] + cpu_emb = cpu_results[name]["embedding"] + mlx_emb = mlx_results[name]["embedding"] + + sim_cpu = cosine_sim(py_emb, cpu_emb) + sim_mlx = cosine_sim(py_emb, mlx_emb) + max_diff_cpu = float(np.max(np.abs(py_emb - cpu_emb))) + max_diff_mlx = float(np.max(np.abs(py_emb - mlx_emb))) + + rows.append([ + name, + str(py_results[name]["tokens"]), + f"{sim_cpu:.8f}", + f"{max_diff_cpu:.2e}", + f"{sim_mlx:.8f}", + f"{max_diff_mlx:.2e}", + ]) + + print_table( + ["sentence", "tokens", "cos(cpu,py)", "max|diff|", "cos(mlx,py)", "max|diff|"], + rows, + [10, 8, 14, 12, 14, 12], + ) + print() + + # --- Summary comparison --- + print("=" * 60) + print("Summary: Median inference time (ms)") + print("=" * 60) + print() + + rows = [] + for name in SENTENCES: + py_ms = py_results[name]["median_ms"] + cpu_ms = cpu_results[name]["median_ms"] + mlx_ms = mlx_results[name]["median_ms"] + toks = py_results[name]["tokens"] + + rows.append([ + name, + str(toks), + f"{py_ms:.1f}", + f"{cpu_ms:.1f}", + f"{py_ms/cpu_ms:.1f}x" if cpu_ms > 0 else "-", + f"{mlx_ms:.1f}", + f"{py_ms/mlx_ms:.1f}x" if mlx_ms > 0 else "-", + ]) + + print_table( + ["sentence", "tok", "PyTorch", "C/BLAS", "speedup", "C/MLX", "speedup"], + rows, + [10, 5, 10, 10, 9, 10, 9], + ) + + print() + print(f"Load time: PyTorch {py_first_ms:.0f} ms (mmap+first embed) | C/BLAS {c_cpu.load_ms:.0f} ms | C/MLX {c_mlx.load_ms:.0f} ms") + print(f"RSS: PyTorch ~{py_total_rss:.0f} MB | C/BLAS ~{cpu_mem:.0f} MB | C/MLX ~{mlx_mem:.0f} MB") + print() + + +if __name__ == "__main__": + main() |
