aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bench.py500
1 files changed, 0 insertions, 500 deletions
diff --git a/bench.py b/bench.py
deleted file mode 100644
index 4cd1554..0000000
--- a/bench.py
+++ /dev/null
@@ -1,500 +0,0 @@
-#!/usr/bin/env python3
-"""
-bench.py - Benchmark pplx-embed: C (CPU/BLAS + MLX) vs Python/PyTorch
-
-Measures:
- - Model load time
- - Inference latency (varying sentence lengths)
- - Memory usage
- - Numerical accuracy (cosine similarity between implementations)
-
-Usage:
- .venv/bin/python3 bench.py [--model-dir /path/to/model] [--runs 5]
-"""
-
-import argparse
-import json
-import math
-import os
-import subprocess
-import sys
-import time
-
-import numpy as np
-import psutil
-import safetensors.torch as st
-import torch
-
-# ---------------------------------------------------------------------------
-# Config
-# ---------------------------------------------------------------------------
-
-SENTENCES = {
- "short": "hello world",
- "medium": "The quick brown fox jumps over the lazy dog near the riverbank",
- "long": (
- "Retrieval-augmented generation is a technique that combines large "
- "language models with external knowledge sources to produce more "
- "accurate and up-to-date responses. The system first retrieves "
- "relevant documents from a corpus using dense vector similarity "
- "search, then conditions the language model on both the query and "
- "the retrieved passages to generate a final answer."
- ),
- "256tok": (
- "In the field of natural language processing, embedding models play "
- "a crucial role in converting text into dense vector representations "
- "that capture semantic meaning. These representations enable a wide "
- "range of downstream tasks including semantic search, clustering, "
- "classification, and retrieval-augmented generation. Modern embedding "
- "models are typically based on transformer architectures that process "
- "input text bidirectionally, allowing each token to attend to all "
- "other tokens in the sequence. This is in contrast to autoregressive "
- "language models which only attend to previous tokens. The resulting "
- "embeddings are obtained by pooling the final hidden states across "
- "all token positions, usually via mean pooling, followed by L2 "
- "normalization to produce unit-length vectors. The cosine similarity "
- "between two such vectors then serves as a measure of semantic "
- "relatedness. Training these models typically involves contrastive "
- "learning objectives where the model learns to produce similar "
- "embeddings for semantically related texts and dissimilar embeddings "
- "for unrelated texts. The choice of training data, negative sampling "
- "strategy, and loss function all significantly impact the quality of "
- "the resulting embeddings."
- ),
-}
-
-
-# ---------------------------------------------------------------------------
-# Python/PyTorch reference (manual forward, no HuggingFace overhead)
-# ---------------------------------------------------------------------------
-
-class PytorchEmbed:
- """Minimal PyTorch forward pass, matching our C implementation exactly."""
-
- def __init__(self, model_dir):
- self.model_dir = model_dir
- self.config = json.load(open(f"{model_dir}/config.json"))
- self.hidden = self.config["hidden_size"]
- self.n_layers = self.config["num_hidden_layers"]
- self.n_heads = self.config["num_attention_heads"]
- self.n_kv_heads = self.config["num_key_value_heads"]
- self.head_dim = self.config.get("head_dim", 128)
- self.eps = self.config.get("rms_norm_eps", 1e-6)
- self.rope_theta = self.config.get("rope_theta", 1e6)
- self.weights = None
-
- def load(self):
- """Load weights from safetensors."""
- # Handle single or multi-shard
- import glob
- shards = sorted(glob.glob(f"{self.model_dir}/model-*.safetensors"))
- if shards:
- self.weights = {}
- for s in shards:
- self.weights.update(st.load_file(s))
- else:
- self.weights = st.load_file(f"{self.model_dir}/model.safetensors")
-
- def _rms_norm(self, x, w):
- rms = torch.sqrt((x * x).mean(-1, keepdim=True) + self.eps)
- return x / rms * w
-
- def _rope_neox(self, x, cos, sin):
- half = x.shape[-1] // 2
- x1, x2 = x[..., :half], x[..., half:]
- return torch.cat([
- x1 * cos[..., :half] - x2 * sin[..., :half],
- x2 * cos[..., :half] + x1 * sin[..., :half],
- ], dim=-1)
-
- def embed(self, token_ids):
- """Run forward pass, return L2-normalized embedding."""
- w = self.weights
- seq = len(token_ids)
- ids = torch.tensor(token_ids)
- hd = self.head_dim
-
- x = w["embed_tokens.weight"][ids].float()
-
- # RoPE tables
- inv_freq = 1.0 / (self.rope_theta ** (torch.arange(0, hd, 2).float() / hd))
- pos = torch.arange(seq).float()
- freqs = torch.outer(pos, inv_freq)
- cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1)
- sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1)
-
- scale = 1.0 / math.sqrt(hd)
- heads_per_kv = self.n_heads // self.n_kv_heads
-
- for i in range(self.n_layers):
- p = f"layers.{i}"
-
- xn = self._rms_norm(x, w[f"{p}.input_layernorm.weight"])
-
- q = xn @ w[f"{p}.self_attn.q_proj.weight"].T
- k = xn @ w[f"{p}.self_attn.k_proj.weight"].T
- v = xn @ w[f"{p}.self_attn.v_proj.weight"].T
-
- q = self._rms_norm(q.view(seq, self.n_heads, hd), w[f"{p}.self_attn.q_norm.weight"])
- k = self._rms_norm(k.view(seq, self.n_kv_heads, hd), w[f"{p}.self_attn.k_norm.weight"])
-
- q = self._rope_neox(q, cos.unsqueeze(1), sin.unsqueeze(1))
- k = self._rope_neox(k, cos.unsqueeze(1), sin.unsqueeze(1))
-
- q = q.reshape(seq, -1)
- k = k.reshape(seq, -1)
-
- attn_out = torch.zeros_like(q)
- for h in range(self.n_heads):
- kv_h = h // heads_per_kv
- qh = q[:, h*hd:(h+1)*hd]
- kh = k[:, kv_h*hd:(kv_h+1)*hd]
- vh = v[:, kv_h*hd:(kv_h+1)*hd]
- scores = torch.softmax((qh @ kh.T) * scale, dim=-1)
- attn_out[:, h*hd:(h+1)*hd] = scores @ vh
-
- x = x + attn_out @ w[f"{p}.self_attn.o_proj.weight"].T
-
- xn = self._rms_norm(x, w[f"{p}.post_attention_layernorm.weight"])
- gate = xn @ w[f"{p}.mlp.gate_proj.weight"].T
- up = xn @ w[f"{p}.mlp.up_proj.weight"].T
- x = x + (torch.sigmoid(gate) * gate * up) @ w[f"{p}.mlp.down_proj.weight"].T
-
- x = self._rms_norm(x, w["norm.weight"])
- emb = x.mean(0)
- emb = emb / emb.norm()
- return emb.numpy()
-
-
-# ---------------------------------------------------------------------------
-# C backend runner (via --daemon mode)
-# ---------------------------------------------------------------------------
-
-class CEmbed:
- """Communicate with the C binary via --daemon JSON-lines protocol."""
-
- def __init__(self, binary, model_dir, backend="cpu"):
- self.binary = binary
- self.model_dir = model_dir
- self.backend = backend
- self.proc = None
- self.load_ms = 0
-
- def start(self):
- cmd = [self.binary, "-d", self.model_dir, "--daemon"]
- if self.backend == "mlx":
- cmd.append("--mlx")
- t0 = time.perf_counter()
- self.proc = subprocess.Popen(
- cmd,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- bufsize=1,
- )
- # Warm up: send one text to trigger model load + kernel compile
- self.proc.stdin.write("warmup\n")
- self.proc.stdin.flush()
- line = self.proc.stdout.readline()
- self.load_ms = (time.perf_counter() - t0) * 1000
- # Parse warmup result to confirm it works
- try:
- json.loads(line)
- except Exception:
- stderr = self.proc.stderr.read()
- raise RuntimeError(f"C daemon failed to start:\n{stderr}")
-
- def embed(self, text):
- self.proc.stdin.write(text + "\n")
- self.proc.stdin.flush()
- line = self.proc.stdout.readline()
- result = json.loads(line)
- if "error" in result:
- raise RuntimeError(result["error"])
- return np.array(result["embedding"], dtype=np.float32), result["ms"]
-
- def stop(self):
- if self.proc:
- self.proc.stdin.close()
- self.proc.wait(timeout=5)
- self.proc = None
-
- def get_memory_mb(self):
- if not self.proc:
- return 0
- try:
- p = psutil.Process(self.proc.pid)
- return p.memory_info().rss / (1024 * 1024)
- except Exception:
- return 0
-
-
-# ---------------------------------------------------------------------------
-# Tokenizer helper (use the same BPE as C)
-# ---------------------------------------------------------------------------
-
-def load_tokenizer(model_dir):
- from transformers import AutoTokenizer
- return AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
-
-
-# ---------------------------------------------------------------------------
-# Benchmark runner
-# ---------------------------------------------------------------------------
-
-def cosine_sim(a, b):
- return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
-
-
-def measure_pytorch(model, tokenizer, texts, n_runs):
- """Measure PyTorch inference times and return embeddings."""
- results = {}
- for name, text in texts.items():
- ids = tokenizer.encode(text, add_special_tokens=False)
- n_tok = len(ids)
-
- # Warmup
- model.embed(ids)
-
- times = []
- emb = None
- for _ in range(n_runs):
- t0 = time.perf_counter()
- emb = model.embed(ids)
- times.append((time.perf_counter() - t0) * 1000)
-
- results[name] = {
- "tokens": n_tok,
- "times_ms": times,
- "median_ms": float(np.median(times)),
- "embedding": emb,
- }
- return results
-
-
-def measure_c(engine, texts, tokenizer, n_runs):
- """Measure C backend inference times and return embeddings."""
- results = {}
- for name, text in texts.items():
- ids = tokenizer.encode(text, add_special_tokens=False)
- n_tok = len(ids)
-
- # Warmup (second call, first was during start)
- engine.embed(text)
-
- times = []
- emb = None
- for _ in range(n_runs):
- emb, ms = engine.embed(text)
- times.append(ms)
-
- results[name] = {
- "tokens": n_tok,
- "times_ms": times,
- "median_ms": float(np.median(times)),
- "embedding": emb,
- }
- return results
-
-
-def print_table(headers, rows, col_widths=None):
- if col_widths is None:
- col_widths = [max(len(str(h)), max(len(str(r[i])) for r in rows))
- for i, h in enumerate(headers)]
- fmt = " ".join(f"{{:<{w}}}" for w in col_widths)
- print(fmt.format(*headers))
- print(fmt.format(*["-" * w for w in col_widths]))
- for row in rows:
- print(fmt.format(*row))
-
-
-def main():
- parser = argparse.ArgumentParser(description="Benchmark pplx-embed")
- parser.add_argument("--model-dir", required=True, help="Model directory")
- parser.add_argument("--binary", default="./pplx_embed", help="C binary path")
- parser.add_argument("--runs", type=int, default=5, help="Runs per measurement")
- args = parser.parse_args()
-
- model_dir = args.model_dir
- config = json.load(open(f"{model_dir}/config.json"))
- dim = config["hidden_size"]
- n_layers = config["num_hidden_layers"]
- model_name = os.path.basename(os.path.dirname(model_dir)) or os.path.basename(model_dir)
-
- print(f"Benchmark: {model_name}")
- print(f" hidden={dim}, layers={n_layers}, runs={args.runs}")
- print()
-
- tokenizer = load_tokenizer(model_dir)
-
- # Token counts
- print("Sentences:")
- for name, text in SENTENCES.items():
- ids = tokenizer.encode(text, add_special_tokens=False)
- preview = text[:60] + "..." if len(text) > 60 else text
- print(f" {name:8s} {len(ids):3d} tokens \"{preview}\"")
- print()
-
- # --- Model size ---
- import glob
- shard_files = sorted(glob.glob(f"{model_dir}/model*.safetensors"))
- model_bytes = sum(os.path.getsize(f) for f in shard_files)
- print(f"Model size on disk: {model_bytes / 1e9:.2f} GB ({len(shard_files)} shard(s))")
- print()
-
- # --- PyTorch ---
- print("=" * 60)
- print("PyTorch (manual forward, float32, CPU)")
- print("=" * 60)
-
- py_model = PytorchEmbed(model_dir)
- proc = psutil.Process()
-
- t0 = time.perf_counter()
- py_model.load()
- py_load_ms = (time.perf_counter() - t0) * 1000
-
- # Force actual loading by running one inference (mmap is lazy)
- warmup_ids = tokenizer.encode("warmup", add_special_tokens=False)
- mem_before = proc.memory_info().rss
- _ = py_model.embed(warmup_ids)
- mem_after = proc.memory_info().rss
- py_first_ms = (time.perf_counter() - t0) * 1000 # total including first inference
- py_mem_mb = (mem_after - mem_before) / (1024 * 1024)
- # Also measure total RSS for a fair comparison
- py_total_rss = proc.memory_info().rss / (1024 * 1024)
-
- print(f" Load time: {py_load_ms:.0f} ms (mmap, lazy)")
- print(f" First inference (load+embed): {py_first_ms:.0f} ms")
- print(f" RSS after first inference: ~{py_total_rss:.0f} MB")
- print()
-
- py_results = measure_pytorch(py_model, tokenizer, SENTENCES, args.runs)
-
- rows = []
- for name in SENTENCES:
- r = py_results[name]
- rows.append([name, str(r["tokens"]), f"{r['median_ms']:.1f}"])
- print_table(["sentence", "tokens", "median ms"], rows, [10, 8, 12])
- print()
-
- # Free pytorch weights
- py_model.weights = None
- torch.cuda.empty_cache() if torch.cuda.is_available() else None
- import gc; gc.collect()
-
- # --- C/BLAS ---
- print("=" * 60)
- print("C/BLAS (Accelerate, multithreaded)")
- print("=" * 60)
-
- c_cpu = CEmbed(args.binary, model_dir, "cpu")
- c_cpu.start()
- cpu_mem = c_cpu.get_memory_mb()
- print(f" Load time: {c_cpu.load_ms:.0f} ms (includes warmup)")
- print(f" RSS: ~{cpu_mem:.0f} MB")
- print()
-
- cpu_results = measure_c(c_cpu, SENTENCES, tokenizer, args.runs)
- c_cpu.stop()
-
- rows = []
- for name in SENTENCES:
- r = cpu_results[name]
- rows.append([name, str(r["tokens"]), f"{r['median_ms']:.1f}"])
- print_table(["sentence", "tokens", "median ms"], rows, [10, 8, 12])
- print()
-
- # --- C/MLX ---
- print("=" * 60)
- print("C/MLX (Metal GPU)")
- print("=" * 60)
-
- c_mlx = CEmbed(args.binary, model_dir, "mlx")
- c_mlx.start()
- mlx_mem = c_mlx.get_memory_mb()
- print(f" Load time: {c_mlx.load_ms:.0f} ms (includes warmup + kernel compile)")
- print(f" RSS: ~{mlx_mem:.0f} MB")
- print()
-
- mlx_results = measure_c(c_mlx, SENTENCES, tokenizer, args.runs)
- c_mlx.stop()
-
- rows = []
- for name in SENTENCES:
- r = mlx_results[name]
- rows.append([name, str(r["tokens"]), f"{r['median_ms']:.1f}"])
- print_table(["sentence", "tokens", "median ms"], rows, [10, 8, 12])
- print()
-
- # --- Accuracy ---
- print("=" * 60)
- print("Accuracy (cosine similarity vs PyTorch reference)")
- print("=" * 60)
- print()
-
- rows = []
- for name in SENTENCES:
- py_emb = py_results[name]["embedding"]
- cpu_emb = cpu_results[name]["embedding"]
- mlx_emb = mlx_results[name]["embedding"]
-
- sim_cpu = cosine_sim(py_emb, cpu_emb)
- sim_mlx = cosine_sim(py_emb, mlx_emb)
- max_diff_cpu = float(np.max(np.abs(py_emb - cpu_emb)))
- max_diff_mlx = float(np.max(np.abs(py_emb - mlx_emb)))
-
- rows.append([
- name,
- str(py_results[name]["tokens"]),
- f"{sim_cpu:.8f}",
- f"{max_diff_cpu:.2e}",
- f"{sim_mlx:.8f}",
- f"{max_diff_mlx:.2e}",
- ])
-
- print_table(
- ["sentence", "tokens", "cos(cpu,py)", "max|diff|", "cos(mlx,py)", "max|diff|"],
- rows,
- [10, 8, 14, 12, 14, 12],
- )
- print()
-
- # --- Summary comparison ---
- print("=" * 60)
- print("Summary: Median inference time (ms)")
- print("=" * 60)
- print()
-
- rows = []
- for name in SENTENCES:
- py_ms = py_results[name]["median_ms"]
- cpu_ms = cpu_results[name]["median_ms"]
- mlx_ms = mlx_results[name]["median_ms"]
- toks = py_results[name]["tokens"]
-
- rows.append([
- name,
- str(toks),
- f"{py_ms:.1f}",
- f"{cpu_ms:.1f}",
- f"{py_ms/cpu_ms:.1f}x" if cpu_ms > 0 else "-",
- f"{mlx_ms:.1f}",
- f"{py_ms/mlx_ms:.1f}x" if mlx_ms > 0 else "-",
- ])
-
- print_table(
- ["sentence", "tok", "PyTorch", "C/BLAS", "speedup", "C/MLX", "speedup"],
- rows,
- [10, 5, 10, 10, 9, 10, 9],
- )
-
- print()
- print(f"Load time: PyTorch {py_first_ms:.0f} ms (mmap+first embed) | C/BLAS {c_cpu.load_ms:.0f} ms | C/MLX {c_mlx.load_ms:.0f} ms")
- print(f"RSS: PyTorch ~{py_total_rss:.0f} MB | C/BLAS ~{cpu_mem:.0f} MB | C/MLX ~{mlx_mem:.0f} MB")
- print()
-
-
-if __name__ == "__main__":
- main()