#!/usr/bin/env python3 """Compare stdin batch_size=1 against true batched stdin execution.""" import argparse import json import math import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] DEFAULT_BINARY = ROOT / "ffwd-cli" def ensure_cli(binary: Path) -> None: """Build the CLI if the default binary is missing.""" if binary == DEFAULT_BINARY and not binary.exists(): subprocess.run(["make", "cpu"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) DEFAULT_TEXTS = [ "query: what is the capital of France?", "document: Paris is the capital of France.", "document: Berlin is the capital of Germany.", "document: Istanbul is a major city in Turkey.", "A short string.", ( "Embedding models convert text into dense vectors for retrieval, " "clustering, and semantic similarity search." ), ] LONG_TEXTS = [ "document: " + ( "Embedding models turn text into dense vectors for semantic retrieval. " "Long document inference stresses bidirectional attention because every " "token attends to every other token in the same packed sequence. " ) * repeats for repeats in (8, 10, 12, 14) ] def run_stdin(binary, model_dir, backend, batch_size, texts, mlx_quantize_bits, mlx_quantize_group_size, cuda_weight_dtype=None): cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)] if backend == "mlx": if mlx_quantize_bits: cmd.extend([ "--gpu-quant-bits", str(mlx_quantize_bits), "--gpu-quant-group-size", str(mlx_quantize_group_size), ]) elif backend == "cuda": # Force exact-F32 weight storage so a BF16 snapshot is judged for kernel # exactness (batched vs singleton GEMMs) rather than BF16 rounding. if cuda_weight_dtype: cmd.extend(["--gpu-weight-dtype", cuda_weight_dtype]) proc = subprocess.run( cmd, input="\n".join(texts) + "\n", text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) if proc.returncode != 0: raise RuntimeError( f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}" ) rows = [] for line in proc.stdout.splitlines(): if line.strip(): row = json.loads(line) if "error" in row: raise RuntimeError(f"stdin returned error: {row}") rows.append(row) if len(rows) != len(texts): raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}") return rows def cosine(a, b): dot = sum(x * y for x, y in zip(a, b)) na = math.sqrt(sum(x * x for x in a)) nb = math.sqrt(sum(y * y for y in b)) return dot / (na * nb) def main(): ap = argparse.ArgumentParser() ap.add_argument("--model-dir", required=True) ap.add_argument("--binary", default=str(DEFAULT_BINARY)) ap.add_argument("--backend", choices=("cpu", "mlx", "cuda"), default="cpu") ap.add_argument("--batch-size", type=int, default=4) ap.add_argument("--max-diff", type=float, default=5e-5) ap.add_argument("--min-cosine", type=float, default=0.99999) ap.add_argument("--long", action="store_true", help="use long inputs that exercise tiled CPU attention") ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits", dest="mlx_quantize_bits", type=int, default=0, choices=(0, 8)) ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size", dest="mlx_quantize_group_size", type=int, default=64) ap.add_argument("--cuda-weight-dtype", choices=("f32", "bf16"), default=None, help="force CUDA weight storage dtype (default: snapshot dtype)") ap.add_argument("texts", nargs="*") args = ap.parse_args() texts = args.texts or (LONG_TEXTS if args.long else DEFAULT_TEXTS) if args.batch_size < 2: raise SystemExit("--batch-size must be >= 2 for a useful parity check") if args.mlx_quantize_bits and args.backend != "mlx": raise SystemExit("--mlx-quant-bits requires --backend mlx") if args.mlx_quantize_group_size <= 0: raise SystemExit("--mlx-quant-group-size must be > 0") ensure_cli(Path(args.binary)) seq_rows = run_stdin(args.binary, args.model_dir, args.backend, 1, texts, args.mlx_quantize_bits, args.mlx_quantize_group_size, args.cuda_weight_dtype) bat_rows = run_stdin(args.binary, args.model_dir, args.backend, args.batch_size, texts, args.mlx_quantize_bits, args.mlx_quantize_group_size, args.cuda_weight_dtype) worst_diff = 0.0 worst_cos = 1.0 for i, (seq, bat) in enumerate(zip(seq_rows, bat_rows)): if seq["dim"] != bat["dim"] or seq["tokens"] != bat["tokens"]: raise RuntimeError(f"metadata mismatch at row {i}: {seq} vs {bat}") a = seq["embedding"] b = bat["embedding"] max_diff = max(abs(x - y) for x, y in zip(a, b)) cos = cosine(a, b) worst_diff = max(worst_diff, max_diff) worst_cos = min(worst_cos, cos) print( f"[{i}] tokens={seq['tokens']} cosine={cos:.8f} " f"max_abs_diff={max_diff:.8g}" ) if worst_diff > args.max_diff or worst_cos < args.min_cosine: raise RuntimeError( f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}" ) seq_ms = sum(float(row["ms"]) for row in seq_rows) batch_ms = sum(float(bat_rows[i]["ms"]) for i in range(0, len(bat_rows), args.batch_size)) print( f"ok: {len(texts)} texts, backend={args.backend}, " f"batch_size={args.batch_size}, worst_diff={worst_diff:.8g}, " f"worst_cos={worst_cos:.8f}, seq_ms_sum={seq_ms:.1f}, " f"batch_chunk_ms_sum={batch_ms:.1f}" + ( f", mlx_q{args.mlx_quantize_bits}" if args.mlx_quantize_bits else "" ) ) if __name__ == "__main__": try: main() except Exception as exc: print(f"error: {exc}", file=sys.stderr) sys.exit(1)