aboutsummaryrefslogtreecommitdiff
path: root/tests/check_batch_parity.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:36:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commit6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch)
tree1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/check_batch_parity.py
parentdbdf714bb57578431924e5d069089bdfd9f4c604 (diff)
downloadffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz
ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/check_batch_parity.py')
-rwxr-xr-xtests/check_batch_parity.py172
1 files changed, 172 insertions, 0 deletions
diff --git a/tests/check_batch_parity.py b/tests/check_batch_parity.py
new file mode 100755
index 0000000..61f4427
--- /dev/null
+++ b/tests/check_batch_parity.py
@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Compare stdin batch_size=1 against true batched stdin execution."""
+
+import argparse
+import json
+import math
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
+
+
+def ensure_cli(binary: Path) -> None:
+ """Build the CLI if the default binary is missing."""
+ if binary == DEFAULT_BINARY and not binary.exists():
+ subprocess.run(["make", "-C", "tools/cli", "all"], cwd=ROOT, check=True,
+ stdout=subprocess.DEVNULL)
+
+
+DEFAULT_TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "document: Istanbul is a major city in Turkey.",
+ "A short string.",
+ (
+ "Embedding models convert text into dense vectors for retrieval, "
+ "clustering, and semantic similarity search."
+ ),
+]
+
+LONG_TEXTS = [
+ "document: " + (
+ "Embedding models turn text into dense vectors for semantic retrieval. "
+ "Long document inference stresses bidirectional attention because every "
+ "token attends to every other token in the same packed sequence. "
+ ) * repeats
+ for repeats in (8, 10, 12, 14)
+]
+
+
+def run_stdin(binary, model_dir, backend, batch_size, texts,
+ mlx_quantize_bits, mlx_quantize_group_size,
+ cuda_weight_dtype=None):
+ cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
+ if backend == "mlx":
+ if mlx_quantize_bits:
+ cmd.extend([
+ "--gpu-quant-bits", str(mlx_quantize_bits),
+ "--gpu-quant-group-size", str(mlx_quantize_group_size),
+ ])
+ elif backend == "cuda":
+ # Force exact-F32 weight storage so a BF16 snapshot is judged for kernel
+ # exactness (batched vs singleton GEMMs) rather than BF16 rounding.
+ if cuda_weight_dtype:
+ cmd.extend(["--gpu-weight-dtype", cuda_weight_dtype])
+
+ proc = subprocess.run(
+ cmd,
+ input="\n".join(texts) + "\n",
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=False,
+ )
+ if proc.returncode != 0:
+ raise RuntimeError(
+ f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}"
+ )
+
+ rows = []
+ for line in proc.stdout.splitlines():
+ if line.strip():
+ row = json.loads(line)
+ if "error" in row:
+ raise RuntimeError(f"stdin returned error: {row}")
+ rows.append(row)
+
+ if len(rows) != len(texts):
+ raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}")
+ return rows
+
+
+def cosine(a, b):
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--binary", default=str(DEFAULT_BINARY))
+ ap.add_argument("--backend", choices=("cpu", "mlx", "cuda"), default="cpu")
+ ap.add_argument("--batch-size", type=int, default=4)
+ ap.add_argument("--max-diff", type=float, default=5e-5)
+ ap.add_argument("--min-cosine", type=float, default=0.99999)
+ ap.add_argument("--long", action="store_true",
+ help="use long inputs that exercise tiled CPU attention")
+ ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits",
+ dest="mlx_quantize_bits", type=int, default=0,
+ choices=(0, 8))
+ ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size",
+ dest="mlx_quantize_group_size", type=int, default=64)
+ ap.add_argument("--cuda-weight-dtype", choices=("f32", "bf16"), default=None,
+ help="force CUDA weight storage dtype (default: snapshot dtype)")
+ ap.add_argument("texts", nargs="*")
+ args = ap.parse_args()
+
+ texts = args.texts or (LONG_TEXTS if args.long else DEFAULT_TEXTS)
+ if args.batch_size < 2:
+ raise SystemExit("--batch-size must be >= 2 for a useful parity check")
+ if args.mlx_quantize_bits and args.backend != "mlx":
+ raise SystemExit("--mlx-quant-bits requires --backend mlx")
+ if args.mlx_quantize_group_size <= 0:
+ raise SystemExit("--mlx-quant-group-size must be > 0")
+
+ ensure_cli(Path(args.binary))
+
+ seq_rows = run_stdin(args.binary, args.model_dir, args.backend, 1, texts,
+ args.mlx_quantize_bits,
+ args.mlx_quantize_group_size,
+ args.cuda_weight_dtype)
+ bat_rows = run_stdin(args.binary, args.model_dir, args.backend,
+ args.batch_size, texts, args.mlx_quantize_bits,
+ args.mlx_quantize_group_size,
+ args.cuda_weight_dtype)
+
+ worst_diff = 0.0
+ worst_cos = 1.0
+ for i, (seq, bat) in enumerate(zip(seq_rows, bat_rows)):
+ if seq["dim"] != bat["dim"] or seq["tokens"] != bat["tokens"]:
+ raise RuntimeError(f"metadata mismatch at row {i}: {seq} vs {bat}")
+ a = seq["embedding"]
+ b = bat["embedding"]
+ max_diff = max(abs(x - y) for x, y in zip(a, b))
+ cos = cosine(a, b)
+ worst_diff = max(worst_diff, max_diff)
+ worst_cos = min(worst_cos, cos)
+ print(
+ f"[{i}] tokens={seq['tokens']} cosine={cos:.8f} "
+ f"max_abs_diff={max_diff:.8g}"
+ )
+
+ if worst_diff > args.max_diff or worst_cos < args.min_cosine:
+ raise RuntimeError(
+ f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}"
+ )
+
+ seq_ms = sum(float(row["ms"]) for row in seq_rows)
+ batch_ms = sum(float(bat_rows[i]["ms"]) for i in range(0, len(bat_rows), args.batch_size))
+ print(
+ f"ok: {len(texts)} texts, backend={args.backend}, "
+ f"batch_size={args.batch_size}, worst_diff={worst_diff:.8g}, "
+ f"worst_cos={worst_cos:.8f}, seq_ms_sum={seq_ms:.1f}, "
+ f"batch_chunk_ms_sum={batch_ms:.1f}"
+ + (
+ f", mlx_q{args.mlx_quantize_bits}"
+ if args.mlx_quantize_bits else ""
+ )
+ )
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except Exception as exc:
+ print(f"error: {exc}", file=sys.stderr)
+ sys.exit(1)