From 6652fcbc653d84e75cb5c08c5f5edf52263ed50f Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 20 Jun 2026 20:36:55 +0200 Subject: tests: upgrade unit tests and add integration/parity tests --- tests/check_batch_parity.py | 172 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100755 tests/check_batch_parity.py (limited to 'tests/check_batch_parity.py') diff --git a/tests/check_batch_parity.py b/tests/check_batch_parity.py new file mode 100755 index 0000000..61f4427 --- /dev/null +++ b/tests/check_batch_parity.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Compare stdin batch_size=1 against true batched stdin execution.""" + +import argparse +import json +import math +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +def ensure_cli(binary: Path) -> None: + """Build the CLI if the default binary is missing.""" + if binary == DEFAULT_BINARY and not binary.exists(): + subprocess.run(["make", "-C", "tools/cli", "all"], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL) + + +DEFAULT_TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: Istanbul is a major city in Turkey.", + "A short string.", + ( + "Embedding models convert text into dense vectors for retrieval, " + "clustering, and semantic similarity search." + ), +] + +LONG_TEXTS = [ + "document: " + ( + "Embedding models turn text into dense vectors for semantic retrieval. " + "Long document inference stresses bidirectional attention because every " + "token attends to every other token in the same packed sequence. " + ) * repeats + for repeats in (8, 10, 12, 14) +] + + +def run_stdin(binary, model_dir, backend, batch_size, texts, + mlx_quantize_bits, mlx_quantize_group_size, + cuda_weight_dtype=None): + cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)] + if backend == "mlx": + if mlx_quantize_bits: + cmd.extend([ + "--gpu-quant-bits", str(mlx_quantize_bits), + "--gpu-quant-group-size", str(mlx_quantize_group_size), + ]) + elif backend == "cuda": + # Force exact-F32 weight storage so a BF16 snapshot is judged for kernel + # exactness (batched vs singleton GEMMs) rather than BF16 rounding. + if cuda_weight_dtype: + cmd.extend(["--gpu-weight-dtype", cuda_weight_dtype]) + + proc = subprocess.run( + cmd, + input="\n".join(texts) + "\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}" + ) + + rows = [] + for line in proc.stdout.splitlines(): + if line.strip(): + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin returned error: {row}") + rows.append(row) + + if len(rows) != len(texts): + raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}") + return rows + + +def cosine(a, b): + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model-dir", required=True) + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--backend", choices=("cpu", "mlx", "cuda"), default="cpu") + ap.add_argument("--batch-size", type=int, default=4) + ap.add_argument("--max-diff", type=float, default=5e-5) + ap.add_argument("--min-cosine", type=float, default=0.99999) + ap.add_argument("--long", action="store_true", + help="use long inputs that exercise tiled CPU attention") + ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits", + dest="mlx_quantize_bits", type=int, default=0, + choices=(0, 8)) + ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size", + dest="mlx_quantize_group_size", type=int, default=64) + ap.add_argument("--cuda-weight-dtype", choices=("f32", "bf16"), default=None, + help="force CUDA weight storage dtype (default: snapshot dtype)") + ap.add_argument("texts", nargs="*") + args = ap.parse_args() + + texts = args.texts or (LONG_TEXTS if args.long else DEFAULT_TEXTS) + if args.batch_size < 2: + raise SystemExit("--batch-size must be >= 2 for a useful parity check") + if args.mlx_quantize_bits and args.backend != "mlx": + raise SystemExit("--mlx-quant-bits requires --backend mlx") + if args.mlx_quantize_group_size <= 0: + raise SystemExit("--mlx-quant-group-size must be > 0") + + ensure_cli(Path(args.binary)) + + seq_rows = run_stdin(args.binary, args.model_dir, args.backend, 1, texts, + args.mlx_quantize_bits, + args.mlx_quantize_group_size, + args.cuda_weight_dtype) + bat_rows = run_stdin(args.binary, args.model_dir, args.backend, + args.batch_size, texts, args.mlx_quantize_bits, + args.mlx_quantize_group_size, + args.cuda_weight_dtype) + + worst_diff = 0.0 + worst_cos = 1.0 + for i, (seq, bat) in enumerate(zip(seq_rows, bat_rows)): + if seq["dim"] != bat["dim"] or seq["tokens"] != bat["tokens"]: + raise RuntimeError(f"metadata mismatch at row {i}: {seq} vs {bat}") + a = seq["embedding"] + b = bat["embedding"] + max_diff = max(abs(x - y) for x, y in zip(a, b)) + cos = cosine(a, b) + worst_diff = max(worst_diff, max_diff) + worst_cos = min(worst_cos, cos) + print( + f"[{i}] tokens={seq['tokens']} cosine={cos:.8f} " + f"max_abs_diff={max_diff:.8g}" + ) + + if worst_diff > args.max_diff or worst_cos < args.min_cosine: + raise RuntimeError( + f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}" + ) + + seq_ms = sum(float(row["ms"]) for row in seq_rows) + batch_ms = sum(float(bat_rows[i]["ms"]) for i in range(0, len(bat_rows), args.batch_size)) + print( + f"ok: {len(texts)} texts, backend={args.backend}, " + f"batch_size={args.batch_size}, worst_diff={worst_diff:.8g}, " + f"worst_cos={worst_cos:.8f}, seq_ms_sum={seq_ms:.1f}, " + f"batch_chunk_ms_sum={batch_ms:.1f}" + + ( + f", mlx_q{args.mlx_quantize_bits}" + if args.mlx_quantize_bits else "" + ) + ) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) -- cgit v1.3.1