From 445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sun, 21 Jun 2026 08:26:31 +0200 Subject: build: rehome check scripts and modernize all.bash --- tests/check_cuda_fast_gemm_parity.py | 188 +++++++++++++++++++++++++++++++++++ tests/check_mlx_model_limits.py | 60 +++++++++++ tests/check_mlx_quantized_parity.py | 104 +++++++++++++++++++ 3 files changed, 352 insertions(+) create mode 100755 tests/check_cuda_fast_gemm_parity.py create mode 100755 tests/check_mlx_model_limits.py create mode 100755 tests/check_mlx_quantized_parity.py (limited to 'tests') diff --git a/tests/check_cuda_fast_gemm_parity.py b/tests/check_cuda_fast_gemm_parity.py new file mode 100755 index 0000000..06ee1f1 --- /dev/null +++ b/tests/check_cuda_fast_gemm_parity.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Check CUDA reduced-precision quality against exact CUDA F32. + +Covers both reduced-precision axes: compute mode (--mode tf32|bf16|16f, +selecting --gpu-gemm-mode) and weight storage (--weights bf16, selecting +--gpu-weight-dtype). The reference run always uses exact F32 weights and compute. +For BF16 weight storage the int8 byte-change rate is inherently large +(~7-15%) and informational only; cosine is the quality gate, so the int8 +threshold defaults to 1.0 in that case unless overridden.""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +DEFAULT_TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + ( + "document: Embedding models turn text into dense vectors for semantic " + "retrieval. Long document inference stresses bidirectional attention " + "because every token attends to every other token in the same packed " + "sequence. " + ) * 3, + ( + "document: Embedding models turn text into dense vectors for semantic " + "retrieval. Long document inference stresses bidirectional attention " + "because every token attends to every other token in the same packed " + "sequence. " + ) * 6, +] +# The longest passage stays under ~512 tokens so short-context encoders +# (BERT/MiniLM/BGE max_position 512, XLM-R 514) accept it; larger 8x/14x inputs +# overflowed their position tables and were (correctly) rejected, which looked +# like a CUDA failure. Still long enough to stress packed-sequence attention. + + +def run_cuda(binary: str, model_dir: str, mode: str, weights: str, + batch_size: int, texts: list[str]) -> list[list[float]]: + cmd = [ + binary, + "-d", model_dir, + "--gpu-gemm-mode", mode, + "--gpu-weight-dtype", weights, + "--stream", + "-b", str(batch_size), + ] + proc = subprocess.run( + cmd, + input="\n".join(texts) + "\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}" + ) + + rows: list[list[float]] = [] + for line in proc.stdout.splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin returned error: {row}") + rows.append([float(x) for x in row["embedding"]]) + if len(rows) != len(texts): + raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}") + return rows + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def quantize_int8_tanh(x: float) -> int: + v = round(math.tanh(x) * 127.0) + if v < -128: + return -128 + if v > 127: + return 127 + return int(v) + + +def compare(name: str, ref: list[list[float]], got: list[list[float]]) -> tuple[float, float, int]: + worst_diff = 0.0 + worst_cos = 1.0 + int8_changes = 0 + for i, (a, b) in enumerate(zip(ref, got)): + if len(a) != len(b): + raise RuntimeError(f"dimension mismatch at row {i}: {len(a)} vs {len(b)}") + max_diff = max(abs(x - y) for x, y in zip(a, b)) + cos = cosine(a, b) + changes = sum( + quantize_int8_tanh(x) != quantize_int8_tanh(y) + for x, y in zip(a, b) + ) + worst_diff = max(worst_diff, max_diff) + worst_cos = min(worst_cos, cos) + int8_changes += changes + print( + f"{name} [{i}] cosine={cos:.8f} " + f"max_abs_diff={max_diff:.8g} int8_changes={changes}" + ) + return worst_diff, worst_cos, int8_changes + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--mode", choices=("f32", "tf32", "bf16", "16f"), + default="tf32") + ap.add_argument("--weights", choices=("f32", "bf16"), default="f32") + ap.add_argument("--batch-size", type=int, default=8) + ap.add_argument("--max-diff", type=float, default=None) + ap.add_argument("--min-cosine", type=float, default=0.999) + ap.add_argument("--max-int8-change-rate", type=float, default=None) + ap.add_argument("texts", nargs="*") + args = ap.parse_args() + + if args.batch_size < 2: + raise SystemExit("--batch-size must be >= 2") + if args.mode == "f32" and args.weights == "f32": + raise SystemExit("nothing to check: --mode and --weights are both f32") + if args.max_diff is None: + args.max_diff = 0.05 if args.weights == "bf16" else 0.01 + if args.max_int8_change_rate is None: + args.max_int8_change_rate = 1.0 if args.weights == "bf16" else 0.02 + + texts = args.texts or DEFAULT_TEXTS + ref_batch = run_cuda(args.binary, args.model_dir, "f32", "f32", + args.batch_size, texts) + fast_batch = run_cuda(args.binary, args.model_dir, args.mode, + args.weights, args.batch_size, texts) + fast_single = run_cuda(args.binary, args.model_dir, args.mode, + args.weights, 1, texts) + + qdiff, qcos, qchanges = compare("quality", ref_batch, fast_batch) + sdiff, scos, schanges = compare("shape", fast_single, fast_batch) + + total_values = len(texts) * len(ref_batch[0]) + qrate = qchanges / total_values + srate = schanges / total_values + print( + f"ok: mode={args.mode} weights={args.weights} " + f"batch_size={args.batch_size} " + f"quality_diff={qdiff:.8g} quality_cos={qcos:.8f} " + f"quality_int8_change_rate={qrate:.6f} " + f"shape_diff={sdiff:.8g} shape_cos={scos:.8f} " + f"shape_int8_change_rate={srate:.6f}" + ) + + failed = ( + qdiff > args.max_diff or qcos < args.min_cosine or + sdiff > args.max_diff or scos < args.min_cosine or + qrate > args.max_int8_change_rate or + srate > args.max_int8_change_rate + ) + if failed: + raise SystemExit( + f"cuda parity failed (mode={args.mode} weights={args.weights}): " + f"quality diff={qdiff:g} cos={qcos:g} int8_rate={qrate:g}; " + f"shape diff={sdiff:g} cos={scos:g} int8_rate={srate:g}" + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/check_mlx_model_limits.py b/tests/check_mlx_model_limits.py new file mode 100755 index 0000000..08eb7ba --- /dev/null +++ b/tests/check_mlx_model_limits.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Verify that MLX rejects mismatched model metadata before touching Metal.""" + +import argparse +import json +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--model-dir", required=True) + args = ap.parse_args() + + binary = Path(args.binary).resolve() + model_dir = Path(args.model_dir).resolve() + with (model_dir / "config.json").open() as f: + config = json.load(f) + + vocab_size = config.get("vocab_size") + if not isinstance(vocab_size, int) or vocab_size <= 1: + raise SystemExit("config.json has no usable vocab_size") + config["vocab_size"] = vocab_size - 1 + + with tempfile.TemporaryDirectory(prefix="ffwd-mlx-invalid-") as td: + tmp = Path(td) + (tmp / "config.json").write_text(json.dumps(config)) + (tmp / "vocab.json").symlink_to(model_dir / "vocab.json") + tensors = list(model_dir.glob("*.safetensors")) + if not tensors: + raise SystemExit("model directory has no safetensor files") + for tensor in tensors: + (tmp / tensor.name).symlink_to(tensor) + + proc = subprocess.run( + [str(binary), "-d", str(tmp), "--stream"], + input="", + text=True, + capture_output=True, + check=False, + ) + + expected = "mlx: bad shape for embed_tokens.weight at dim 0" + if proc.returncode == 0: + raise SystemExit("MLX unexpectedly accepted mismatched vocab_size") + if expected not in proc.stderr: + raise SystemExit(f"unexpected stderr:\n{proc.stderr}") + if "MLX error:" in proc.stderr: + raise SystemExit(f"MLX touched Metal before rejecting the model:\n{proc.stderr}") + print("ok: MLX rejected mismatched tensor shape before Metal initialization") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/check_mlx_quantized_parity.py b/tests/check_mlx_quantized_parity.py new file mode 100755 index 0000000..2cf0add --- /dev/null +++ b/tests/check_mlx_quantized_parity.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Compare normal MLX embeddings against opt-in quantized MLX weights.""" + +import argparse +import json +import math +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: Istanbul is a major city in Turkey.", + "The quick brown fox jumps over the lazy dog near the riverbank.", + ( + "Retrieval augmented generation combines embedding models with " + "external knowledge sources and ranks documents by semantic similarity." + ), +] + + +def run(binary: str, model_dir: str, texts: list[str], batch_size: int, + bits: int, group_size: int) -> list[list[float]]: + cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)] + if bits: + cmd.extend([ + "--gpu-quant-bits", str(bits), + "--gpu-quant-group-size", str(group_size), + ]) + proc = subprocess.run( + cmd, + input="\n".join(texts) + "\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}" + ) + rows = [] + for line in proc.stdout.splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin returned error: {row}") + rows.append(row["embedding"]) + if len(rows) != len(texts): + raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}") + return rows + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--batch-size", type=int, default=4) + ap.add_argument("--bits", type=int, default=8, choices=(8,)) + ap.add_argument("--group-size", type=int, default=64) + ap.add_argument("--max-diff", type=float, default=0.08) + ap.add_argument("--min-cosine", type=float, default=0.995) + args = ap.parse_args() + + ref = run(args.binary, args.model_dir, TEXTS, args.batch_size, 0, + args.group_size) + got = run(args.binary, args.model_dir, TEXTS, args.batch_size, args.bits, + args.group_size) + + worst_diff = 0.0 + worst_cos = 1.0 + for i, (a, b) in enumerate(zip(ref, got)): + max_diff = max(abs(x - y) for x, y in zip(a, b)) + cos = cosine(a, b) + worst_diff = max(worst_diff, max_diff) + worst_cos = min(worst_cos, cos) + print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}") + + print( + f"ok: bits={args.bits} group_size={args.group_size} " + f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}" + ) + if worst_diff > args.max_diff or worst_cos < args.min_cosine: + raise SystemExit( + f"quantized parity failed: diff={worst_diff:g}, cos={worst_cos:g}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- cgit v1.3.1