diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:36:55 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch) | |
| tree | 1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/check_contextual_batch_parity.py | |
| parent | dbdf714bb57578431924e5d069089bdfd9f4c604 (diff) | |
| download | ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip | |
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/check_contextual_batch_parity.py')
| -rw-r--r-- | tests/check_contextual_batch_parity.py | 85 |
1 files changed, 85 insertions, 0 deletions
diff --git a/tests/check_contextual_batch_parity.py b/tests/check_contextual_batch_parity.py new file mode 100644 index 0000000..6cd7021 --- /dev/null +++ b/tests/check_contextual_batch_parity.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Run the contextual batch-versus-singleton parity check. + +The C side is tests/contextual_batch.c, built by the Makefile target +`parity-context-driver`; this script builds it and runs it with the requested +parameters. It compares the single-document path against the batched path in C +(no external reference), so it needs a model directory but no Python ML stack. + + tests/check_contextual_batch_parity.py --model-dir DIR +""" + +import argparse +import os +import platform +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def build_driver(cc: str, backend: str) -> Path: + """Build the contextual-batch driver via the Makefile; return its path.""" + target = "parity-context-driver-mlx" if backend == "mlx" else "parity-context-driver" + binary = "contextual_batch_mlx" if backend == "mlx" else "contextual_batch" + proc = subprocess.run(["make", "-C", "tests", f"CC={cc}", target], cwd=ROOT, + text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + raise SystemExit(proc.returncode) + return ROOT / "tests" / binary + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--model-dir", required=True) + ap.add_argument("--backend", choices=("cpu", "mlx"), default="cpu") + ap.add_argument("--cc", default=os.environ.get("CC", "cc")) + ap.add_argument("--runs", type=int, default=0, + help="run N timed contextual document batches after parity") + ap.add_argument("--max-diff", type=float, default=0.00005) + ap.add_argument("--min-cosine", type=float, default=0.99999) + ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits", + dest="mlx_quantize_bits", type=int, default=0, + choices=(0, 8)) + ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size", + dest="mlx_quantize_group_size", type=int, default=64) + ap.add_argument("--synthetic-docs", type=int, default=0, + help="generate N synthetic documents instead of the smoke set") + ap.add_argument("--synthetic-chunks", type=int, default=0, + help="chunks per synthetic document") + ap.add_argument("--synthetic-repeats", type=int, default=1, + help="repeated text blocks per synthetic chunk") + ap.add_argument("--synthetic-ragged", action="store_true", + help="scale synthetic chunk count by document index") + args = ap.parse_args() + if args.mlx_quantize_bits and args.backend != "mlx": + raise SystemExit("--mlx-quant-bits requires --backend mlx") + if args.mlx_quantize_group_size <= 0: + raise SystemExit("--mlx-quant-group-size must be > 0") + if args.backend == "mlx" and platform.system() != "Darwin": + raise SystemExit("MLX checks require macOS") + + driver = build_driver(args.cc, args.backend) + return subprocess.run( + [ + str(driver), + args.model_dir, + str(args.runs), + str(args.max_diff), + str(args.min_cosine), + str(args.mlx_quantize_bits), + str(args.mlx_quantize_group_size), + str(args.synthetic_docs), + str(args.synthetic_chunks), + str(args.synthetic_repeats), + "1" if args.synthetic_ragged else "0", + ], + cwd=ROOT, + ).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) |
