1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
#!/usr/bin/env python3
"""Run the contextual batch-vs-singleton parity check.
The C test is tests/contextual_batch.c, built by the Makefile target
`parity-context-driver`.
This script builds the driver and runs it with the requested parameters. It
compares the single-document path against the batched path in C, with no
external reference.
Requires a model directory.
Usage:
tests/check_contextual_batch_parity.py --model-dir DIR
"""
import argparse
import os
import platform
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def build_driver(cc: str, backend: str) -> Path:
"""Build the contextual-batch driver via the Makefile; return its path."""
target = "parity-context-driver-mlx" if backend == "mlx" else "parity-context-driver"
binary = "contextual_batch_mlx" if backend == "mlx" else "contextual_batch"
proc = subprocess.run(["make", "-C", "tests", f"CC={cc}", target], cwd=ROOT,
text=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, check=False)
if proc.returncode != 0:
sys.stderr.write(proc.stdout)
raise SystemExit(proc.returncode)
return ROOT / "tests" / binary
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--model-dir", required=True)
ap.add_argument("--backend", choices=("cpu", "mlx"), default="cpu")
ap.add_argument("--cc", default=os.environ.get("CC", "cc"))
ap.add_argument("--runs", type=int, default=0,
help="run N timed contextual document batches after parity")
ap.add_argument("--max-diff", type=float, default=0.00005)
ap.add_argument("--min-cosine", type=float, default=0.99999)
ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits",
dest="mlx_quantize_bits", type=int, default=0,
choices=(0, 8))
ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size",
dest="mlx_quantize_group_size", type=int, default=64)
ap.add_argument("--synthetic-docs", type=int, default=0,
help="generate N synthetic documents instead of the smoke set")
ap.add_argument("--synthetic-chunks", type=int, default=0,
help="chunks per synthetic document")
ap.add_argument("--synthetic-repeats", type=int, default=1,
help="repeated text blocks per synthetic chunk")
ap.add_argument("--synthetic-ragged", action="store_true",
help="scale synthetic chunk count by document index")
args = ap.parse_args()
if args.mlx_quantize_bits and args.backend != "mlx":
raise SystemExit("--mlx-quant-bits requires --backend mlx")
if args.mlx_quantize_group_size <= 0:
raise SystemExit("--mlx-quant-group-size must be > 0")
if args.backend == "mlx" and platform.system() != "Darwin":
raise SystemExit("MLX checks require macOS")
driver = build_driver(args.cc, args.backend)
return subprocess.run(
[
str(driver),
args.model_dir,
str(args.runs),
str(args.max_diff),
str(args.min_cosine),
str(args.mlx_quantize_bits),
str(args.mlx_quantize_group_size),
str(args.synthetic_docs),
str(args.synthetic_chunks),
str(args.synthetic_repeats),
"1" if args.synthetic_ragged else "0",
],
cwd=ROOT,
).returncode
if __name__ == "__main__":
raise SystemExit(main())
|