aboutsummaryrefslogtreecommitdiff
path: root/tests/check_contextual_batch_parity.py
blob: e7f1e9436e4f37c83685adbfaf0326db651f3f59 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
#!/usr/bin/env python3
"""Run the contextual batch-vs-singleton parity check.

The C test is tests/contextual_batch.c, built by the Makefile target
`parity-context-driver`.

This script builds the driver and runs it with the requested parameters. It
compares the single-document path against the batched path in C, with no
external reference.

Requires a model directory.

Usage:
  tests/check_contextual_batch_parity.py --model-dir DIR
"""

import argparse
import os
import platform
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]


def build_driver(cc: str, backend: str) -> Path:
    """Build the contextual-batch driver via the Makefile; return its path."""
    target = "parity-context-driver-mlx" if backend == "mlx" else "parity-context-driver"
    binary = "contextual_batch_mlx" if backend == "mlx" else "contextual_batch"
    proc = subprocess.run(["make", "-C", "tests", f"CC={cc}", target], cwd=ROOT,
                          text=True, stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT, check=False)
    if proc.returncode != 0:
        sys.stderr.write(proc.stdout)
        raise SystemExit(proc.returncode)
    return ROOT / "tests" / binary


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--model-dir", required=True)
    ap.add_argument("--backend", choices=("cpu", "mlx"), default="cpu")
    ap.add_argument("--cc", default=os.environ.get("CC", "cc"))
    ap.add_argument("--runs", type=int, default=0,
                    help="run N timed contextual document batches after parity")
    ap.add_argument("--max-diff", type=float, default=0.00005)
    ap.add_argument("--min-cosine", type=float, default=0.99999)
    ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits",
                    dest="mlx_quantize_bits", type=int, default=0,
                    choices=(0, 8))
    ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size",
                    dest="mlx_quantize_group_size", type=int, default=64)
    ap.add_argument("--synthetic-docs", type=int, default=0,
                    help="generate N synthetic documents instead of the smoke set")
    ap.add_argument("--synthetic-chunks", type=int, default=0,
                    help="chunks per synthetic document")
    ap.add_argument("--synthetic-repeats", type=int, default=1,
                    help="repeated text blocks per synthetic chunk")
    ap.add_argument("--synthetic-ragged", action="store_true",
                    help="scale synthetic chunk count by document index")
    args = ap.parse_args()
    if args.mlx_quantize_bits and args.backend != "mlx":
        raise SystemExit("--mlx-quant-bits requires --backend mlx")
    if args.mlx_quantize_group_size <= 0:
        raise SystemExit("--mlx-quant-group-size must be > 0")
    if args.backend == "mlx" and platform.system() != "Darwin":
        raise SystemExit("MLX checks require macOS")

    driver = build_driver(args.cc, args.backend)
    return subprocess.run(
        [
            str(driver),
            args.model_dir,
            str(args.runs),
            str(args.max_diff),
            str(args.min_cosine),
            str(args.mlx_quantize_bits),
            str(args.mlx_quantize_group_size),
            str(args.synthetic_docs),
            str(args.synthetic_chunks),
            str(args.synthetic_repeats),
            "1" if args.synthetic_ragged else "0",
        ],
        cwd=ROOT,
    ).returncode


if __name__ == "__main__":
    raise SystemExit(main())