aboutsummaryrefslogtreecommitdiff
path: root/tests/check_mlx_quantized_parity.py
blob: 1bcb81a2d474e81270a92772472500f12345d681 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#!/usr/bin/env python3
"""Compare normal MLX embeddings against opt-in quantized MLX weights."""

import argparse
import json
import math
import subprocess
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BINARY = ROOT / "ffwd-cli"


TEXTS = [
    "query: what is the capital of France?",
    "document: Paris is the capital of France.",
    "document: Berlin is the capital of Germany.",
    "document: Istanbul is a major city in Turkey.",
    "The quick brown fox jumps over the lazy dog near the riverbank.",
    (
        "Retrieval augmented generation combines embedding models with "
        "external knowledge sources and ranks documents by semantic similarity."
    ),
]


def run(binary: str, model_dir: str, texts: list[str], batch_size: int,
        bits: int, group_size: int) -> list[list[float]]:
    cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
    if bits:
        cmd.extend([
            "--gpu-quant-bits", str(bits),
            "--gpu-quant-group-size", str(group_size),
        ])
    proc = subprocess.run(
        cmd,
        input="\n".join(texts) + "\n",
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        check=False,
    )
    if proc.returncode != 0:
        raise RuntimeError(
            f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}"
        )
    rows = []
    for line in proc.stdout.splitlines():
        if not line.strip():
            continue
        row = json.loads(line)
        if "error" in row:
            raise RuntimeError(f"stdin returned error: {row}")
        rows.append(row["embedding"])
    if len(rows) != len(texts):
        raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}")
    return rows


def cosine(a: list[float], b: list[float]) -> float:
    dot = sum(x * y for x, y in zip(a, b))
    na = math.sqrt(sum(x * x for x in a))
    nb = math.sqrt(sum(y * y for y in b))
    return dot / (na * nb)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--binary", default=str(DEFAULT_BINARY))
    ap.add_argument("--model-dir", required=True)
    ap.add_argument("--batch-size", type=int, default=4)
    ap.add_argument("--bits", type=int, default=8, choices=(8,))
    ap.add_argument("--group-size", type=int, default=64)
    ap.add_argument("--max-diff", type=float, default=0.08)
    ap.add_argument("--min-cosine", type=float, default=0.995)
    args = ap.parse_args()

    ref = run(args.binary, args.model_dir, TEXTS, args.batch_size, 0,
              args.group_size)
    got = run(args.binary, args.model_dir, TEXTS, args.batch_size, args.bits,
              args.group_size)

    worst_diff = 0.0
    worst_cos = 1.0
    for i, (a, b) in enumerate(zip(ref, got)):
        max_diff = max(abs(x - y) for x, y in zip(a, b))
        cos = cosine(a, b)
        worst_diff = max(worst_diff, max_diff)
        worst_cos = min(worst_cos, cos)
        print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}")

    print(
        f"ok: bits={args.bits} group_size={args.group_size} "
        f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}"
    )
    if worst_diff > args.max_diff or worst_cos < args.min_cosine:
        raise SystemExit(
            f"quantized parity failed: diff={worst_diff:g}, cos={worst_cos:g}"
        )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())