1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
#!/usr/bin/env python3
"""Compare embeddings from two model directories through the CLI."""
from __future__ import annotations
import argparse
import json
import math
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
def ensure_cli(binary: Path) -> None:
"""Build the CLI if the default binary is missing."""
if binary == DEFAULT_BINARY and not binary.exists():
subprocess.run(["make", "-C", "tools/cli", "all"], cwd=ROOT, check=True,
stdout=subprocess.DEVNULL)
TEXTS = [
"query: what is the capital of France?",
"document: Paris is the capital of France.",
"document: Berlin is the capital of Germany.",
"The quick brown fox jumps over the lazy dog near the riverbank",
(
"Retrieval augmented generation combines embedding models with external "
"knowledge sources and ranks documents by semantic similarity."
),
]
def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int,
backend: str, threads: int) -> list[list[float]]:
cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
if backend == "cpu" and threads > 0:
cmd.extend(["-t", str(threads)])
proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
assert proc.stdin is not None
assert proc.stdout is not None
proc.stdin.write("\n".join(texts) + "\n")
proc.stdin.close()
embeddings = []
for _ in texts:
line = proc.stdout.readline()
if not line:
stderr = proc.stderr.read() if proc.stderr else ""
raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}")
row = json.loads(line)
if "error" in row:
raise RuntimeError(f"stdin error for {model_dir}: {row}")
embeddings.append([float(x) for x in row["embedding"]])
rc = proc.wait(timeout=10)
if rc != 0:
stderr = proc.stderr.read() if proc.stderr else ""
raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}")
return embeddings
def cosine(a: list[float], b: list[float]) -> float:
dot = sum(x * y for x, y in zip(a, b))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
return dot / (na * nb)
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--binary", default=str(DEFAULT_BINARY))
ap.add_argument("--a", required=True, help="reference model directory")
ap.add_argument("--b", required=True, help="comparison model directory")
ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu")
ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"],
help="backend for --a; defaults to --backend")
ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"],
help="backend for --b; defaults to --backend")
ap.add_argument("--batch-size", type=int, default=4)
ap.add_argument("--threads", type=int, default=0)
ap.add_argument("--max-diff", type=float, default=0.01)
ap.add_argument("--min-cos", type=float, default=0.999)
args = ap.parse_args()
if args.batch_size <= 0:
raise SystemExit("--batch-size must be > 0")
ensure_cli(Path(args.binary))
if not Path(args.binary).exists():
raise SystemExit(f"binary not found: {args.binary}")
backend_a = args.backend_a or args.backend
backend_b = args.backend_b or args.backend
emb_a = run_model(args.binary, args.a, TEXTS, args.batch_size,
backend_a, args.threads)
emb_b = run_model(args.binary, args.b, TEXTS, args.batch_size,
backend_b, args.threads)
worst_diff = 0.0
worst_cos = 1.0
for i, (a, b) in enumerate(zip(emb_a, emb_b)):
if len(a) != len(b):
raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}")
max_diff = max(abs(x - y) for x, y in zip(a, b))
cos = cosine(a, b)
worst_diff = max(worst_diff, max_diff)
worst_cos = min(worst_cos, cos)
print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}")
print(
f"ok: backend_a={backend_a} backend_b={backend_b} "
f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}"
)
if worst_diff > args.max_diff or worst_cos < args.min_cos:
raise SystemExit(
f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
|