1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
#!/usr/bin/env python3
"""Compare stdin batch_size=1 against true batched stdin execution."""
import argparse
import json
import math
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BINARY = ROOT / "ffwd-cli"
def ensure_cli(binary: Path) -> None:
"""Build the CLI if the default binary is missing."""
if binary == DEFAULT_BINARY and not binary.exists():
subprocess.run(["make", "cpu"], cwd=ROOT, check=True,
stdout=subprocess.DEVNULL)
DEFAULT_TEXTS = [
"query: what is the capital of France?",
"document: Paris is the capital of France.",
"document: Berlin is the capital of Germany.",
"document: Istanbul is a major city in Turkey.",
"A short string.",
(
"Embedding models convert text into dense vectors for retrieval, "
"clustering, and semantic similarity search."
),
]
LONG_TEXTS = [
"document: " + (
"Embedding models turn text into dense vectors for semantic retrieval. "
"Long document inference stresses bidirectional attention because every "
"token attends to every other token in the same packed sequence. "
) * repeats
for repeats in (8, 10, 12, 14)
]
def run_stdin(binary, model_dir, backend, batch_size, texts,
mlx_quantize_bits, mlx_quantize_group_size,
cuda_weight_dtype=None):
cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
if backend == "mlx":
if mlx_quantize_bits:
cmd.extend([
"--gpu-quant-bits", str(mlx_quantize_bits),
"--gpu-quant-group-size", str(mlx_quantize_group_size),
])
elif backend == "cuda":
# Force exact-F32 weight storage so a BF16 snapshot is judged for kernel
# exactness (batched vs singleton GEMMs) rather than BF16 rounding.
if cuda_weight_dtype:
cmd.extend(["--gpu-weight-dtype", cuda_weight_dtype])
proc = subprocess.run(
cmd,
input="\n".join(texts) + "\n",
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
)
if proc.returncode != 0:
raise RuntimeError(
f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}"
)
rows = []
for line in proc.stdout.splitlines():
if line.strip():
row = json.loads(line)
if "error" in row:
raise RuntimeError(f"stdin returned error: {row}")
rows.append(row)
if len(rows) != len(texts):
raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}")
return rows
def cosine(a, b):
dot = sum(x * y for x, y in zip(a, b))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
return dot / (na * nb)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model-dir", required=True)
ap.add_argument("--binary", default=str(DEFAULT_BINARY))
ap.add_argument("--backend", choices=("cpu", "mlx", "cuda"), default="cpu")
ap.add_argument("--batch-size", type=int, default=4)
ap.add_argument("--max-diff", type=float, default=5e-5)
ap.add_argument("--min-cosine", type=float, default=0.99999)
ap.add_argument("--long", action="store_true",
help="use long inputs that exercise tiled CPU attention")
ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits",
dest="mlx_quantize_bits", type=int, default=0,
choices=(0, 8))
ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size",
dest="mlx_quantize_group_size", type=int, default=64)
ap.add_argument("--cuda-weight-dtype", choices=("f32", "bf16"), default=None,
help="force CUDA weight storage dtype (default: snapshot dtype)")
ap.add_argument("texts", nargs="*")
args = ap.parse_args()
texts = args.texts or (LONG_TEXTS if args.long else DEFAULT_TEXTS)
if args.batch_size < 2:
raise SystemExit("--batch-size must be >= 2 for a useful parity check")
if args.mlx_quantize_bits and args.backend != "mlx":
raise SystemExit("--mlx-quant-bits requires --backend mlx")
if args.mlx_quantize_group_size <= 0:
raise SystemExit("--mlx-quant-group-size must be > 0")
ensure_cli(Path(args.binary))
seq_rows = run_stdin(args.binary, args.model_dir, args.backend, 1, texts,
args.mlx_quantize_bits,
args.mlx_quantize_group_size,
args.cuda_weight_dtype)
bat_rows = run_stdin(args.binary, args.model_dir, args.backend,
args.batch_size, texts, args.mlx_quantize_bits,
args.mlx_quantize_group_size,
args.cuda_weight_dtype)
worst_diff = 0.0
worst_cos = 1.0
for i, (seq, bat) in enumerate(zip(seq_rows, bat_rows)):
if seq["dim"] != bat["dim"] or seq["tokens"] != bat["tokens"]:
raise RuntimeError(f"metadata mismatch at row {i}: {seq} vs {bat}")
a = seq["embedding"]
b = bat["embedding"]
max_diff = max(abs(x - y) for x, y in zip(a, b))
cos = cosine(a, b)
worst_diff = max(worst_diff, max_diff)
worst_cos = min(worst_cos, cos)
print(
f"[{i}] tokens={seq['tokens']} cosine={cos:.8f} "
f"max_abs_diff={max_diff:.8g}"
)
if worst_diff > args.max_diff or worst_cos < args.min_cosine:
raise RuntimeError(
f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}"
)
seq_ms = sum(float(row["ms"]) for row in seq_rows)
batch_ms = sum(float(bat_rows[i]["ms"]) for i in range(0, len(bat_rows), args.batch_size))
print(
f"ok: {len(texts)} texts, backend={args.backend}, "
f"batch_size={args.batch_size}, worst_diff={worst_diff:.8g}, "
f"worst_cos={worst_cos:.8f}, seq_ms_sum={seq_ms:.1f}, "
f"batch_chunk_ms_sum={batch_ms:.1f}"
+ (
f", mlx_q{args.mlx_quantize_bits}"
if args.mlx_quantize_bits else ""
)
)
if __name__ == "__main__":
try:
main()
except Exception as exc:
print(f"error: {exc}", file=sys.stderr)
sys.exit(1)
|