1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
#!/usr/bin/env python3
"""Compare stdin batch_size=1 against true batched stdin execution."""
import argparse
import json
import math
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BINARY = ROOT / "ffwd-cli"
def ensure_cli(binary: Path) -> None:
"""Build the CLI if the default binary is missing."""
if binary == DEFAULT_BINARY and not binary.exists():
subprocess.run(["make", "cpu"], cwd=ROOT, check=True,
stdout=subprocess.DEVNULL)
DEFAULT_TEXTS = [
"query: what is the capital of France?",
"document: Paris is the capital of France.",
"document: Berlin is the capital of Germany.",
"document: Istanbul is a major city in Turkey.",
"A short string.",
(
"Embedding models convert text into dense vectors for retrieval, "
"clustering, and semantic similarity search."
),
]
LONG_TEXTS = [
"document: " + (
"Embedding models turn text into dense vectors for semantic retrieval. "
"Long document inference stresses bidirectional attention because every "
"token attends to every other token in the same packed sequence. "
) * repeats
for repeats in (8, 10, 12, 14)
]
def detect_pooling_mode(model_dir: str) -> str:
"""Return the sentence pooling mode used by ffwd_config_load, if obvious."""
root = Path(model_dir)
pooling_path = root / "1_Pooling" / "config.json"
if pooling_path.exists():
try:
cfg = json.loads(pooling_path.read_text())
except (OSError, json.JSONDecodeError):
return "unknown"
if cfg.get("pooling_mode_lasttoken"):
return "last-token"
if cfg.get("pooling_mode_cls_token"):
return "cls"
if cfg.get("pooling_mode_mean_tokens"):
return "mean"
return "unknown"
config_path = root / "config.json"
if config_path.exists():
try:
cfg = json.loads(config_path.read_text())
except (OSError, json.JSONDecodeError):
return "unknown"
if cfg.get("model_type") == "qwen3":
return "last-token"
return "unknown"
def run_stdin(binary, model_dir, backend, batch_size, texts,
mlx_quantize_bits, mlx_quantize_group_size,
cuda_weight_dtype=None):
cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
if backend == "mlx":
if mlx_quantize_bits:
cmd.extend([
"--gpu-quant-bits", str(mlx_quantize_bits),
"--gpu-quant-group-size", str(mlx_quantize_group_size),
])
elif backend == "cuda":
# Force exact-F32 weight storage so a BF16 snapshot is judged for kernel
# exactness (batched vs singleton GEMMs) rather than BF16 rounding.
if cuda_weight_dtype:
cmd.extend(["--gpu-weight-dtype", cuda_weight_dtype])
proc = subprocess.run(
cmd,
input="\n".join(texts) + "\n",
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False,
)
if proc.returncode != 0:
raise RuntimeError(
f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}"
)
rows = []
for line in proc.stdout.splitlines():
if line.strip():
row = json.loads(line)
if "error" in row:
raise RuntimeError(f"stdin returned error: {row}")
rows.append(row)
if len(rows) != len(texts):
raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}")
return rows
def cosine(a, b):
dot = sum(x * y for x, y in zip(a, b))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
return dot / (na * nb)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model-dir", required=True)
ap.add_argument("--binary", default=str(DEFAULT_BINARY))
ap.add_argument("--backend", choices=("cpu", "mlx", "cuda"), default="cpu")
ap.add_argument("--batch-size", type=int, default=4)
ap.add_argument("--max-diff", type=float, default=5e-5)
ap.add_argument("--min-cosine", type=float, default=0.99999)
ap.add_argument("--cuda-last-token-max-diff", type=float, default=0.002,
help=("effective max-abs gate for CUDA last-token pooling; "
"keeps cosine strict while allowing batched-GEMM "
"float reordering"))
ap.add_argument("--cuda-last-token-min-cosine", type=float, default=0.9999,
help="effective cosine gate for CUDA last-token pooling")
ap.add_argument("--long", action="store_true",
help="use long inputs that exercise tiled CPU attention")
ap.add_argument("--mlx-quant-bits", "--mlx-quantize-bits",
dest="mlx_quantize_bits", type=int, default=0,
choices=(0, 8))
ap.add_argument("--mlx-quant-group-size", "--mlx-quantize-group-size",
dest="mlx_quantize_group_size", type=int, default=64)
ap.add_argument("--cuda-weight-dtype", choices=("f32", "bf16", "f16"), default=None,
help="force CUDA weight storage dtype (default: snapshot dtype)")
ap.add_argument("texts", nargs="*")
args = ap.parse_args()
texts = args.texts or (LONG_TEXTS if args.long else DEFAULT_TEXTS)
if args.batch_size < 2:
raise SystemExit("--batch-size must be >= 2 for a useful parity check")
if args.mlx_quantize_bits and args.backend != "mlx":
raise SystemExit("--mlx-quant-bits requires --backend mlx")
if args.mlx_quantize_group_size <= 0:
raise SystemExit("--mlx-quant-group-size must be > 0")
if args.cuda_last_token_max_diff <= 0:
raise SystemExit("--cuda-last-token-max-diff must be > 0")
if not 0.0 < args.cuda_last_token_min_cosine <= 1.0:
raise SystemExit("--cuda-last-token-min-cosine must be in (0, 1]")
ensure_cli(Path(args.binary))
pooling_mode = detect_pooling_mode(args.model_dir)
effective_max_diff = args.max_diff
effective_min_cosine = args.min_cosine
calibrated_cuda_last_token = args.backend == "cuda" and pooling_mode == "last-token"
if calibrated_cuda_last_token and effective_max_diff < args.cuda_last_token_max_diff:
effective_max_diff = args.cuda_last_token_max_diff
if calibrated_cuda_last_token and effective_min_cosine > args.cuda_last_token_min_cosine:
effective_min_cosine = args.cuda_last_token_min_cosine
seq_rows = run_stdin(args.binary, args.model_dir, args.backend, 1, texts,
args.mlx_quantize_bits,
args.mlx_quantize_group_size,
args.cuda_weight_dtype)
bat_rows = run_stdin(args.binary, args.model_dir, args.backend,
args.batch_size, texts, args.mlx_quantize_bits,
args.mlx_quantize_group_size,
args.cuda_weight_dtype)
worst_diff = 0.0
worst_cos = 1.0
for i, (seq, bat) in enumerate(zip(seq_rows, bat_rows)):
if seq["dim"] != bat["dim"] or seq["tokens"] != bat["tokens"]:
raise RuntimeError(f"metadata mismatch at row {i}: {seq} vs {bat}")
a = seq["embedding"]
b = bat["embedding"]
max_diff = max(abs(x - y) for x, y in zip(a, b))
cos = cosine(a, b)
worst_diff = max(worst_diff, max_diff)
worst_cos = min(worst_cos, cos)
print(
f"[{i}] tokens={seq['tokens']} cosine={cos:.8f} "
f"max_abs_diff={max_diff:.8g}"
)
if worst_diff > effective_max_diff or worst_cos < effective_min_cosine:
raise RuntimeError(
f"batch parity failed: worst_diff={worst_diff:g}, worst_cos={worst_cos:g}, "
f"max_diff_gate={effective_max_diff:g}, min_cosine_gate={effective_min_cosine:g}"
)
calibration_note = ""
if calibrated_cuda_last_token:
calibration_note = (f", pooling={pooling_mode}, max_diff_gate={effective_max_diff:g}, "
f"min_cosine_gate={effective_min_cosine:g}")
seq_ms = sum(float(row["ms"]) for row in seq_rows)
batch_ms = sum(float(bat_rows[i]["ms"]) for i in range(0, len(bat_rows), args.batch_size))
print(
f"ok: {len(texts)} texts, backend={args.backend}, "
f"batch_size={args.batch_size}, worst_diff={worst_diff:.8g}, "
f"worst_cos={worst_cos:.8f}, seq_ms_sum={seq_ms:.1f}, "
f"batch_chunk_ms_sum={batch_ms:.1f}"
+ calibration_note
+ (
f", mlx_q{args.mlx_quantize_bits}"
if args.mlx_quantize_bits else ""
)
)
if __name__ == "__main__":
try:
main()
except Exception as exc:
print(f"error: {exc}", file=sys.stderr)
sys.exit(1)
|