diff options
Diffstat (limited to '')
| -rwxr-xr-x | devtools/quant_precision_drift.py | 197 |
1 files changed, 197 insertions, 0 deletions
diff --git a/devtools/quant_precision_drift.py b/devtools/quant_precision_drift.py new file mode 100755 index 0000000..896d2f8 --- /dev/null +++ b/devtools/quant_precision_drift.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Compare f32 -> low-precision int8-output drift across engines. + +The native BF16 path changes ~10-15% of int8 output bytes versus F32 (while +keeping cosine ~0.9999 and identical retrieval). This script checks whether that +is normal by measuring the *same* f32 -> low-precision drift in independent +implementations of the same model: + + - ONNX : released model.onnx (F32) vs model_quantized.onnx (official int8) + - HF : sentence-transformers / torch, loaded in F32 vs BF16 + - native : ffwd binary, an F32 model dir vs a BF16 model dir + +For every engine it reports, per low-precision variant against that engine's own +F32 output: the int8 byte-change rate, the int8-vector cosine, and the float +relative-L2 diff. If ONNX-int8 and HF-bf16 drift like ours, the drift is inherent +to low precision, not a backend defect. + +Reference engines output int8-scale embeddings (round); native outputs raw +floats quantized with tanh*127 (matching the server's base64_int8 encoder). + +Needs the reference stack, so run it with uv (Python 3.12 for the numpy pin): + + uv run --python 3.12 --with-requirements requirements-reference.txt \ + devtools/quant_precision_drift.py --model-dir DIR --engines hf +""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +DEFAULT_TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: The Eiffel Tower is a famous landmark located in Paris, France.", + "document: Photosynthesis converts sunlight into chemical energy in plants.", + ( + "document: Embedding models turn text into dense vectors for semantic " + "retrieval. Long document inference stresses bidirectional attention " + "because every token attends to every other token in the same packed " + "sequence. " + ) * 8, +] + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + if na == 0.0 or nb == 0.0: + return 1.0 + return dot / (na * nb) + + +def rel_l2(a: list[float], b: list[float]) -> float: + num = math.sqrt(sum((x - y) ** 2 for x, y in zip(a, b))) + den = math.sqrt(sum(x * x for x in a)) + return num / den if den else 0.0 + + +def drift(ref: list[list[int]], low: list[list[int]]) -> dict[str, float]: + if len(ref) != len(low): + raise RuntimeError(f"row count mismatch: {len(ref)} vs {len(low)}") + total = 0 + changed = 0 + worst_cos = 1.0 + worst_l2 = 0.0 + for a, b in zip(ref, low): + if len(a) != len(b): + raise RuntimeError(f"dim mismatch: {len(a)} vs {len(b)}") + total += len(a) + changed += sum(1 for x, y in zip(a, b) if x != y) + worst_cos = min(worst_cos, cosine([float(x) for x in a], + [float(y) for y in b])) + worst_l2 = max(worst_l2, rel_l2([float(x) for x in a], + [float(y) for y in b])) + return { + "int8_change_rate": changed / total if total else 0.0, + "worst_int8_cosine": worst_cos, + "worst_rel_l2": worst_l2, + } + + +# --- engines ------------------------------------------------------------- + +def onnx_int8(model_dir: str, onnx_file: str, texts: list[str]) -> list[list[int]]: + import numpy as np + import onnxruntime as ort + from transformers import AutoTokenizer + + tok = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, + local_files_only=True) + sess = ort.InferenceSession(str(Path(model_dir) / "onnx" / onnx_file), + providers=["CPUExecutionProvider"]) + t = tok(texts, padding=True, truncation=True, return_tensors="np") + feed = {"input_ids": t["input_ids"].astype(np.int64), + "attention_mask": t["attention_mask"].astype(np.int64)} + # outputs: [0]=hidden states, [2]=pooled int8-scale embedding (see + # compare_reference_backends.worker_onnx). + out = sess.run(None, feed)[2] + return [[int(round(float(x))) for x in row] for row in out] + + +def hf_int8(model_dir: str, texts: list[str], dtype: str) -> list[list[int]]: + import torch + from sentence_transformers import SentenceTransformer + + td = {"f32": torch.float32, "bf16": torch.bfloat16}[dtype] + model = SentenceTransformer(model_dir, trust_remote_code=True, + local_files_only=True, + model_kwargs={"torch_dtype": td}) + out = model.encode(texts, show_progress_bar=False) + return [[int(round(float(x))) for x in row] for row in out.tolist()] + + +def native_int8(binary: str, model_dir: str, backend: str, + texts: list[str]) -> list[list[int]]: + cmd = [binary, "-d", model_dir, f"--{backend}", "--stream", "-b", "8"] + proc = subprocess.run(cmd, input="\n".join(texts) + "\n", text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise RuntimeError(f"native failed: {' '.join(cmd)}\n{proc.stderr[-800:]}") + rows = [] + for line in proc.stdout.splitlines(): + if not line.strip(): + continue + emb = json.loads(line)["embedding"] + rows.append([max(-128, min(127, round(math.tanh(float(x)) * 127.0))) + for x in emb]) + return rows + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--model-dir", required=True, + help="F32 reference model dir (HF snapshot with onnx/)") + ap.add_argument("--engines", default="onnx,hf", + help="comma list: onnx,hf,native") + ap.add_argument("--native-binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--native-backend", default="mlx", choices=("cpu", "mlx")) + ap.add_argument("--native-bf16-dir", + help="BF16 model dir for the native engine (required for native)") + ap.add_argument("--onnx-low", default="model_quantized.onnx", + help="quantized ONNX file (vs model.onnx)") + args = ap.parse_args() + + texts = DEFAULT_TEXTS + engines = [e for e in args.engines.split(",") if e] + results: list[tuple[str, str, dict[str, float]]] = [] + + for eng in engines: + try: + if eng == "onnx": + ref = onnx_int8(args.model_dir, "model.onnx", texts) + low = onnx_int8(args.model_dir, args.onnx_low, texts) + results.append(("ONNX", f"f32 vs {args.onnx_low}", drift(ref, low))) + elif eng == "hf": + ref = hf_int8(args.model_dir, texts, "f32") + low = hf_int8(args.model_dir, texts, "bf16") + results.append(("HF/torch", "f32 vs bf16", drift(ref, low))) + elif eng == "native": + if not args.native_bf16_dir: + raise RuntimeError("--native-bf16-dir required for native") + ref = native_int8(args.native_binary, args.model_dir, + args.native_backend, texts) + low = native_int8(args.native_binary, args.native_bf16_dir, + args.native_backend, texts) + results.append((f"native/{args.native_backend}", "f32 vs bf16", + drift(ref, low))) + else: + print(f"unknown engine: {eng}", file=sys.stderr) + except Exception as exc: # one engine failing should not block the rest + print(f" [{eng}] SKIPPED: {type(exc).__name__}: {exc}", + file=sys.stderr) + + print(f"\nf32 -> low-precision int8 drift ({len(texts)} texts)\n") + print(f"{'engine':14s} {'comparison':24s} {'int8_change':>11s} " + f"{'int8_cosine':>12s} {'rel_l2':>8s}") + print("-" * 74) + for name, comp, d in results: + print(f"{name:14s} {comp:24s} {d['int8_change_rate']*100:10.1f}% " + f"{d['worst_int8_cosine']:12.5f} {d['worst_rel_l2']*100:7.2f}%") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
