aboutsummaryrefslogtreecommitdiff
path: root/devtools/compare_model_embeddings.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:31:06 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitdbdf714bb57578431924e5d069089bdfd9f4c604 (patch)
tree4f5b59ef132220310e1e4b1dde64f5e38782369a /devtools/compare_model_embeddings.py
parente9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff)
downloadffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz
ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to 'devtools/compare_model_embeddings.py')
-rwxr-xr-xdevtools/compare_model_embeddings.py130
1 files changed, 130 insertions, 0 deletions
diff --git a/devtools/compare_model_embeddings.py b/devtools/compare_model_embeddings.py
new file mode 100755
index 0000000..95154a2
--- /dev/null
+++ b/devtools/compare_model_embeddings.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Compare embeddings from two model directories through the CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import subprocess
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
+
+
+def ensure_cli(binary: Path) -> None:
+ """Build the CLI if the default binary is missing."""
+ if binary == DEFAULT_BINARY and not binary.exists():
+ subprocess.run(["make", "-C", "tools/cli", "all"], cwd=ROOT, check=True,
+ stdout=subprocess.DEVNULL)
+
+
+TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "The quick brown fox jumps over the lazy dog near the riverbank",
+ (
+ "Retrieval augmented generation combines embedding models with external "
+ "knowledge sources and ranks documents by semantic similarity."
+ ),
+]
+
+
+def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int,
+ backend: str, threads: int) -> list[list[float]]:
+ cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
+ if backend == "cpu" and threads > 0:
+ cmd.extend(["-t", str(threads)])
+
+ proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ )
+ assert proc.stdin is not None
+ assert proc.stdout is not None
+ proc.stdin.write("\n".join(texts) + "\n")
+ proc.stdin.close()
+
+ embeddings = []
+ for _ in texts:
+ line = proc.stdout.readline()
+ if not line:
+ stderr = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}")
+ row = json.loads(line)
+ if "error" in row:
+ raise RuntimeError(f"stdin error for {model_dir}: {row}")
+ embeddings.append([float(x) for x in row["embedding"]])
+
+ rc = proc.wait(timeout=10)
+ if rc != 0:
+ stderr = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}")
+ return embeddings
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--binary", default=str(DEFAULT_BINARY))
+ ap.add_argument("--a", required=True, help="reference model directory")
+ ap.add_argument("--b", required=True, help="comparison model directory")
+ ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu")
+ ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"],
+ help="backend for --a; defaults to --backend")
+ ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"],
+ help="backend for --b; defaults to --backend")
+ ap.add_argument("--batch-size", type=int, default=4)
+ ap.add_argument("--threads", type=int, default=0)
+ ap.add_argument("--max-diff", type=float, default=0.01)
+ ap.add_argument("--min-cos", type=float, default=0.999)
+ args = ap.parse_args()
+
+ if args.batch_size <= 0:
+ raise SystemExit("--batch-size must be > 0")
+ ensure_cli(Path(args.binary))
+ if not Path(args.binary).exists():
+ raise SystemExit(f"binary not found: {args.binary}")
+
+ backend_a = args.backend_a or args.backend
+ backend_b = args.backend_b or args.backend
+ emb_a = run_model(args.binary, args.a, TEXTS, args.batch_size,
+ backend_a, args.threads)
+ emb_b = run_model(args.binary, args.b, TEXTS, args.batch_size,
+ backend_b, args.threads)
+
+ worst_diff = 0.0
+ worst_cos = 1.0
+ for i, (a, b) in enumerate(zip(emb_a, emb_b)):
+ if len(a) != len(b):
+ raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}")
+ max_diff = max(abs(x - y) for x, y in zip(a, b))
+ cos = cosine(a, b)
+ worst_diff = max(worst_diff, max_diff)
+ worst_cos = min(worst_cos, cos)
+ print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}")
+
+ print(
+ f"ok: backend_a={backend_a} backend_b={backend_b} "
+ f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}"
+ )
+ if worst_diff > args.max_diff or worst_cos < args.min_cos:
+ raise SystemExit(
+ f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())