aboutsummaryrefslogtreecommitdiff
path: root/tests/check_late_interaction.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:36:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commit6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch)
tree1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/check_late_interaction.py
parentdbdf714bb57578431924e5d069089bdfd9f4c604 (diff)
downloadffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz
ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/check_late_interaction.py')
-rw-r--r--tests/check_late_interaction.py159
1 files changed, 159 insertions, 0 deletions
diff --git a/tests/check_late_interaction.py b/tests/check_late_interaction.py
new file mode 100644
index 0000000..e9dedb4
--- /dev/null
+++ b/tests/check_late_interaction.py
@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""Smoke-check late-interaction token vectors and MaxSim ranking.
+
+The C side is tests/late_check.c, built by the Makefile target
+`late-check-driver`. Plain mode runs the driver against a real late model and
+checks the expected ranking; it needs only python3 and a late model directory
+(e.g. pplx-embed-v1-late-0.6b). With --compare-pylate it also compares the C
+token vectors against PyLate, which needs the pylate package:
+
+ tests/check_late_interaction.py --model-dir DIR
+ uv run --python 3.12 --with pylate tests/check_late_interaction.py \
+ --model-dir DIR --compare-pylate
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def build_driver(cc: str, backend: str) -> Path:
+ """Build the late-interaction driver via the Makefile; return its path."""
+ target = "late-check-driver-mlx" if backend == "mlx" else "late-check-driver"
+ binary = "late_check_mlx" if backend == "mlx" else "late_check"
+ proc = subprocess.run(["make", "-C", "tests", f"CC={cc}", target], cwd=ROOT,
+ text=True, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, check=False)
+ if proc.returncode != 0:
+ sys.stderr.write(proc.stdout)
+ raise SystemExit(proc.returncode)
+ return ROOT / "tests" / binary
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def compare_with_pylate(model_dir: str, c_json: dict,
+ max_score_diff: float, min_vector_cosine: float) -> None:
+ try:
+ from pylate import models
+ from pylate.scores import colbert_scores
+ except ImportError as e:
+ raise SystemExit(
+ "missing PyLate dependency; run with: uv run --python 3.12 --with pylate "
+ "tests/check_late_interaction.py --compare-pylate ..."
+ ) from e
+
+ query = "What motivates scientific discovery?"
+ docs = [
+ "Scientists explore the universe driven by curiosity.",
+ "Children learn through curious exploration.",
+ "Historical discoveries began with curious questions.",
+ ]
+
+ model = models.ColBERT(
+ model_name_or_path=model_dir,
+ trust_remote_code=True,
+ device="cpu",
+ )
+ q = model.encode([query], is_query=True, convert_to_tensor=True,
+ show_progress_bar=False)[0].detach().cpu()
+ d = [
+ x.detach().cpu()
+ for x in model.encode(docs, is_query=False, convert_to_tensor=True,
+ show_progress_bar=False)
+ ]
+
+ c_query = c_json["query"]
+ if len(c_query) != q.shape[0] or len(c_query[0]) != q.shape[1]:
+ raise RuntimeError(
+ f"query shape mismatch: C={len(c_query)}x{len(c_query[0])} "
+ f"PyLate={tuple(q.shape)}"
+ )
+
+ worst_cos = 1.0
+ for i, row in enumerate(c_query):
+ worst_cos = min(worst_cos, cosine(row, q[i].tolist()))
+
+ py_scores = []
+ for i, py_doc in enumerate(d):
+ c_doc = c_json["documents"][i]["vectors"]
+ if len(c_doc) != py_doc.shape[0] or len(c_doc[0]) != py_doc.shape[1]:
+ raise RuntimeError(
+ f"doc{i} shape mismatch: C={len(c_doc)}x{len(c_doc[0])} "
+ f"PyLate={tuple(py_doc.shape)}"
+ )
+ for j, row in enumerate(c_doc):
+ worst_cos = min(worst_cos, cosine(row, py_doc[j].tolist()))
+ py_scores.append(float(colbert_scores(q.unsqueeze(0),
+ py_doc.unsqueeze(0))[0, 0]))
+
+ c_scores = [float(x) for x in c_json["scores"]]
+ worst_score_diff = max(abs(a - b) for a, b in zip(c_scores, py_scores))
+ py_best = max(range(len(py_scores)), key=lambda i: py_scores[i])
+ if c_json["best"] != py_best:
+ raise RuntimeError(f"best mismatch: C={c_json['best']} PyLate={py_best}")
+ if worst_score_diff > max_score_diff or worst_cos < min_vector_cosine:
+ raise RuntimeError(
+ f"late PyLate parity failed: worst_score_diff={worst_score_diff:g} "
+ f"worst_vector_cosine={worst_cos:g}"
+ )
+
+ print(
+ "pylate parity: "
+ f"worst_score_diff={worst_score_diff:.8g} "
+ f"worst_vector_cosine={worst_cos:.8f} best=doc{py_best}"
+ )
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--backend", choices=("cpu", "mlx"), default="cpu")
+ ap.add_argument("--cc", default=os.environ.get("CC", "cc"))
+ ap.add_argument("--compare-pylate", action="store_true",
+ help="compare all retained C token vectors against PyLate")
+ ap.add_argument("--max-score-diff", type=float, default=1e-4)
+ ap.add_argument("--min-vector-cosine", type=float, default=0.99999)
+ ap.add_argument("--runs", type=int, default=0,
+ help="run N timed packed MaxSim reranking loops")
+ args = ap.parse_args()
+ binary = build_driver(args.cc, args.backend)
+
+ if args.compare_pylate:
+ proc = subprocess.run(
+ [str(binary), args.model_dir, "--json"],
+ cwd=ROOT,
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=False,
+ )
+ if proc.returncode != 0:
+ sys.stderr.write(proc.stdout)
+ sys.stderr.write(proc.stderr)
+ return proc.returncode
+ c_json = json.loads(proc.stdout)
+ compare_with_pylate(args.model_dir, c_json, args.max_score_diff,
+ args.min_vector_cosine)
+ return 0
+ cmd = [str(binary), args.model_dir]
+ if args.runs:
+ cmd += ["--runs", str(args.runs)]
+ return subprocess.run(cmd, cwd=ROOT).returncode
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())