aboutsummaryrefslogtreecommitdiff
path: root/tests/check_reference_parity.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:36:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commit6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch)
tree1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/check_reference_parity.py
parentdbdf714bb57578431924e5d069089bdfd9f4c604 (diff)
downloadffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz
ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/check_reference_parity.py')
-rw-r--r--tests/check_reference_parity.py204
1 files changed, 204 insertions, 0 deletions
diff --git a/tests/check_reference_parity.py b/tests/check_reference_parity.py
new file mode 100644
index 0000000..81488b3
--- /dev/null
+++ b/tests/check_reference_parity.py
@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Compare the ffwd server against the official model snapshot.
+
+Builds and launches ffwd-server with the given pplx-embed model, then for each
+text requests both float and base64_int8 embeddings from /v1/embeddings and
+compares them against SentenceTransformer(model_dir).encode(). pplx-embed emits
+unnormalized int8, so this checks the server's int8 output against the reference.
+
+Pass --base-url to test an already-running server instead of launching one. The
+reference side needs torch + sentence-transformers, so run it with uv:
+
+ uv run --python 3.12 --with-requirements requirements-reference.txt \
+ tests/check_reference_parity.py --model-dir DIR
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import math
+import os
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+HOST = "127.0.0.1"
+API_KEY = "tt-test-key"
+SERVER_BIN = ROOT / "tools/server/ffwd-server"
+
+DEFAULT_TEXTS = [
+ "document: Your text string goes here",
+ "document: what is the capital of France?",
+ "document: Redis is an in-memory data structure server used for caching.",
+]
+
+
+def ensure_server() -> None:
+ """Build ffwd-server if its binary is missing."""
+ if not SERVER_BIN.exists():
+ subprocess.run(["make", "-C", "tools/server", "all"], cwd=ROOT, check=True,
+ stdout=subprocess.DEVNULL)
+ if not SERVER_BIN.exists():
+ sys.exit(f"ffwd-server not found and could not be built: {SERVER_BIN}")
+
+
+def free_port() -> int:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.bind((HOST, 0))
+ port = s.getsockname()[1]
+ s.close()
+ return port
+
+
+def post_json(base_url: str, payload: dict, api_key: str | None) -> dict:
+ body = json.dumps(payload).encode("utf-8")
+ headers = {"Content-Type": "application/json"}
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+ req = urllib.request.Request(base_url.rstrip("/") + "/v1/embeddings",
+ data=body, headers=headers)
+ try:
+ with urllib.request.urlopen(req, timeout=120) as resp:
+ return json.loads(resp.read().decode("utf-8"))
+ except urllib.error.HTTPError as exc:
+ text = exc.read().decode("utf-8", errors="replace")
+ raise RuntimeError(f"HTTP {exc.code}: {text}") from exc
+
+
+def die_with_log(proc: subprocess.Popen, log_path: str, msg: str) -> None:
+ try:
+ with open(log_path) as f:
+ tail = "".join(f.readlines()[-20:])
+ except OSError:
+ tail = "(no server log)"
+ try:
+ proc.kill()
+ except OSError:
+ pass
+ sys.exit(f"{msg}\n--- server log tail ---\n{tail}")
+
+
+def wait_ready(base_url: str, api_key: str, model: str,
+ proc: subprocess.Popen, log_path: str, timeout_s: float = 90.0) -> None:
+ """Poll until a real embeddings request returns, retrying while the model loads."""
+ deadline = time.monotonic() + timeout_s
+ warmup = {"model": model, "input": "ready?", "encoding_format": "float"}
+ while time.monotonic() < deadline:
+ if proc.poll() is not None:
+ die_with_log(proc, log_path, "server exited during startup")
+ try:
+ post_json(base_url, warmup, api_key)
+ return
+ except (OSError, RuntimeError):
+ pass
+ time.sleep(0.2)
+ die_with_log(proc, log_path, "server did not become ready in time")
+
+
+def decode_int8(value: str) -> list[int]:
+ return [b - 256 if b > 127 else b for b in base64.b64decode(value)]
+
+
+def cosine(a: list[int], b: list[int]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(x * x for x in b))
+ return dot / (na * nb) if na and nb else 0.0
+
+
+def local_vectors(base_url: str, api_key: str, model: str,
+ text: str) -> tuple[list[int], list[int]]:
+ common = {"model": model, "input": text}
+ floats = post_json(base_url, {**common, "encoding_format": "float"}, api_key)
+ encoded = post_json(base_url, {**common, "encoding_format": "base64_int8"}, api_key)
+
+ float_values = floats["data"][0]["embedding"]
+ if not isinstance(float_values, list):
+ raise RuntimeError("float response did not contain an array")
+ float_int8 = [round(float(x) * 128.0) for x in float_values]
+
+ b64_value = encoded["data"][0]["embedding"]
+ if not isinstance(b64_value, str):
+ raise RuntimeError("base64_int8 response did not contain a string")
+ return float_int8, decode_int8(b64_value)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--model", default="pplx-embed-v1-0.6b")
+ ap.add_argument("--base-url", default=None,
+ help="test a running server instead of launching one")
+ ap.add_argument("--api-key", default=API_KEY)
+ ap.add_argument("--max-int8-diff", type=int, default=1)
+ ap.add_argument("--min-cosine", type=float, default=0.9999)
+ ap.add_argument("--text", action="append", dest="texts")
+ args = ap.parse_args()
+
+ from sentence_transformers import SentenceTransformer
+ texts = args.texts or DEFAULT_TEXTS
+ reference = SentenceTransformer(args.model_dir, trust_remote_code=True,
+ local_files_only=True)
+
+ proc: subprocess.Popen | None = None
+ log_path: str | None = None
+ base_url = args.base_url
+ api_key = args.api_key
+
+ if base_url is None:
+ ensure_server()
+ port = free_port()
+ base_url = f"http://{HOST}:{port}"
+ log = open(tempfile.mktemp(prefix="ffwd-ref-server-", suffix=".log"), "w")
+ log_path = log.name
+ argv = [str(SERVER_BIN), "--host", HOST, "--port", str(port),
+ "--api-key", api_key, "-b", "2",
+ "--model", f"{args.model}={args.model_dir}:api=perplexity"]
+ proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT)
+ log.close()
+ wait_ready(base_url, api_key, args.model, proc, log_path)
+
+ try:
+ ok = True
+ for text in texts:
+ official = [round(float(x)) for x in reference.encode([text])[0]]
+ local_float, local_int8 = local_vectors(base_url, api_key, args.model, text)
+ if local_float != local_int8:
+ raise RuntimeError("local float and base64_int8 responses disagree")
+ max_diff = max(abs(a - b) for a, b in zip(local_int8, official))
+ equal = sum(a == b for a, b in zip(local_int8, official))
+ score = cosine(local_int8, official)
+ passed = (len(local_int8) == len(official)
+ and max_diff <= args.max_int8_diff
+ and score >= args.min_cosine)
+ ok = ok and passed
+ print(f"{'ok' if passed else 'FAIL'}: equal={equal}/{len(official)} "
+ f"max_int8_diff={max_diff} cosine={score:.8f} text={text!r}")
+ finally:
+ if proc is not None:
+ proc.terminate()
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ if log_path:
+ try:
+ os.unlink(log_path)
+ except OSError:
+ pass
+
+ print(f"{'ok' if ok else 'FAIL'}: server reference parity for {len(texts)} "
+ f"texts ({args.model})")
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())