diff options
Diffstat (limited to 'tests/check_reference_parity.py')
| -rw-r--r-- | tests/check_reference_parity.py | 204 |
1 files changed, 204 insertions, 0 deletions
diff --git a/tests/check_reference_parity.py b/tests/check_reference_parity.py new file mode 100644 index 0000000..81488b3 --- /dev/null +++ b/tests/check_reference_parity.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Compare the ffwd server against the official model snapshot. + +Builds and launches ffwd-server with the given pplx-embed model, then for each +text requests both float and base64_int8 embeddings from /v1/embeddings and +compares them against SentenceTransformer(model_dir).encode(). pplx-embed emits +unnormalized int8, so this checks the server's int8 output against the reference. + +Pass --base-url to test an already-running server instead of launching one. The +reference side needs torch + sentence-transformers, so run it with uv: + + uv run --python 3.12 --with-requirements requirements-reference.txt \ + tests/check_reference_parity.py --model-dir DIR +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOST = "127.0.0.1" +API_KEY = "tt-test-key" +SERVER_BIN = ROOT / "tools/server/ffwd-server" + +DEFAULT_TEXTS = [ + "document: Your text string goes here", + "document: what is the capital of France?", + "document: Redis is an in-memory data structure server used for caching.", +] + + +def ensure_server() -> None: + """Build ffwd-server if its binary is missing.""" + if not SERVER_BIN.exists(): + subprocess.run(["make", "-C", "tools/server", "all"], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL) + if not SERVER_BIN.exists(): + sys.exit(f"ffwd-server not found and could not be built: {SERVER_BIN}") + + +def free_port() -> int: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((HOST, 0)) + port = s.getsockname()[1] + s.close() + return port + + +def post_json(base_url: str, payload: dict, api_key: str | None) -> dict: + body = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + req = urllib.request.Request(base_url.rstrip("/") + "/v1/embeddings", + data=body, headers=headers) + try: + with urllib.request.urlopen(req, timeout=120) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + text = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {exc.code}: {text}") from exc + + +def die_with_log(proc: subprocess.Popen, log_path: str, msg: str) -> None: + try: + with open(log_path) as f: + tail = "".join(f.readlines()[-20:]) + except OSError: + tail = "(no server log)" + try: + proc.kill() + except OSError: + pass + sys.exit(f"{msg}\n--- server log tail ---\n{tail}") + + +def wait_ready(base_url: str, api_key: str, model: str, + proc: subprocess.Popen, log_path: str, timeout_s: float = 90.0) -> None: + """Poll until a real embeddings request returns, retrying while the model loads.""" + deadline = time.monotonic() + timeout_s + warmup = {"model": model, "input": "ready?", "encoding_format": "float"} + while time.monotonic() < deadline: + if proc.poll() is not None: + die_with_log(proc, log_path, "server exited during startup") + try: + post_json(base_url, warmup, api_key) + return + except (OSError, RuntimeError): + pass + time.sleep(0.2) + die_with_log(proc, log_path, "server did not become ready in time") + + +def decode_int8(value: str) -> list[int]: + return [b - 256 if b > 127 else b for b in base64.b64decode(value)] + + +def cosine(a: list[int], b: list[int]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(x * x for x in b)) + return dot / (na * nb) if na and nb else 0.0 + + +def local_vectors(base_url: str, api_key: str, model: str, + text: str) -> tuple[list[int], list[int]]: + common = {"model": model, "input": text} + floats = post_json(base_url, {**common, "encoding_format": "float"}, api_key) + encoded = post_json(base_url, {**common, "encoding_format": "base64_int8"}, api_key) + + float_values = floats["data"][0]["embedding"] + if not isinstance(float_values, list): + raise RuntimeError("float response did not contain an array") + float_int8 = [round(float(x) * 128.0) for x in float_values] + + b64_value = encoded["data"][0]["embedding"] + if not isinstance(b64_value, str): + raise RuntimeError("base64_int8 response did not contain a string") + return float_int8, decode_int8(b64_value) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--model", default="pplx-embed-v1-0.6b") + ap.add_argument("--base-url", default=None, + help="test a running server instead of launching one") + ap.add_argument("--api-key", default=API_KEY) + ap.add_argument("--max-int8-diff", type=int, default=1) + ap.add_argument("--min-cosine", type=float, default=0.9999) + ap.add_argument("--text", action="append", dest="texts") + args = ap.parse_args() + + from sentence_transformers import SentenceTransformer + texts = args.texts or DEFAULT_TEXTS + reference = SentenceTransformer(args.model_dir, trust_remote_code=True, + local_files_only=True) + + proc: subprocess.Popen | None = None + log_path: str | None = None + base_url = args.base_url + api_key = args.api_key + + if base_url is None: + ensure_server() + port = free_port() + base_url = f"http://{HOST}:{port}" + log = open(tempfile.mktemp(prefix="ffwd-ref-server-", suffix=".log"), "w") + log_path = log.name + argv = [str(SERVER_BIN), "--host", HOST, "--port", str(port), + "--api-key", api_key, "-b", "2", + "--model", f"{args.model}={args.model_dir}:api=perplexity"] + proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT) + log.close() + wait_ready(base_url, api_key, args.model, proc, log_path) + + try: + ok = True + for text in texts: + official = [round(float(x)) for x in reference.encode([text])[0]] + local_float, local_int8 = local_vectors(base_url, api_key, args.model, text) + if local_float != local_int8: + raise RuntimeError("local float and base64_int8 responses disagree") + max_diff = max(abs(a - b) for a, b in zip(local_int8, official)) + equal = sum(a == b for a, b in zip(local_int8, official)) + score = cosine(local_int8, official) + passed = (len(local_int8) == len(official) + and max_diff <= args.max_int8_diff + and score >= args.min_cosine) + ok = ok and passed + print(f"{'ok' if passed else 'FAIL'}: equal={equal}/{len(official)} " + f"max_int8_diff={max_diff} cosine={score:.8f} text={text!r}") + finally: + if proc is not None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + if log_path: + try: + os.unlink(log_path) + except OSError: + pass + + print(f"{'ok' if ok else 'FAIL'}: server reference parity for {len(texts)} " + f"texts ({args.model})") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) |
