aboutsummaryrefslogtreecommitdiff
path: root/tests/check_reference_parity.py
blob: 7d88bcd10db3b11750a5457d18f29f1f29a43ad6 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
#!/usr/bin/env python3
"""Compare the ffwd server against the official model snapshot.

Builds and launches ffwd-server with the given pplx-embed model, then for each
text requests both float and base64_int8 embeddings from /v1/embeddings and
compares them against SentenceTransformer(model_dir).encode(). pplx-embed emits
unnormalized int8, so this checks the server's int8 output against the reference.

Pass --base-url to test an already-running server instead of launching one. The
reference side needs torch + sentence-transformers, so run it with uv:

  uv run --python 3.12 --with-requirements requirements-reference.txt \
      tests/check_reference_parity.py --model-dir DIR
"""

from __future__ import annotations

import argparse
import base64
import json
import math
import os
import socket
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
HOST = "127.0.0.1"
API_KEY = "tt-test-key"
SERVER_BIN = ROOT / "ffwd-server"

DEFAULT_TEXTS = [
    "document: Your text string goes here",
    "document: what is the capital of France?",
    "document: Redis is an in-memory data structure server used for caching.",
]


def ensure_server() -> None:
    """Build ffwd-server if its binary is missing."""
    if not SERVER_BIN.exists():
        subprocess.run(["make", "cpu"], cwd=ROOT, check=True,
                       stdout=subprocess.DEVNULL)
    if not SERVER_BIN.exists():
        sys.exit(f"ffwd-server not found and could not be built: {SERVER_BIN}")


def free_port() -> int:
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind((HOST, 0))
    port = s.getsockname()[1]
    s.close()
    return port


def post_json(base_url: str, payload: dict, api_key: str | None) -> dict:
    body = json.dumps(payload).encode("utf-8")
    headers = {"Content-Type": "application/json"}
    if api_key:
        headers["Authorization"] = f"Bearer {api_key}"
    req = urllib.request.Request(base_url.rstrip("/") + "/v1/embeddings",
                                 data=body, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=120) as resp:
            return json.loads(resp.read().decode("utf-8"))
    except urllib.error.HTTPError as exc:
        text = exc.read().decode("utf-8", errors="replace")
        raise RuntimeError(f"HTTP {exc.code}: {text}") from exc


def die_with_log(proc: subprocess.Popen, log_path: str, msg: str) -> None:
    try:
        with open(log_path) as f:
            tail = "".join(f.readlines()[-20:])
    except OSError:
        tail = "(no server log)"
    try:
        proc.kill()
    except OSError:
        pass
    sys.exit(f"{msg}\n--- server log tail ---\n{tail}")


def wait_ready(base_url: str, api_key: str, model: str,
               proc: subprocess.Popen, log_path: str, timeout_s: float = 90.0) -> None:
    """Poll until a real embeddings request returns, retrying while the model loads."""
    deadline = time.monotonic() + timeout_s
    warmup = {"model": model, "input": "ready?", "encoding_format": "float"}
    while time.monotonic() < deadline:
        if proc.poll() is not None:
            die_with_log(proc, log_path, "server exited during startup")
        try:
            post_json(base_url, warmup, api_key)
            return
        except (OSError, RuntimeError):
            pass
        time.sleep(0.2)
    die_with_log(proc, log_path, "server did not become ready in time")


def decode_int8(value: str) -> list[int]:
    return [b - 256 if b > 127 else b for b in base64.b64decode(value)]


def cosine(a: list[int], b: list[int]) -> float:
    dot = sum(x * y for x, y in zip(a, b))
    na = math.sqrt(sum(x * x for x in a))
    nb = math.sqrt(sum(x * x for x in b))
    return dot / (na * nb) if na and nb else 0.0


def local_vectors(base_url: str, api_key: str, model: str,
                  text: str) -> tuple[list[int], list[int]]:
    common = {"model": model, "input": text}
    floats = post_json(base_url, {**common, "encoding_format": "float"}, api_key)
    encoded = post_json(base_url, {**common, "encoding_format": "base64_int8"}, api_key)

    float_values = floats["data"][0]["embedding"]
    if not isinstance(float_values, list):
        raise RuntimeError("float response did not contain an array")
    float_int8 = [round(float(x) * 128.0) for x in float_values]

    b64_value = encoded["data"][0]["embedding"]
    if not isinstance(b64_value, str):
        raise RuntimeError("base64_int8 response did not contain a string")
    return float_int8, decode_int8(b64_value)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--model-dir", required=True)
    ap.add_argument("--model", default="pplx-embed-v1-0.6b")
    ap.add_argument("--base-url", default=None,
                    help="test a running server instead of launching one")
    ap.add_argument("--api-key", default=API_KEY)
    ap.add_argument("--max-int8-diff", type=int, default=1)
    ap.add_argument("--min-cosine", type=float, default=0.9999)
    ap.add_argument("--text", action="append", dest="texts")
    args = ap.parse_args()

    from sentence_transformers import SentenceTransformer
    texts = args.texts or DEFAULT_TEXTS
    reference = SentenceTransformer(args.model_dir, trust_remote_code=True,
                                    local_files_only=True)

    proc: subprocess.Popen | None = None
    log_path: str | None = None
    base_url = args.base_url
    api_key = args.api_key

    if base_url is None:
        ensure_server()
        port = free_port()
        base_url = f"http://{HOST}:{port}"
        log = open(tempfile.mktemp(prefix="ffwd-ref-server-", suffix=".log"), "w")
        log_path = log.name
        argv = [str(SERVER_BIN), "--host", HOST, "--port", str(port),
                "--api-key", api_key, "-b", "2",
                "--model", f"{args.model}={args.model_dir}:api=perplexity"]
        proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT)
        log.close()
        wait_ready(base_url, api_key, args.model, proc, log_path)

    try:
        ok = True
        for text in texts:
            official = [round(float(x)) for x in reference.encode([text])[0]]
            local_float, local_int8 = local_vectors(base_url, api_key, args.model, text)
            if local_float != local_int8:
                raise RuntimeError("local float and base64_int8 responses disagree")
            max_diff = max(abs(a - b) for a, b in zip(local_int8, official))
            equal = sum(a == b for a, b in zip(local_int8, official))
            score = cosine(local_int8, official)
            passed = (len(local_int8) == len(official)
                      and max_diff <= args.max_int8_diff
                      and score >= args.min_cosine)
            ok = ok and passed
            print(f"{'ok' if passed else 'FAIL'}: equal={equal}/{len(official)} "
                  f"max_int8_diff={max_diff} cosine={score:.8f} text={text!r}")
    finally:
        if proc is not None:
            proc.terminate()
            try:
                proc.wait(timeout=5)
            except subprocess.TimeoutExpired:
                proc.kill()
            if log_path:
                try:
                    os.unlink(log_path)
                except OSError:
                    pass

    print(f"{'ok' if ok else 'FAIL'}: server reference parity for {len(texts)} "
          f"texts ({args.model})")
    return 0 if ok else 1


if __name__ == "__main__":
    raise SystemExit(main())