1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
#!/usr/bin/env python3
"""Verify the local server through the official OpenAI Python SDK.
The server's /v1/embeddings is OpenAI-shaped, so an OpenAI client should work
against it unchanged. This builds and launches ffwd-server with a model, then
drives client.embeddings.create through the OpenAI SDK, checking the response
shape and a retrieval ranking. Pass --base-url to test a running server instead.
The OpenAI SDK is not in requirements-reference.txt, so run with uv:
uv run --python 3.12 --with openai \
tests/check_openai_sdk_compat.py --model-dir DIR
"""
from __future__ import annotations
import argparse
import json
import math
import os
import socket
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path
try:
from openai import OpenAI
except ImportError as e:
raise SystemExit(
"missing dependency; run with: uv run --python 3.12 --with openai "
"tests/check_openai_sdk_compat.py ..."
) from e
ROOT = Path(__file__).resolve().parents[1]
HOST = "127.0.0.1"
SERVER_BIN = ROOT / "ffwd-server"
DOCS = [
("Capitals", "Paris is the capital of France."),
("Capitals", "Berlin is the capital of Germany."),
("Science", "Photosynthesis converts sunlight into chemical energy in plants."),
("Databases", "Redis is an in-memory data structure server used for caching."),
]
QUERY = "What is the capital of France?"
def ensure_server() -> None:
if not SERVER_BIN.exists():
subprocess.run(["make", "cpu"], cwd=ROOT, check=True,
stdout=subprocess.DEVNULL)
if not SERVER_BIN.exists():
sys.exit(f"ffwd-server not found and could not be built: {SERVER_BIN}")
def free_port() -> int:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, 0))
port = s.getsockname()[1]
s.close()
return port
def die_with_log(proc: subprocess.Popen, log_path: str, msg: str) -> None:
try:
with open(log_path) as f:
tail = "".join(f.readlines()[-20:])
except OSError:
tail = "(no server log)"
try:
proc.kill()
except OSError:
pass
sys.exit(f"{msg}\n--- server log tail ---\n{tail}")
def wait_ready(base_url: str, api_key: str, model: str,
proc: subprocess.Popen, log_path: str, timeout_s: float = 90.0) -> None:
deadline = time.monotonic() + timeout_s
body = json.dumps({"model": model, "input": "ready?",
"encoding_format": "float"}).encode()
req = urllib.request.Request(
base_url.rstrip("/") + "/v1/embeddings", data=body,
headers={"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}"})
while time.monotonic() < deadline:
if proc.poll() is not None:
die_with_log(proc, log_path, "server exited during startup")
try:
with urllib.request.urlopen(req, timeout=5):
return
except (OSError, urllib.error.HTTPError):
pass
time.sleep(0.2)
die_with_log(proc, log_path, "server did not become ready in time")
def cosine(a: list[float], b: list[float]) -> float:
dot = sum(x * y for x, y in zip(a, b))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
return dot / (na * nb) if na and nb else 0.0
def embed(client: OpenAI, model: str, texts: list[str],
dimensions: int) -> list[list[float]]:
resp = client.embeddings.create(model=model, input=texts,
encoding_format="float", dimensions=dimensions)
if len(resp.data) != len(texts):
raise RuntimeError(f"got {len(resp.data)} rows, expected {len(texts)}")
rows = []
for item in sorted(resp.data, key=lambda d: d.index):
if not isinstance(item.embedding, list):
raise RuntimeError("embedding is not a float array")
rows.append([float(x) for x in item.embedding])
return rows
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--base-url", default=None,
help="test a running server instead of launching one")
ap.add_argument("--model-dir", default=None, help="model dir (for launch)")
ap.add_argument("--model", default="text-embedding-local")
ap.add_argument("--api-key", default="dummy")
ap.add_argument("--dimensions", type=int, default=128)
args = ap.parse_args()
proc: subprocess.Popen | None = None
log_path: str | None = None
base_url = args.base_url
if base_url is None:
if not args.model_dir:
raise SystemExit("--model-dir is required when launching the server")
ensure_server()
port = free_port()
base_url = f"http://{HOST}:{port}"
argv = [str(SERVER_BIN), "--host", HOST, "--port", str(port),
"--api-key", args.api_key, "-b", "2",
"--model", f"{args.model}={args.model_dir}:api=perplexity"
f":min_dim={args.dimensions}"]
log = open(tempfile.mktemp(prefix="ffwd-openai-sdk-", suffix=".log"), "w")
log_path = log.name
proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT)
log.close()
wait_ready(base_url, args.api_key, args.model, proc, log_path)
try:
client = OpenAI(base_url=base_url.rstrip("/") + "/v1", api_key=args.api_key)
print("standard OpenAI SDK call:")
doc_vecs = embed(client, args.model, [t for _, t in DOCS], args.dimensions)
if any(len(v) != args.dimensions for v in doc_vecs):
raise RuntimeError("decoded dimension mismatch")
q_vec = embed(client, args.model, [QUERY], args.dimensions)[0]
ranked = sorted(
({"title": title, "text": text, "score": cosine(q_vec, vec)}
for (title, text), vec in zip(DOCS, doc_vecs)),
key=lambda r: r["score"], reverse=True)
print(f" dimensions={args.dimensions} rows={len(doc_vecs)}")
for row in ranked[:3]:
print(f" {row['score']: .4f} [{row['title']}] {row['text']}")
ok = ranked[0]["text"] == "Paris is the capital of France."
print(f" verdict={'ok' if ok else 'FAIL'}: expected the Paris document on top")
print(f"overall={'ok' if ok else 'FAIL'}")
finally:
if proc is not None:
proc.terminate()
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
proc.kill()
if log_path:
try:
os.unlink(log_path)
except OSError:
pass
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(main())
|