aboutsummaryrefslogtreecommitdiff
path: root/tests/check_openai_sdk_compat.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:36:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commit6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch)
tree1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/check_openai_sdk_compat.py
parentdbdf714bb57578431924e5d069089bdfd9f4c604 (diff)
downloadffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz
ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/check_openai_sdk_compat.py')
-rw-r--r--tests/check_openai_sdk_compat.py189
1 files changed, 189 insertions, 0 deletions
diff --git a/tests/check_openai_sdk_compat.py b/tests/check_openai_sdk_compat.py
new file mode 100644
index 0000000..5ba582e
--- /dev/null
+++ b/tests/check_openai_sdk_compat.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""Verify the local server through the official OpenAI Python SDK.
+
+The server's /v1/embeddings is OpenAI-shaped, so an OpenAI client should work
+against it unchanged. This builds and launches ffwd-server with a model, then
+drives client.embeddings.create through the OpenAI SDK, checking the response
+shape and a retrieval ranking. Pass --base-url to test a running server instead.
+
+The OpenAI SDK is not in requirements-reference.txt, so run with uv:
+
+ uv run --python 3.12 --with openai \
+ tests/check_openai_sdk_compat.py --model-dir DIR
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+try:
+ from openai import OpenAI
+except ImportError as e:
+ raise SystemExit(
+ "missing dependency; run with: uv run --python 3.12 --with openai "
+ "tests/check_openai_sdk_compat.py ..."
+ ) from e
+
+ROOT = Path(__file__).resolve().parents[1]
+HOST = "127.0.0.1"
+SERVER_BIN = ROOT / "tools/server/ffwd-server"
+
+DOCS = [
+ ("Capitals", "Paris is the capital of France."),
+ ("Capitals", "Berlin is the capital of Germany."),
+ ("Science", "Photosynthesis converts sunlight into chemical energy in plants."),
+ ("Databases", "Redis is an in-memory data structure server used for caching."),
+]
+QUERY = "What is the capital of France?"
+
+
+def ensure_server() -> None:
+ if not SERVER_BIN.exists():
+ subprocess.run(["make", "-C", "tools/server", "all"], cwd=ROOT, check=True,
+ stdout=subprocess.DEVNULL)
+ if not SERVER_BIN.exists():
+ sys.exit(f"ffwd-server not found and could not be built: {SERVER_BIN}")
+
+
+def free_port() -> int:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.bind((HOST, 0))
+ port = s.getsockname()[1]
+ s.close()
+ return port
+
+
+def die_with_log(proc: subprocess.Popen, log_path: str, msg: str) -> None:
+ try:
+ with open(log_path) as f:
+ tail = "".join(f.readlines()[-20:])
+ except OSError:
+ tail = "(no server log)"
+ try:
+ proc.kill()
+ except OSError:
+ pass
+ sys.exit(f"{msg}\n--- server log tail ---\n{tail}")
+
+
+def wait_ready(base_url: str, api_key: str, model: str,
+ proc: subprocess.Popen, log_path: str, timeout_s: float = 90.0) -> None:
+ deadline = time.monotonic() + timeout_s
+ body = json.dumps({"model": model, "input": "ready?",
+ "encoding_format": "float"}).encode()
+ req = urllib.request.Request(
+ base_url.rstrip("/") + "/v1/embeddings", data=body,
+ headers={"Content-Type": "application/json",
+ "Authorization": f"Bearer {api_key}"})
+ while time.monotonic() < deadline:
+ if proc.poll() is not None:
+ die_with_log(proc, log_path, "server exited during startup")
+ try:
+ with urllib.request.urlopen(req, timeout=5):
+ return
+ except (OSError, urllib.error.HTTPError):
+ pass
+ time.sleep(0.2)
+ die_with_log(proc, log_path, "server did not become ready in time")
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb) if na and nb else 0.0
+
+
+def embed(client: OpenAI, model: str, texts: list[str],
+ dimensions: int) -> list[list[float]]:
+ resp = client.embeddings.create(model=model, input=texts,
+ encoding_format="float", dimensions=dimensions)
+ if len(resp.data) != len(texts):
+ raise RuntimeError(f"got {len(resp.data)} rows, expected {len(texts)}")
+ rows = []
+ for item in sorted(resp.data, key=lambda d: d.index):
+ if not isinstance(item.embedding, list):
+ raise RuntimeError("embedding is not a float array")
+ rows.append([float(x) for x in item.embedding])
+ return rows
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--base-url", default=None,
+ help="test a running server instead of launching one")
+ ap.add_argument("--model-dir", default=None, help="model dir (for launch)")
+ ap.add_argument("--model", default="text-embedding-local")
+ ap.add_argument("--api-key", default="dummy")
+ ap.add_argument("--dimensions", type=int, default=128)
+ args = ap.parse_args()
+
+ proc: subprocess.Popen | None = None
+ log_path: str | None = None
+ base_url = args.base_url
+
+ if base_url is None:
+ if not args.model_dir:
+ raise SystemExit("--model-dir is required when launching the server")
+ ensure_server()
+ port = free_port()
+ base_url = f"http://{HOST}:{port}"
+ argv = [str(SERVER_BIN), "--host", HOST, "--port", str(port),
+ "--api-key", args.api_key, "-b", "2",
+ "--model", f"{args.model}={args.model_dir}:api=perplexity"
+ f":min_dim={args.dimensions}"]
+ log = open(tempfile.mktemp(prefix="ffwd-openai-sdk-", suffix=".log"), "w")
+ log_path = log.name
+ proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT)
+ log.close()
+ wait_ready(base_url, args.api_key, args.model, proc, log_path)
+
+ try:
+ client = OpenAI(base_url=base_url.rstrip("/") + "/v1", api_key=args.api_key)
+
+ print("standard OpenAI SDK call:")
+ doc_vecs = embed(client, args.model, [t for _, t in DOCS], args.dimensions)
+ if any(len(v) != args.dimensions for v in doc_vecs):
+ raise RuntimeError("decoded dimension mismatch")
+ q_vec = embed(client, args.model, [QUERY], args.dimensions)[0]
+
+ ranked = sorted(
+ ({"title": title, "text": text, "score": cosine(q_vec, vec)}
+ for (title, text), vec in zip(DOCS, doc_vecs)),
+ key=lambda r: r["score"], reverse=True)
+
+ print(f" dimensions={args.dimensions} rows={len(doc_vecs)}")
+ for row in ranked[:3]:
+ print(f" {row['score']: .4f} [{row['title']}] {row['text']}")
+ ok = ranked[0]["text"] == "Paris is the capital of France."
+ print(f" verdict={'ok' if ok else 'FAIL'}: expected the Paris document on top")
+ print(f"overall={'ok' if ok else 'FAIL'}")
+ finally:
+ if proc is not None:
+ proc.terminate()
+ try:
+ proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ if log_path:
+ try:
+ os.unlink(log_path)
+ except OSError:
+ pass
+
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())