diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:36:55 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch) | |
| tree | 1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/check_openai_sdk_compat.py | |
| parent | dbdf714bb57578431924e5d069089bdfd9f4c604 (diff) | |
| download | ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip | |
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/check_openai_sdk_compat.py')
| -rw-r--r-- | tests/check_openai_sdk_compat.py | 189 |
1 files changed, 189 insertions, 0 deletions
diff --git a/tests/check_openai_sdk_compat.py b/tests/check_openai_sdk_compat.py new file mode 100644 index 0000000..5ba582e --- /dev/null +++ b/tests/check_openai_sdk_compat.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Verify the local server through the official OpenAI Python SDK. + +The server's /v1/embeddings is OpenAI-shaped, so an OpenAI client should work +against it unchanged. This builds and launches ffwd-server with a model, then +drives client.embeddings.create through the OpenAI SDK, checking the response +shape and a retrieval ranking. Pass --base-url to test a running server instead. + +The OpenAI SDK is not in requirements-reference.txt, so run with uv: + + uv run --python 3.12 --with openai \ + tests/check_openai_sdk_compat.py --model-dir DIR +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +try: + from openai import OpenAI +except ImportError as e: + raise SystemExit( + "missing dependency; run with: uv run --python 3.12 --with openai " + "tests/check_openai_sdk_compat.py ..." + ) from e + +ROOT = Path(__file__).resolve().parents[1] +HOST = "127.0.0.1" +SERVER_BIN = ROOT / "tools/server/ffwd-server" + +DOCS = [ + ("Capitals", "Paris is the capital of France."), + ("Capitals", "Berlin is the capital of Germany."), + ("Science", "Photosynthesis converts sunlight into chemical energy in plants."), + ("Databases", "Redis is an in-memory data structure server used for caching."), +] +QUERY = "What is the capital of France?" + + +def ensure_server() -> None: + if not SERVER_BIN.exists(): + subprocess.run(["make", "-C", "tools/server", "all"], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL) + if not SERVER_BIN.exists(): + sys.exit(f"ffwd-server not found and could not be built: {SERVER_BIN}") + + +def free_port() -> int: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind((HOST, 0)) + port = s.getsockname()[1] + s.close() + return port + + +def die_with_log(proc: subprocess.Popen, log_path: str, msg: str) -> None: + try: + with open(log_path) as f: + tail = "".join(f.readlines()[-20:]) + except OSError: + tail = "(no server log)" + try: + proc.kill() + except OSError: + pass + sys.exit(f"{msg}\n--- server log tail ---\n{tail}") + + +def wait_ready(base_url: str, api_key: str, model: str, + proc: subprocess.Popen, log_path: str, timeout_s: float = 90.0) -> None: + deadline = time.monotonic() + timeout_s + body = json.dumps({"model": model, "input": "ready?", + "encoding_format": "float"}).encode() + req = urllib.request.Request( + base_url.rstrip("/") + "/v1/embeddings", data=body, + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {api_key}"}) + while time.monotonic() < deadline: + if proc.poll() is not None: + die_with_log(proc, log_path, "server exited during startup") + try: + with urllib.request.urlopen(req, timeout=5): + return + except (OSError, urllib.error.HTTPError): + pass + time.sleep(0.2) + die_with_log(proc, log_path, "server did not become ready in time") + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) if na and nb else 0.0 + + +def embed(client: OpenAI, model: str, texts: list[str], + dimensions: int) -> list[list[float]]: + resp = client.embeddings.create(model=model, input=texts, + encoding_format="float", dimensions=dimensions) + if len(resp.data) != len(texts): + raise RuntimeError(f"got {len(resp.data)} rows, expected {len(texts)}") + rows = [] + for item in sorted(resp.data, key=lambda d: d.index): + if not isinstance(item.embedding, list): + raise RuntimeError("embedding is not a float array") + rows.append([float(x) for x in item.embedding]) + return rows + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--base-url", default=None, + help="test a running server instead of launching one") + ap.add_argument("--model-dir", default=None, help="model dir (for launch)") + ap.add_argument("--model", default="text-embedding-local") + ap.add_argument("--api-key", default="dummy") + ap.add_argument("--dimensions", type=int, default=128) + args = ap.parse_args() + + proc: subprocess.Popen | None = None + log_path: str | None = None + base_url = args.base_url + + if base_url is None: + if not args.model_dir: + raise SystemExit("--model-dir is required when launching the server") + ensure_server() + port = free_port() + base_url = f"http://{HOST}:{port}" + argv = [str(SERVER_BIN), "--host", HOST, "--port", str(port), + "--api-key", args.api_key, "-b", "2", + "--model", f"{args.model}={args.model_dir}:api=perplexity" + f":min_dim={args.dimensions}"] + log = open(tempfile.mktemp(prefix="ffwd-openai-sdk-", suffix=".log"), "w") + log_path = log.name + proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT) + log.close() + wait_ready(base_url, args.api_key, args.model, proc, log_path) + + try: + client = OpenAI(base_url=base_url.rstrip("/") + "/v1", api_key=args.api_key) + + print("standard OpenAI SDK call:") + doc_vecs = embed(client, args.model, [t for _, t in DOCS], args.dimensions) + if any(len(v) != args.dimensions for v in doc_vecs): + raise RuntimeError("decoded dimension mismatch") + q_vec = embed(client, args.model, [QUERY], args.dimensions)[0] + + ranked = sorted( + ({"title": title, "text": text, "score": cosine(q_vec, vec)} + for (title, text), vec in zip(DOCS, doc_vecs)), + key=lambda r: r["score"], reverse=True) + + print(f" dimensions={args.dimensions} rows={len(doc_vecs)}") + for row in ranked[:3]: + print(f" {row['score']: .4f} [{row['title']}] {row['text']}") + ok = ranked[0]["text"] == "Paris is the capital of France." + print(f" verdict={'ok' if ok else 'FAIL'}: expected the Paris document on top") + print(f"overall={'ok' if ok else 'FAIL'}") + finally: + if proc is not None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + if log_path: + try: + os.unlink(log_path) + except OSError: + pass + + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) |
