aboutsummaryrefslogtreecommitdiff
path: root/devtools
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xdevtools/compare_model_embeddings.py130
-rwxr-xr-xdevtools/compare_reference_backends.py644
-rwxr-xr-xdevtools/convert_model_bf16.py (renamed from tools/convert_model_bf16.py)0
-rwxr-xr-xdevtools/gen_exports.py91
-rw-r--r--devtools/gen_test_vectors.py209
-rwxr-xr-x[-rw-r--r--]devtools/gen_wordpiece_unicode.py (renamed from tools/gen_wordpiece_unicode.py)6
-rwxr-xr-xdevtools/inspect_model_artifacts.py260
-rw-r--r--devtools/openapi.yaml (renamed from tools/openapi.yaml)8
-rwxr-xr-xdevtools/quant_precision_drift.py197
-rwxr-xr-xdevtools/rank_embeddings.py (renamed from tools/rank_embeddings.py)2
10 files changed, 1539 insertions, 8 deletions
diff --git a/devtools/compare_model_embeddings.py b/devtools/compare_model_embeddings.py
new file mode 100755
index 0000000..95154a2
--- /dev/null
+++ b/devtools/compare_model_embeddings.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Compare embeddings from two model directories through the CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import subprocess
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
+
+
+def ensure_cli(binary: Path) -> None:
+ """Build the CLI if the default binary is missing."""
+ if binary == DEFAULT_BINARY and not binary.exists():
+ subprocess.run(["make", "-C", "tools/cli", "all"], cwd=ROOT, check=True,
+ stdout=subprocess.DEVNULL)
+
+
+TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "The quick brown fox jumps over the lazy dog near the riverbank",
+ (
+ "Retrieval augmented generation combines embedding models with external "
+ "knowledge sources and ranks documents by semantic similarity."
+ ),
+]
+
+
+def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int,
+ backend: str, threads: int) -> list[list[float]]:
+ cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
+ if backend == "cpu" and threads > 0:
+ cmd.extend(["-t", str(threads)])
+
+ proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ )
+ assert proc.stdin is not None
+ assert proc.stdout is not None
+ proc.stdin.write("\n".join(texts) + "\n")
+ proc.stdin.close()
+
+ embeddings = []
+ for _ in texts:
+ line = proc.stdout.readline()
+ if not line:
+ stderr = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}")
+ row = json.loads(line)
+ if "error" in row:
+ raise RuntimeError(f"stdin error for {model_dir}: {row}")
+ embeddings.append([float(x) for x in row["embedding"]])
+
+ rc = proc.wait(timeout=10)
+ if rc != 0:
+ stderr = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}")
+ return embeddings
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--binary", default=str(DEFAULT_BINARY))
+ ap.add_argument("--a", required=True, help="reference model directory")
+ ap.add_argument("--b", required=True, help="comparison model directory")
+ ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu")
+ ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"],
+ help="backend for --a; defaults to --backend")
+ ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"],
+ help="backend for --b; defaults to --backend")
+ ap.add_argument("--batch-size", type=int, default=4)
+ ap.add_argument("--threads", type=int, default=0)
+ ap.add_argument("--max-diff", type=float, default=0.01)
+ ap.add_argument("--min-cos", type=float, default=0.999)
+ args = ap.parse_args()
+
+ if args.batch_size <= 0:
+ raise SystemExit("--batch-size must be > 0")
+ ensure_cli(Path(args.binary))
+ if not Path(args.binary).exists():
+ raise SystemExit(f"binary not found: {args.binary}")
+
+ backend_a = args.backend_a or args.backend
+ backend_b = args.backend_b or args.backend
+ emb_a = run_model(args.binary, args.a, TEXTS, args.batch_size,
+ backend_a, args.threads)
+ emb_b = run_model(args.binary, args.b, TEXTS, args.batch_size,
+ backend_b, args.threads)
+
+ worst_diff = 0.0
+ worst_cos = 1.0
+ for i, (a, b) in enumerate(zip(emb_a, emb_b)):
+ if len(a) != len(b):
+ raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}")
+ max_diff = max(abs(x - y) for x, y in zip(a, b))
+ cos = cosine(a, b)
+ worst_diff = max(worst_diff, max_diff)
+ worst_cos = min(worst_cos, cos)
+ print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}")
+
+ print(
+ f"ok: backend_a={backend_a} backend_b={backend_b} "
+ f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}"
+ )
+ if worst_diff > args.max_diff or worst_cos < args.min_cos:
+ raise SystemExit(
+ f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/devtools/compare_reference_backends.py b/devtools/compare_reference_backends.py
new file mode 100755
index 0000000..0f6d758
--- /dev/null
+++ b/devtools/compare_reference_backends.py
@@ -0,0 +1,644 @@
+#!/usr/bin/env python3
+"""Compare ffwd output with offline reference implementations.
+
+Benchmarks the native CLI (stdin) or server (http) against sentence-transformers
+and ONNX, reporting cosine parity, latency, and peak RSS. Needs the reference
+stack, so run it with uv (Python 3.12 for the numpy pin):
+
+ uv run --python 3.12 --with-requirements requirements-reference.txt \
+ devtools/compare_reference_backends.py --model-dir DIR \
+ --native-mode stdin --native-backend cpu --references sentence-transformers
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import math
+import os
+import signal
+import statistics
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any, Callable
+
+
+DEFAULT_STANDARD = [
+ "Scientists explore the universe driven by curiosity.",
+ "Children learn through curious exploration.",
+ "Historical discoveries began with curious questions.",
+ "Redis is an in-memory data structure server often used for caching.",
+ "SQLite stores relational data in a single local database file.",
+]
+
+DEFAULT_CONTEXTUAL = [
+ [
+ "Paris is the capital city of France.",
+ "The Eiffel Tower is a landmark in Paris.",
+ "France is a country in Western Europe.",
+ ],
+ [
+ "Redis stores data in memory.",
+ "SQLite stores relational data in a local file.",
+ ],
+]
+
+RESULT_PREFIX = "EMBED_REFERENCE_RESULT="
+
+
+def repo_root() -> Path:
+ return Path(__file__).resolve().parents[1]
+
+
+def percentile(values: list[float], pct: float) -> float:
+ ordered = sorted(values)
+ pos = max(0, math.ceil((pct / 100.0) * len(ordered)) - 1)
+ return ordered[pos]
+
+
+def cosine(a: list[int], b: list[int]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ aa = sum(x * x for x in a)
+ bb = sum(x * x for x in b)
+ return dot / math.sqrt(aa * bb) if aa and bb else 0.0
+
+
+def int8_rows(value: Any) -> list[list[int]]:
+ rows = value.tolist() if hasattr(value, "tolist") else value
+ return [[int(round(float(x))) for x in row] for row in rows]
+
+
+def flatten_contextual(value: Any) -> list[list[int]]:
+ rows: list[list[int]] = []
+ for document in value:
+ rows.extend(int8_rows(document))
+ return rows
+
+
+def labels(kind: str, inputs: Any) -> list[str]:
+ if kind == "standard":
+ return [f"text[{i}]" for i in range(len(inputs))]
+ return [
+ f"doc[{di}]/chunk[{ci}]"
+ for di, document in enumerate(inputs)
+ for ci in range(len(document))
+ ]
+
+
+def offline_environment(cache_dir: Path) -> dict[str, str]:
+ cache_dir.mkdir(parents=True, exist_ok=True)
+ env = os.environ.copy()
+ env.update(
+ {
+ "HF_HOME": str(cache_dir),
+ "HF_HUB_CACHE": str(cache_dir / "hub"),
+ "HF_MODULES_CACHE": str(cache_dir / "modules"),
+ "TRANSFORMERS_CACHE": str(cache_dir / "transformers"),
+ "HF_HUB_OFFLINE": "1",
+ "TRANSFORMERS_OFFLINE": "1",
+ "HF_DATASETS_OFFLINE": "1",
+ "TOKENIZERS_PARALLELISM": "false",
+ }
+ )
+ return env
+
+
+def rss_mb(pid: int | None) -> float | None:
+ if not pid:
+ return None
+ proc = subprocess.run(
+ ["ps", "-o", "rss=", "-p", str(pid)],
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
+ check=False,
+ )
+ try:
+ return int(proc.stdout.strip()) / 1024.0
+ except ValueError:
+ return None
+
+
+def post_json(url: str, payload: dict[str, Any], timeout: float) -> Any:
+ request = urllib.request.Request(
+ url,
+ data=json.dumps(payload).encode("utf-8"),
+ headers={"Content-Type": "application/json"},
+ )
+ try:
+ with urllib.request.urlopen(request, timeout=timeout) as response:
+ return json.loads(response.read().decode("utf-8"))
+ except urllib.error.HTTPError as exc:
+ body = exc.read().decode("utf-8", errors="replace")
+ raise RuntimeError(f"HTTP {exc.code}: {body}") from exc
+ except urllib.error.URLError as exc:
+ raise RuntimeError(f"request failed for {url}: {exc.reason}") from exc
+
+
+def decode_native(response: dict[str, Any], kind: str) -> list[list[int]]:
+ if kind == "standard":
+ entries = response["data"]
+ else:
+ entries = [chunk for document in response["data"] for chunk in document["data"]]
+ rows = []
+ for entry in entries:
+ encoded = entry["embedding"]
+ raw = base64.b64decode(encoded)
+ rows.append([x - 256 if x > 127 else x for x in raw])
+ return rows
+
+
+def quantize_native(rows: list[dict[str, Any]]) -> list[list[int]]:
+ vectors = []
+ for row in rows:
+ if "error" in row:
+ raise RuntimeError(f"native stdin returned an error: {row['error']}")
+ vectors.append(
+ [
+ max(-128, min(127, round(math.tanh(float(x)) * 127.0)))
+ for x in row["embedding"]
+ ]
+ )
+ return vectors
+
+
+class RssGuard:
+ def __init__(self, proc: subprocess.Popen[str], ceiling_mb: float):
+ self.proc = proc
+ self.ceiling_mb = ceiling_mb
+ self.peak_mb = 0.0
+ self.error: str | None = None
+ self.stop_event = threading.Event()
+ self.thread = threading.Thread(target=self.run, daemon=True)
+
+ def start(self) -> None:
+ self.thread.start()
+
+ def run(self) -> None:
+ while not self.stop_event.wait(0.025):
+ current = rss_mb(self.proc.pid) or 0.0
+ self.peak_mb = max(self.peak_mb, current)
+ if current > self.ceiling_mb:
+ self.error = (
+ f"native stdin exceeded RSS ceiling: "
+ f"{current:.1f} > {self.ceiling_mb} MiB"
+ )
+ self.proc.kill()
+ return
+ if self.proc.poll() is not None:
+ return
+
+ def stop(self) -> None:
+ self.stop_event.set()
+ self.thread.join()
+ self.peak_mb = max(self.peak_mb, rss_mb(self.proc.pid) or 0.0)
+
+
+def benchmark_native_stdin(args: argparse.Namespace) -> dict[str, Any]:
+ if args.kind != "standard":
+ raise RuntimeError("native stdin comparison currently supports standard embeddings only")
+ cmd = [
+ args.native_binary,
+ "-d",
+ args.model_dir,
+ "--stream",
+ "-b",
+ str(len(args.inputs)),
+ ]
+ if args.native_backend == "cpu":
+ cmd.extend(["-t", str(args.threads)])
+ proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ bufsize=1,
+ )
+ guard = RssGuard(proc, args.max_native_rss_mb)
+ guard.start()
+
+ def send() -> tuple[list[list[int]], float]:
+ if not proc.stdin or not proc.stdout:
+ raise RuntimeError("native stdin pipes are unavailable")
+ proc.stdin.write("\n".join(args.inputs) + "\n")
+ proc.stdin.flush()
+ rows = []
+ for _ in args.inputs:
+ line = proc.stdout.readline()
+ if not line:
+ errors = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(guard.error or f"native stdin exited early:\n{errors}")
+ rows.append(json.loads(line))
+ return quantize_native(rows), float(rows[0]["ms"])
+
+ try:
+ start = time.perf_counter()
+ vectors, _ = send()
+ load_ms = (time.perf_counter() - start) * 1000.0
+ for _ in range(args.warmups):
+ vectors, _ = send()
+ elapsed = []
+ for _ in range(args.runs):
+ vectors, engine_ms = send()
+ elapsed.append(engine_ms)
+ finally:
+ if proc.stdin:
+ proc.stdin.close()
+ try:
+ proc.wait(timeout=10)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ proc.wait()
+ guard.stop()
+ if guard.error:
+ raise RuntimeError(guard.error)
+ if proc.returncode != 0:
+ errors = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(f"native stdin failed ({proc.returncode}):\n{errors}")
+ return {
+ "name": f"native-stdin-{args.native_backend}",
+ "vectors": vectors,
+ "load_ms": load_ms,
+ "p50_ms": statistics.median(elapsed),
+ "p95_ms": percentile(elapsed, 95.0),
+ "rss_mb": guard.peak_mb,
+ }
+
+
+def benchmark_http(
+ name: str,
+ url: str,
+ payload: dict[str, Any],
+ decode: Callable[[Any], list[list[int]]],
+ warmups: int,
+ runs: int,
+ timeout: float,
+ pid: int | None,
+) -> dict[str, Any]:
+ for _ in range(warmups):
+ decode(post_json(url, payload, timeout))
+ elapsed = []
+ vectors = None
+ peak_rss = rss_mb(pid)
+ for _ in range(runs):
+ start = time.perf_counter()
+ vectors = decode(post_json(url, payload, timeout))
+ elapsed.append((time.perf_counter() - start) * 1000.0)
+ current = rss_mb(pid)
+ peak_rss = max(peak_rss or 0.0, current or 0.0) or None
+ return {
+ "name": name,
+ "vectors": vectors,
+ "load_ms": None,
+ "p50_ms": statistics.median(elapsed),
+ "p95_ms": percentile(elapsed, 95.0),
+ "rss_mb": peak_rss,
+ }
+
+
+def validate_artifacts(model_dir: Path, require_onnx: bool) -> None:
+ model_weights = model_dir / "model.safetensors"
+ model_index = model_dir / "model.safetensors.index.json"
+ required = [
+ model_dir / "config.json",
+ model_dir / "tokenizer.json",
+ ]
+ if require_onnx:
+ required.append(model_dir / "onnx" / "model.onnx")
+ missing = [str(path) for path in required if not path.exists()]
+ if not model_weights.exists() and not model_index.exists():
+ missing.append(f"{model_weights} or {model_index}")
+ if require_onnx and not list((model_dir / "onnx").glob("model.onnx_data*")):
+ missing.append(str(model_dir / "onnx" / "model.onnx_data*"))
+ if missing:
+ raise SystemExit("missing local model artifacts:\n " + "\n ".join(missing))
+
+
+def run_guarded_worker(args: argparse.Namespace, backend: str) -> dict[str, Any]:
+ cache_dir = Path(args.cache_dir).expanduser().resolve()
+ env = offline_environment(cache_dir)
+ cmd = [
+ sys.executable,
+ str(Path(__file__).resolve()),
+ "--worker",
+ "--reference",
+ backend,
+ "--kind",
+ args.kind,
+ "--model-dir",
+ str(Path(args.model_dir).expanduser().resolve()),
+ "--runs",
+ str(args.runs),
+ "--warmups",
+ str(args.warmups),
+ "--threads",
+ str(args.threads),
+ "--input-json",
+ json.dumps(args.inputs),
+ ]
+ with tempfile.TemporaryFile(mode="w+") as stdout, tempfile.TemporaryFile(
+ mode="w+"
+ ) as stderr:
+ proc = subprocess.Popen(
+ cmd,
+ env=env,
+ stdin=subprocess.DEVNULL,
+ stdout=stdout,
+ stderr=stderr,
+ text=True,
+ )
+ start = time.monotonic()
+ peak = 0.0
+ error = None
+ while proc.poll() is None:
+ current = rss_mb(proc.pid) or 0.0
+ peak = max(peak, current)
+ if peak > args.max_reference_rss_mb:
+ error = (
+ f"{backend} exceeded RSS ceiling: "
+ f"{peak:.1f} > {args.max_reference_rss_mb} MiB"
+ )
+ proc.send_signal(signal.SIGKILL)
+ break
+ if time.monotonic() - start > args.timeout:
+ error = f"{backend} exceeded timeout: {args.timeout:.1f}s"
+ proc.send_signal(signal.SIGKILL)
+ break
+ time.sleep(0.025)
+ proc.wait()
+ stdout.seek(0)
+ stderr.seek(0)
+ output = stdout.read()
+ errors = stderr.read()
+ if error:
+ raise RuntimeError(error)
+ if proc.returncode != 0:
+ raise RuntimeError(f"{backend} failed ({proc.returncode}):\n{errors or output}")
+ marker = output.rfind(RESULT_PREFIX)
+ if marker >= 0:
+ result = json.loads(output[marker + len(RESULT_PREFIX) :].strip())
+ result["rss_mb"] = max(float(result["rss_mb"]), peak)
+ return result
+ raise RuntimeError(f"{backend} did not emit a result:\n{errors or output}")
+
+
+def worker_sentencetransformers(args: argparse.Namespace, inputs: Any) -> Any:
+ import torch
+
+ torch.set_num_threads(args.threads)
+ if args.kind == "standard":
+ from sentence_transformers import SentenceTransformer
+
+ model = SentenceTransformer(
+ args.model_dir, trust_remote_code=True, local_files_only=True
+ )
+ return model, lambda: int8_rows(model.encode(inputs, show_progress_bar=False))
+
+ from transformers import AutoModel
+
+ model = AutoModel.from_pretrained(
+ args.model_dir, trust_remote_code=True, local_files_only=True
+ )
+ return model, lambda: flatten_contextual(model.encode(inputs, show_progress_bar=False))
+
+
+def worker_onnx(args: argparse.Namespace, inputs: Any) -> Any:
+ import numpy as np
+ import onnxruntime as ort
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(
+ args.model_dir, trust_remote_code=True, local_files_only=True
+ )
+ options = ort.SessionOptions()
+ options.intra_op_num_threads = args.threads
+ session = ort.InferenceSession(
+ str(Path(args.model_dir) / "onnx" / "model.onnx"),
+ sess_options=options,
+ providers=["CPUExecutionProvider"],
+ )
+ texts = inputs if args.kind == "standard" else [tokenizer.sep_token.join(x) for x in inputs]
+ tokenized = tokenizer(texts, padding=True, truncation=True, return_tensors="np")
+ ort_inputs = {
+ "input_ids": tokenized["input_ids"].astype(np.int64),
+ "attention_mask": tokenized["attention_mask"].astype(np.int64),
+ }
+
+ def encode() -> list[list[int]]:
+ outputs = session.run(None, ort_inputs)
+ if args.kind == "standard":
+ return int8_rows(outputs[2])
+ rows = []
+ for input_ids, mask, hidden in zip(
+ ort_inputs["input_ids"], ort_inputs["attention_mask"], outputs[0]
+ ):
+ start = 0
+ end = int(mask.sum())
+ for sep in np.flatnonzero((input_ids == tokenizer.sep_token_id) & mask.astype(bool)):
+ rows.append(int8_rows([np.rint(np.tanh(hidden[start:sep].mean(axis=0)) * 127)])[0])
+ start = int(sep) + 1
+ rows.append(int8_rows([np.rint(np.tanh(hidden[start:end].mean(axis=0)) * 127)])[0])
+ return rows
+
+ return session, encode
+
+
+def self_max_rss_mb() -> float:
+ import resource
+
+ value = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
+ return value / (1024.0 * 1024.0) if sys.platform == "darwin" else value / 1024.0
+
+
+def worker_main(args: argparse.Namespace) -> int:
+ inputs = json.loads(args.input_json)
+ start = time.perf_counter()
+ if args.reference == "sentence-transformers":
+ owner, encode = worker_sentencetransformers(args, inputs)
+ elif args.reference == "onnx":
+ owner, encode = worker_onnx(args, inputs)
+ else:
+ raise SystemExit(f"unsupported reference: {args.reference}")
+ load_ms = (time.perf_counter() - start) * 1000.0
+ for _ in range(args.warmups):
+ encode()
+ vectors = None
+ elapsed = []
+ for _ in range(args.runs):
+ start = time.perf_counter()
+ vectors = encode()
+ elapsed.append((time.perf_counter() - start) * 1000.0)
+ del owner
+ result = {
+ "name": args.reference,
+ "vectors": vectors,
+ "load_ms": load_ms,
+ "p50_ms": statistics.median(elapsed),
+ "p95_ms": percentile(elapsed, 95.0),
+ "rss_mb": self_max_rss_mb(),
+ }
+ print(RESULT_PREFIX + json.dumps(result, separators=(",", ":")))
+ return 0
+
+
+def print_matrix(name: str, vectors: list[list[int]], row_labels: list[str]) -> None:
+ print(f"\n{name} cosine similarity matrix")
+ print(" " * 16 + " ".join(f"{label:>14}" for label in row_labels))
+ for label, row in zip(row_labels, vectors):
+ scores = " ".join(f"{cosine(row, other):14.6f}" for other in vectors)
+ print(f"{label:<16}{scores}")
+
+
+def print_summary(results: list[dict[str, Any]], row_labels: list[str]) -> None:
+ print("\nPerformance")
+ print(f"{'backend':<24} {'load ms':>10} {'p50 ms':>10} {'p95 ms':>10} {'peak RSS MiB':>14}")
+ for result in results:
+ load = "-" if result["load_ms"] is None else f"{result['load_ms']:.1f}"
+ rss = "-" if result["rss_mb"] is None else f"{result['rss_mb']:.1f}"
+ print(
+ f"{result['name']:<24} {load:>10} {result['p50_ms']:>10.1f} "
+ f"{result['p95_ms']:>10.1f} {rss:>14}"
+ )
+
+ print("\nParity")
+ baseline = results[0]
+ for result in results[1:]:
+ if len(result["vectors"]) != len(baseline["vectors"]):
+ raise RuntimeError(
+ f"{result['name']} returned {len(result['vectors'])} vectors; "
+ f"expected {len(baseline['vectors'])}"
+ )
+ pairs = list(zip(baseline["vectors"], result["vectors"]))
+ if any(len(a) != len(b) for a, b in pairs):
+ raise RuntimeError(f"{result['name']} returned an unexpected dimension")
+ scores = [cosine(a, b) for a, b in pairs]
+ diffs = [max(abs(x - y) for x, y in zip(a, b)) for a, b in pairs]
+ print(
+ f"{baseline['name']} vs {result['name']}: "
+ f"min_cosine={min(scores):.8f} mean_cosine={statistics.mean(scores):.8f} "
+ f"max_int8_diff={max(diffs)}"
+ )
+
+
+def load_report(path: str, kind: str, inputs: Any) -> list[dict[str, Any]]:
+ report = json.loads(Path(path).read_text())
+ if report.get("kind") != kind or report.get("inputs") != inputs:
+ raise RuntimeError("baseline JSON does not match the selected kind and inputs")
+ results = report.get("results")
+ if not isinstance(results, list) or not results:
+ raise RuntimeError("baseline JSON does not contain results")
+ return results
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--model", default="pplx-embed-v1-0.6b")
+ ap.add_argument("--kind", choices=("standard", "contextual"), default="standard")
+ ap.add_argument("--base-url", default="http://127.0.0.1:8000")
+ ap.add_argument("--native-mode", choices=("http", "stdin", "none"), default="http")
+ ap.add_argument("--native-backend", choices=("cpu", "mlx"), default="mlx")
+ ap.add_argument("--native-binary", default=str(repo_root() / "tools/cli/ffwd-cli"))
+ ap.add_argument("--max-native-rss-mb", type=float, default=32768.0)
+ ap.add_argument("--skip-native", action="store_true")
+ ap.add_argument("--server-pid", type=int)
+ ap.add_argument("--tei-url", help="optional TEI base URL; standard mode only")
+ ap.add_argument("--tei-pid", type=int)
+ ap.add_argument("--references", default="sentence-transformers,onnx")
+ ap.add_argument("--runs", type=int, default=3)
+ ap.add_argument("--warmups", type=int, default=1)
+ ap.add_argument("--threads", type=int, default=4)
+ ap.add_argument("--timeout", type=float, default=300.0)
+ ap.add_argument("--max-reference-rss-mb", type=float, default=6144.0)
+ ap.add_argument(
+ "--cache-dir", default=str(repo_root() / ".venv" / "reference-cache")
+ )
+ ap.add_argument("--input-json")
+ ap.add_argument("--baseline-json", help="prepend results from an earlier sequential run")
+ ap.add_argument("--json-out", help="write inputs and measured results for a later run")
+ ap.add_argument("--worker", action="store_true", help=argparse.SUPPRESS)
+ ap.add_argument("--reference", help=argparse.SUPPRESS)
+ args = ap.parse_args()
+
+ if args.runs < 1 or args.warmups < 0 or args.threads < 1:
+ raise SystemExit("runs and threads must be positive; warmups must be non-negative")
+ if args.worker:
+ return worker_main(args)
+ if args.tei_url and args.kind != "standard":
+ raise SystemExit("TEI comparison currently supports --kind standard only")
+ if args.skip_native:
+ args.native_mode = "none"
+
+ args.inputs = json.loads(args.input_json) if args.input_json else (
+ DEFAULT_STANDARD if args.kind == "standard" else DEFAULT_CONTEXTUAL
+ )
+ references = [x.strip() for x in args.references.split(",") if x.strip()]
+ validate_artifacts(Path(args.model_dir).expanduser().resolve(), "onnx" in references)
+ row_labels = labels(args.kind, args.inputs)
+ results = load_report(args.baseline_json, args.kind, args.inputs) if args.baseline_json else []
+
+ if args.native_mode == "stdin":
+ results.append(benchmark_native_stdin(args))
+ elif args.native_mode == "http":
+ endpoint = "/v1/embeddings" if args.kind == "standard" else "/v1/contextualizedembeddings"
+ results.append(
+ benchmark_http(
+ "native-http",
+ args.base_url.rstrip("/") + endpoint,
+ {"model": args.model, "input": args.inputs},
+ lambda response: decode_native(response, args.kind),
+ args.warmups,
+ args.runs,
+ args.timeout,
+ args.server_pid,
+ )
+ )
+ for backend in references:
+ if backend not in ("sentence-transformers", "onnx"):
+ raise SystemExit(f"unsupported reference backend: {backend}")
+ results.append(run_guarded_worker(args, backend))
+ if args.tei_url:
+ results.append(
+ benchmark_http(
+ "tei-http",
+ args.tei_url.rstrip("/") + "/embed",
+ {"inputs": args.inputs, "normalize": False},
+ int8_rows,
+ args.warmups,
+ args.runs,
+ args.timeout,
+ args.tei_pid,
+ )
+ )
+ if not results:
+ raise SystemExit("select at least one implementation to measure")
+
+ print(f"kind={args.kind} vectors={len(row_labels)} runs={args.runs} warmups={args.warmups}")
+ for result in results:
+ print_matrix(result["name"], result["vectors"], row_labels)
+ print_summary(results, row_labels)
+ if args.json_out:
+ Path(args.json_out).write_text(
+ json.dumps(
+ {"kind": args.kind, "inputs": args.inputs, "results": results},
+ indent=2,
+ )
+ + "\n"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except RuntimeError as exc:
+ print(f"error: {exc}", file=sys.stderr)
+ raise SystemExit(1)
diff --git a/tools/convert_model_bf16.py b/devtools/convert_model_bf16.py
index d59ccf2..d59ccf2 100755
--- a/tools/convert_model_bf16.py
+++ b/devtools/convert_model_bf16.py
diff --git a/devtools/gen_exports.py b/devtools/gen_exports.py
new file mode 100755
index 0000000..215c45e
--- /dev/null
+++ b/devtools/gen_exports.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Generate the public-ABI linker export map for libffwd from the header.
+
+The shared library is built with -fvisibility=hidden, so the only symbols that
+leave it are the declarations marked FFWD_API in libffwd/ffwd.h. This script
+reads those annotations and writes the two linker inputs the build consumes:
+
+ ffwd.exports Mach-O (-Wl,-exported_symbols_list): one _symbol per line
+ ffwd.map ELF (-Wl,--version-script): FFWD { global: ...; local: *; }
+
+Run it as a release step when the libffwd public API changes:
+
+ python3 devtools/gen_exports.py # write ffwd.exports and ffwd.map
+ python3 devtools/gen_exports.py --check # verify they are in sync (CI), no write
+ python3 devtools/gen_exports.py --outdir D # write somewhere else
+"""
+import argparse
+import os
+import re
+import sys
+
+
+def find_symbols(header_text):
+ # Drop comments and preprocessor lines so the FFWD_API macro definition
+ # (#ifndef/#define FFWD_API ...) and doc comments are not mistaken for
+ # declarations.
+ text = re.sub(r"/\*.*?\*/", "", header_text, flags=re.S)
+ text = re.sub(r"//[^\n]*", "", text)
+ text = "\n".join(ln for ln in text.splitlines() if not ln.lstrip().startswith("#"))
+
+ syms = set()
+ for m in re.finditer(r"\bFFWD_API\b(.*?)([(;])", text, flags=re.S):
+ ids = re.findall(r"[A-Za-z_][A-Za-z0-9_]*", m.group(1))
+ if ids and ids[-1].startswith("ffwd_"):
+ syms.add(ids[-1])
+ return sorted(syms)
+
+
+def render_exports(syms):
+ return "".join("_%s\n" % s for s in syms)
+
+
+def render_map(syms):
+ body = "\n".join(" %s;" % s for s in syms)
+ return "FFWD {\n global:\n%s\n local:\n *;\n};\n" % body
+
+
+def main():
+ here = os.path.dirname(os.path.abspath(__file__))
+ root = os.path.dirname(here)
+ ap = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ ap.add_argument("--header", default=os.path.join(root, "libffwd", "ffwd.h"))
+ ap.add_argument("--outdir", default=root, help="directory for ffwd.exports / ffwd.map")
+ ap.add_argument("--check", action="store_true", help="verify files are in sync; do not write")
+ args = ap.parse_args()
+
+ with open(args.header) as f:
+ syms = find_symbols(f.read())
+ if not syms:
+ sys.exit("error: no FFWD_API symbols found in %s" % args.header)
+
+ targets = (
+ (os.path.join(args.outdir, "ffwd.exports"), render_exports(syms)),
+ (os.path.join(args.outdir, "ffwd.map"), render_map(syms)),
+ )
+
+ if args.check:
+ stale = False
+ for path, want in targets:
+ try:
+ have = open(path).read()
+ except FileNotFoundError:
+ have = None
+ if have != want:
+ print("out of sync: %s" % path)
+ stale = True
+ if stale:
+ sys.exit("run: python3 %s" % os.path.relpath(__file__, os.getcwd()))
+ print("in sync (%d symbols)" % len(syms))
+ return
+
+ for path, want in targets:
+ with open(path, "w") as f:
+ f.write(want)
+ print("wrote ffwd.exports and ffwd.map (%d symbols)" % len(syms))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/devtools/gen_test_vectors.py b/devtools/gen_test_vectors.py
new file mode 100644
index 0000000..44206ec
--- /dev/null
+++ b/devtools/gen_test_vectors.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Generate tokenizer parity vectors for the hermetic tokenizer checks.
+
+tests/check_tokenizer_parity.py and tests/check_wordpiece_parity.py read
+the stored vectors generated by this script. Regenerate the vectors when
+the test corpus changes or the reference tokenizer is bumped, then check
+the result into git.
+
+For each tokenizer family it writes, under tests/test-vectors/<family>/:
+ - the vocab files the C driver loads (vocab.json + merges.txt for BPE;
+ vocab.txt + tokenizer_config.json for WordPiece),
+ - expected.json: provenance plus a list of {text, ids} cases.
+
+- The BPE reference is the fast Qwen tokenizer (what the C BPE backend
+targets).
+- The WordPiece reference is the slow BertTokenizer (the exact reference
+for the C WordPiece backend).
+- Both encode with add_special_tokens=False, matching the C drivers,
+which emit core ids and leave [CLS]/[SEP] to the frontend.
+
+ uv run --with transformers devtools/gen_test_vectors.py
+ uv run --with transformers devtools/gen_test_vectors.py --family bpe
+ uv run --with transformers devtools/gen_test_vectors.py --bpe-model DIR --fuzz 128
+"""
+
+import argparse
+import json
+import random
+import re
+import shutil
+import sys
+from pathlib import Path
+
+# BPE corpus: ASCII, accents, decomposed Unicode, emoji, digits, and long
+# repeated strings that stress the merge loop.
+BPE_TEXTS = [
+ "",
+ "hello world",
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "punctuation: hello, world!!! (test)",
+ "repeated spaces\tand\nnewlines",
+ "unicode: cafe, caf\xe9, Istanbul, İstanbul",
+ "unicode decomposed: café, façade",
+ "emoji: embeddings \U0001f50d are useful",
+ "numbers: 12345 67.89 -0.12",
+ "a " * 256,
+ "antidisestablishmentarianism" * 64,
+ ("document: " + "dense retrieval systems need stable tokenization. " * 256),
+]
+
+BPE_FUZZ_ALPHABET = (
+ "abcdefghijklmnopqrstuvwxyz"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "0123456789"
+ " \t\n\r"
+ ".,!?;:()[]{}'\"-/"
+ "éèêëáàâäíìîïóòôöúùûüñç"
+ "¿¡«»—…"
+ "中文测试Русскийعربيहिन्दी"
+ "🔍👍🏽🇺🇸"
+)
+
+# WordPiece corpus: stays within the implemented coverage (ASCII, Latin-1
+# accents, common punctuation, CJK char-splitting) plus a multilingual set.
+WORDPIECE_TEXTS = [
+ "",
+ "hello world",
+ "The capital of France is Paris.",
+ "Tokenization, sub-words, and WordPiece!",
+ "unaffable antidisestablishmentarianism",
+ "café résumé naïve Zürich Málaga",
+ "COVID-19 affected 2020-2021 globally.",
+ "don't can't won't it's o'clock",
+ "e-mail user@example.com or http://test.org/path?q=1",
+ "numbers: 12345 67.89 -0.12 3.14159 (approx)",
+ "repeated spaces\tand\nnewlines\rhere",
+ "Hyphenated state-of-the-art well-known co-operate",
+ "MixedCase camelCase snake_case ALLCAPS",
+ "中 文 字 符",
+ "a " * 200,
+ "supercalifragilisticexpialidocious pneumonoultramicroscopicsilicovolcanoconiosis",
+ "Müller wohnt in Zürich, Straße.",
+ "L'élève préfère — n'est-ce pas?",
+ "¿Cómo estás? ¡Muy bien!",
+ "你好,世界。こんにちは",
+]
+
+WORDPIECE_FUZZ_ALPHABET = (
+ "abcdefghijklmnopqrstuvwxyz"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "0123456789"
+ " \t\n\r"
+ ".,!?;:()[]{}'\"-/@#"
+ "éèêëáàâäíìîïóòôöúùûüñçßæøþ"
+)
+
+
+def repo_root() -> Path:
+ return Path(__file__).resolve().parents[1]
+
+
+def transformers_version() -> str:
+ import transformers
+ return transformers.__version__
+
+
+def snapshot_revision(local_path: str) -> str:
+ """Best-effort: pull the commit hash from an HF cache path, else ''."""
+ m = re.search(r"/snapshots/([0-9a-f]{8,})/", local_path + "/")
+ return m.group(1) if m else ""
+
+
+def resolve_file(model: str, filename: str) -> Path:
+ """Return a local path to one of the model's files, from a local dir or the
+ HF cache (download only if absent)."""
+ local = Path(model) / filename
+ if local.exists():
+ return local
+ from huggingface_hub import hf_hub_download
+ return Path(hf_hub_download(model, filename))
+
+
+def fuzz_cases(alphabet: str, count: int, seed: int) -> list[str]:
+ rng = random.Random(seed)
+ out = []
+ for _ in range(count):
+ n = rng.randint(0, 96)
+ out.append("".join(rng.choice(alphabet) for _ in range(n)))
+ return out
+
+
+def write_vectors(out_dir: Path, family: str, model: str, revision: str,
+ texts: list[str], encode) -> None:
+ out_dir.mkdir(parents=True, exist_ok=True)
+ cases = [{"text": t, "ids": encode(t)} for t in texts]
+ payload = {
+ "family": family,
+ "model": model,
+ "revision": revision,
+ "transformers_version": transformers_version(),
+ "add_special_tokens": False,
+ "cases": cases,
+ }
+ (out_dir / "expected.json").write_text(
+ json.dumps(payload, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
+ print(f"ok: wrote {len(cases)} {family} cases to {out_dir}")
+
+
+def gen_bpe(root: Path, model: str, fuzz: int, seed: int) -> None:
+ from transformers import AutoTokenizer
+ tok = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
+ out_dir = root / "tests/test-vectors/bpe"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ # Copy the shipped vocab.json + merges.txt the C BPE loader reads. The fast
+ # tokenizer's save_vocabulary does not re-emit them, so copy the originals
+ # the reference tokenizer was built from.
+ src = ""
+ for fn in ("vocab.json", "merges.txt"):
+ path = resolve_file(model, fn)
+ src = str(path)
+ shutil.copyfile(path, out_dir / fn)
+ texts = list(BPE_TEXTS) + fuzz_cases(BPE_FUZZ_ALPHABET, fuzz, seed)
+ revision = snapshot_revision(src)
+ write_vectors(out_dir, "bpe", model, revision, texts,
+ lambda t: tok.encode(t, add_special_tokens=False))
+
+
+def gen_wordpiece(root: Path, model: str, fuzz: int, seed: int) -> None:
+ from transformers import BertTokenizer
+ tok = BertTokenizer.from_pretrained(model)
+ out_dir = root / "tests/test-vectors/wordpiece"
+ out_dir.mkdir(parents=True, exist_ok=True)
+ # Copy vocab.txt and write the tokenizer_config.json the C loader reads for
+ # do_lower_case (save_pretrained would emit tokenizer.json, which the C
+ # side ignores).
+ src = resolve_file(model, "vocab.txt")
+ shutil.copyfile(src, out_dir / "vocab.txt")
+ lower = "true" if tok.do_lower_case else "false"
+ (out_dir / "tokenizer_config.json").write_text(
+ '{"do_lower_case": %s}\n' % lower, encoding="utf-8")
+ texts = list(WORDPIECE_TEXTS) + fuzz_cases(WORDPIECE_FUZZ_ALPHABET, fuzz, seed)
+ revision = snapshot_revision(str(src))
+ write_vectors(out_dir, "wordpiece", model, revision, texts,
+ lambda t: tok.encode(t, add_special_tokens=False))
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--family", choices=("bpe", "wordpiece", "both"), default="both")
+ ap.add_argument("--bpe-model", default="Qwen/Qwen3-Embedding-0.6B",
+ help="HF name or local dir for the BPE reference tokenizer")
+ ap.add_argument("--wordpiece-model", default="bert-base-uncased",
+ help="HF name or local dir for the WordPiece reference tokenizer")
+ ap.add_argument("--fuzz", type=int, default=64,
+ help="deterministic random cases to bake in per family")
+ ap.add_argument("--seed", type=int, default=1)
+ args = ap.parse_args()
+
+ root = repo_root()
+ if args.family in ("bpe", "both"):
+ gen_bpe(root, args.bpe_model, args.fuzz, args.seed)
+ if args.family in ("wordpiece", "both"):
+ gen_wordpiece(root, args.wordpiece_model, args.fuzz, args.seed)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py
index 24649f1..2f8642e 100644..100755
--- a/tools/gen_wordpiece_unicode.py
+++ b/devtools/gen_wordpiece_unicode.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece
+"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece
BasicTokenizer's punctuation and control classification, matching transformers'
BertTokenizer._is_punctuation and _is_control exactly.
@@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
Run when bumping the target Unicode version (the C side is data-only otherwise):
- uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h
+ uv run python devtools/gen_wordpiece_unicode.py > unicode.h
"""
import unicodedata
@@ -94,7 +94,7 @@ def main():
for f in [fold(cp)]
if f != [cp]
]
- print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
+ print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata
* (Unicode {unicodedata.unidata_version}). Do not edit by hand.
*
* Tables for the BasicTokenizer, matching transformers' BertTokenizer.
diff --git a/devtools/inspect_model_artifacts.py b/devtools/inspect_model_artifacts.py
new file mode 100755
index 0000000..0265832
--- /dev/null
+++ b/devtools/inspect_model_artifacts.py
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+"""Inspect and detect local model snapshots, across model families.
+
+For each model it detects:
+ - the architecture (from config.json: model_type / architectures / dims),
+ - the tokenizer family - BPE (vocab.json + merges.txt), WordPiece (vocab.txt),
+ or SentencePiece (sentencepiece.bpe.model / spiece.model / tokenizer.model),
+ - the weight layout - single safetensors, sharded safetensors, or legacy
+ pytorch_model.bin,
+ - the serving kind - base encoder, pooled sentence-embedding, or projected /
+ late-interaction (1_Dense) - plus the Sentence-Transformers pipeline.
+
+ONNX exports and other extras are reported when present.
+
+A MODEL_DIR argument may be a model directory (one with config.json) or any
+directory to search. Point it at a single snapshot, a folder of models, or the
+whole Hugging Face hub cache (~/.cache/huggingface/hub) and it finds each model.
+
+A snapshot is "ok" when it has config.json, some weights, and some tokenizer;
+everything else is informational.
+
+Usage: inspect_model_artifacts.py MODEL_DIR [MODEL_DIR ...]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+
+TOKENIZER_ARTIFACTS = (
+ "tokenizer.json",
+ "tokenizer_config.json",
+ "special_tokens_map.json",
+ "added_tokens.json",
+ "vocab.json",
+ "merges.txt",
+ "vocab.txt",
+ "sentencepiece.bpe.model",
+ "spiece.model",
+ "tokenizer.model",
+)
+
+# The first family whose required files are all present wins.
+TOKENIZER_FAMILIES = (
+ ("BPE", ("vocab.json", "merges.txt")),
+ ("WordPiece", ("vocab.txt",)),
+ ("SentencePiece", ("sentencepiece.bpe.model",)),
+ ("SentencePiece", ("spiece.model",)),
+ ("SentencePiece", ("tokenizer.model",)),
+)
+
+EXTRA_ARTIFACTS = (
+ "config_sentence_transformers.json",
+ "sentence_bert_config.json",
+ "modules.json",
+ "1_Pooling",
+ "1_Dense",
+ "onnx",
+ "generation_config.json",
+)
+
+
+def human_bytes(size: int) -> str:
+ value = float(size)
+ for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+ if value < 1024.0 or unit == "TiB":
+ return f"{value:.1f} {unit}"
+ value /= 1024.0
+ return f"{value:.1f} TiB"
+
+
+def describe(path: Path) -> str:
+ if not path.exists():
+ return "MISSING"
+ if path.is_dir():
+ return f"{sum(1 for _ in path.iterdir())} entries"
+ return human_bytes(path.resolve().stat().st_size)
+
+
+def fmt(label: str, path: Path, indent: int = 4, width: int = 32) -> str:
+ return f"{' ' * indent}{label:<{width}} {describe(path)}"
+
+
+def is_model_dir(d: Path) -> bool:
+ return (d / "config.json").is_file()
+
+
+def discover(root: Path) -> list[Path]:
+ """Return the model directories at or under `root`.
+
+ A model directory has a config.json. If `root` is itself one, return it;
+ otherwise search descendants (e.g. a folder of models or the HF hub cache),
+ skipping onnx sub-exports and dropping any dir nested under another model.
+ """
+ root = root.expanduser().resolve()
+ if is_model_dir(root):
+ return [root]
+ found = {
+ cfg.parent
+ for cfg in root.rglob("config.json")
+ if cfg.parent.name != "onnx"
+ }
+ out = []
+ for d in sorted(found):
+ if not any(d != o and str(d).startswith(str(o) + os.sep) for o in found):
+ out.append(d)
+ return out
+
+
+def tokenizer_family(d: Path) -> str:
+ for name, required in TOKENIZER_FAMILIES:
+ if all((d / f).exists() for f in required):
+ return name
+ if (d / "tokenizer.json").exists():
+ return "fast-only (tokenizer.json)"
+ return "none"
+
+
+def config_summary(d: Path) -> str | None:
+ cfg = d / "config.json"
+ if not cfg.exists():
+ return None
+ try:
+ data = json.loads(cfg.read_text())
+ except (OSError, ValueError):
+ return "config.json: unreadable"
+ parts = []
+ if data.get("model_type"):
+ parts.append(str(data["model_type"]))
+ arch = data.get("architectures") or []
+ if arch:
+ parts.append(str(arch[0]))
+ for key in ("hidden_size", "num_hidden_layers", "num_attention_heads",
+ "num_key_value_heads", "vocab_size", "max_position_embeddings"):
+ if key in data:
+ parts.append(f"{key}={data[key]}")
+ return " ".join(parts) if parts else "config.json: no recognized fields"
+
+
+def modules_pipeline(d: Path) -> str | None:
+ mj = d / "modules.json"
+ if not mj.exists():
+ return None
+ try:
+ mods = json.loads(mj.read_text())
+ except (OSError, ValueError):
+ return None
+ names = [(m.get("type") or "").rsplit(".", 1)[-1] or m.get("name") or "?"
+ for m in mods]
+ return " -> ".join(n for n in names if n) or None
+
+
+def model_kind(d: Path) -> str:
+ if (d / "1_Dense").is_dir():
+ return "projected / late-interaction (1_Dense)"
+ if (d / "1_Pooling").is_dir() or (d / "modules.json").exists():
+ return "pooled sentence-embedding"
+ return "base encoder / LM"
+
+
+def weight_lines(d: Path) -> tuple[list[str], bool]:
+ """Return (formatted lines, whether any weights were found)."""
+ lines: list[str] = []
+ st_index = d / "model.safetensors.index.json"
+ st_shards = sorted(d.glob("model-*-of-*.safetensors"))
+ pt_index = d / "pytorch_model.bin.index.json"
+ pt_shards = sorted(d.glob("pytorch_model-*-of-*.bin"))
+ onnx = sorted((d / "onnx").glob("*.onnx")) if (d / "onnx").is_dir() else []
+ found = False
+
+ if (d / "model.safetensors").exists():
+ lines.append(fmt("model.safetensors", d / "model.safetensors"))
+ found = True
+ if st_index.exists() and st_shards:
+ lines.append(fmt("model.safetensors.index.json", st_index))
+ lines += [fmt(s.name, s, indent=6) for s in st_shards]
+ found = True
+ if (d / "pytorch_model.bin").exists():
+ lines.append(fmt("pytorch_model.bin", d / "pytorch_model.bin"))
+ found = True
+ if pt_index.exists() and pt_shards:
+ lines.append(fmt("pytorch_model.bin.index.json", pt_index))
+ lines += [fmt(s.name, s, indent=6) for s in pt_shards]
+ found = True
+ if onnx:
+ lines += [fmt(f"onnx/{o.name}", o) for o in onnx]
+ found = True
+ if not found:
+ lines.append(" (no safetensors / pytorch / onnx weights)")
+ return lines, found
+
+
+def inspect(model_dir: Path) -> bool:
+ d = model_dir.expanduser().resolve()
+ print(f"snapshot: {d}")
+ if not d.is_dir():
+ print(" NOT A DIRECTORY")
+ return False
+
+ print(f" model: {config_summary(d) or 'config.json MISSING'}")
+ print(f" kind: {model_kind(d)}")
+ pipeline = modules_pipeline(d)
+ if pipeline:
+ print(f" pipeline: {pipeline}")
+
+ family = tokenizer_family(d)
+ print(f" tokenizer: {family}")
+ for name in TOKENIZER_ARTIFACTS:
+ if (d / name).exists():
+ print(fmt(name, d / name))
+
+ print(" weights:")
+ wlines, has_weights = weight_lines(d)
+ print("\n".join(wlines))
+
+ extras = [name for name in EXTRA_ARTIFACTS if (d / name).exists()]
+ if extras:
+ print(" extras:")
+ for name in extras:
+ print(fmt(name, d / name))
+
+ ok = (d / "config.json").exists() and has_weights and family != "none"
+ print(f" verdict: {'ok' if ok else 'incomplete'}")
+ return ok
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR")
+ args = ap.parse_args()
+
+ ok = True
+ first = True
+ for name in args.model_dirs:
+ root = Path(os.path.expandvars(name)).expanduser().resolve()
+ found = discover(root)
+ if not found:
+ if not first:
+ print()
+ print(f"snapshot: {root}\n no config.json found at or under this path")
+ ok = first = False
+ continue
+ if len(found) > 1:
+ if not first:
+ print()
+ print(f"# {root}: {len(found)} models")
+ first = False
+ for d in found:
+ if not first:
+ print()
+ ok = inspect(d) and ok
+ first = False
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/openapi.yaml b/devtools/openapi.yaml
index 6dd3818..5140186 100644
--- a/tools/openapi.yaml
+++ b/devtools/openapi.yaml
@@ -1,9 +1,9 @@
openapi: 3.0.3
info:
- title: embed.c server API
+ title: ffwd server API
version: "1.0.0"
description: >
- HTTP API served by `embed-server`. Each loaded model has an operator-chosen
+ HTTP API served by `ffwd-server`. Each loaded model has an operator-chosen
label; request `model` fields match that label. Embedding encoding follows
the serving API selected at load time (`api=openai` or `api=perplexity`).
Always compare embeddings with cosine similarity.
@@ -12,7 +12,7 @@ info:
servers:
- url: http://127.0.0.1:8000
# Auth is optional: it applies only when the server is started with --api-key
-# (or EMBED_API_KEY). The empty requirement allows unauthenticated calls.
+# (or FFWD_API_KEY). The empty requirement allows unauthenticated calls.
security:
- {}
- bearerAuth: []
@@ -86,7 +86,7 @@ components:
properties:
model:
type: string
- description: Served label chosen when starting `embed-server`.
+ description: Served label chosen when starting `ffwd-server`.
example: qwen
input:
description: A single string or an array of up to 512 strings.
diff --git a/devtools/quant_precision_drift.py b/devtools/quant_precision_drift.py
new file mode 100755
index 0000000..896d2f8
--- /dev/null
+++ b/devtools/quant_precision_drift.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""Compare f32 -> low-precision int8-output drift across engines.
+
+The native BF16 path changes ~10-15% of int8 output bytes versus F32 (while
+keeping cosine ~0.9999 and identical retrieval). This script checks whether that
+is normal by measuring the *same* f32 -> low-precision drift in independent
+implementations of the same model:
+
+ - ONNX : released model.onnx (F32) vs model_quantized.onnx (official int8)
+ - HF : sentence-transformers / torch, loaded in F32 vs BF16
+ - native : ffwd binary, an F32 model dir vs a BF16 model dir
+
+For every engine it reports, per low-precision variant against that engine's own
+F32 output: the int8 byte-change rate, the int8-vector cosine, and the float
+relative-L2 diff. If ONNX-int8 and HF-bf16 drift like ours, the drift is inherent
+to low precision, not a backend defect.
+
+Reference engines output int8-scale embeddings (round); native outputs raw
+floats quantized with tanh*127 (matching the server's base64_int8 encoder).
+
+Needs the reference stack, so run it with uv (Python 3.12 for the numpy pin):
+
+ uv run --python 3.12 --with-requirements requirements-reference.txt \
+ devtools/quant_precision_drift.py --model-dir DIR --engines hf
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
+
+
+DEFAULT_TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "document: The Eiffel Tower is a famous landmark located in Paris, France.",
+ "document: Photosynthesis converts sunlight into chemical energy in plants.",
+ (
+ "document: Embedding models turn text into dense vectors for semantic "
+ "retrieval. Long document inference stresses bidirectional attention "
+ "because every token attends to every other token in the same packed "
+ "sequence. "
+ ) * 8,
+]
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ if na == 0.0 or nb == 0.0:
+ return 1.0
+ return dot / (na * nb)
+
+
+def rel_l2(a: list[float], b: list[float]) -> float:
+ num = math.sqrt(sum((x - y) ** 2 for x, y in zip(a, b)))
+ den = math.sqrt(sum(x * x for x in a))
+ return num / den if den else 0.0
+
+
+def drift(ref: list[list[int]], low: list[list[int]]) -> dict[str, float]:
+ if len(ref) != len(low):
+ raise RuntimeError(f"row count mismatch: {len(ref)} vs {len(low)}")
+ total = 0
+ changed = 0
+ worst_cos = 1.0
+ worst_l2 = 0.0
+ for a, b in zip(ref, low):
+ if len(a) != len(b):
+ raise RuntimeError(f"dim mismatch: {len(a)} vs {len(b)}")
+ total += len(a)
+ changed += sum(1 for x, y in zip(a, b) if x != y)
+ worst_cos = min(worst_cos, cosine([float(x) for x in a],
+ [float(y) for y in b]))
+ worst_l2 = max(worst_l2, rel_l2([float(x) for x in a],
+ [float(y) for y in b]))
+ return {
+ "int8_change_rate": changed / total if total else 0.0,
+ "worst_int8_cosine": worst_cos,
+ "worst_rel_l2": worst_l2,
+ }
+
+
+# --- engines -------------------------------------------------------------
+
+def onnx_int8(model_dir: str, onnx_file: str, texts: list[str]) -> list[list[int]]:
+ import numpy as np
+ import onnxruntime as ort
+ from transformers import AutoTokenizer
+
+ tok = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True,
+ local_files_only=True)
+ sess = ort.InferenceSession(str(Path(model_dir) / "onnx" / onnx_file),
+ providers=["CPUExecutionProvider"])
+ t = tok(texts, padding=True, truncation=True, return_tensors="np")
+ feed = {"input_ids": t["input_ids"].astype(np.int64),
+ "attention_mask": t["attention_mask"].astype(np.int64)}
+ # outputs: [0]=hidden states, [2]=pooled int8-scale embedding (see
+ # compare_reference_backends.worker_onnx).
+ out = sess.run(None, feed)[2]
+ return [[int(round(float(x))) for x in row] for row in out]
+
+
+def hf_int8(model_dir: str, texts: list[str], dtype: str) -> list[list[int]]:
+ import torch
+ from sentence_transformers import SentenceTransformer
+
+ td = {"f32": torch.float32, "bf16": torch.bfloat16}[dtype]
+ model = SentenceTransformer(model_dir, trust_remote_code=True,
+ local_files_only=True,
+ model_kwargs={"torch_dtype": td})
+ out = model.encode(texts, show_progress_bar=False)
+ return [[int(round(float(x))) for x in row] for row in out.tolist()]
+
+
+def native_int8(binary: str, model_dir: str, backend: str,
+ texts: list[str]) -> list[list[int]]:
+ cmd = [binary, "-d", model_dir, f"--{backend}", "--stream", "-b", "8"]
+ proc = subprocess.run(cmd, input="\n".join(texts) + "\n", text=True,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if proc.returncode != 0:
+ raise RuntimeError(f"native failed: {' '.join(cmd)}\n{proc.stderr[-800:]}")
+ rows = []
+ for line in proc.stdout.splitlines():
+ if not line.strip():
+ continue
+ emb = json.loads(line)["embedding"]
+ rows.append([max(-128, min(127, round(math.tanh(float(x)) * 127.0)))
+ for x in emb])
+ return rows
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--model-dir", required=True,
+ help="F32 reference model dir (HF snapshot with onnx/)")
+ ap.add_argument("--engines", default="onnx,hf",
+ help="comma list: onnx,hf,native")
+ ap.add_argument("--native-binary", default=str(DEFAULT_BINARY))
+ ap.add_argument("--native-backend", default="mlx", choices=("cpu", "mlx"))
+ ap.add_argument("--native-bf16-dir",
+ help="BF16 model dir for the native engine (required for native)")
+ ap.add_argument("--onnx-low", default="model_quantized.onnx",
+ help="quantized ONNX file (vs model.onnx)")
+ args = ap.parse_args()
+
+ texts = DEFAULT_TEXTS
+ engines = [e for e in args.engines.split(",") if e]
+ results: list[tuple[str, str, dict[str, float]]] = []
+
+ for eng in engines:
+ try:
+ if eng == "onnx":
+ ref = onnx_int8(args.model_dir, "model.onnx", texts)
+ low = onnx_int8(args.model_dir, args.onnx_low, texts)
+ results.append(("ONNX", f"f32 vs {args.onnx_low}", drift(ref, low)))
+ elif eng == "hf":
+ ref = hf_int8(args.model_dir, texts, "f32")
+ low = hf_int8(args.model_dir, texts, "bf16")
+ results.append(("HF/torch", "f32 vs bf16", drift(ref, low)))
+ elif eng == "native":
+ if not args.native_bf16_dir:
+ raise RuntimeError("--native-bf16-dir required for native")
+ ref = native_int8(args.native_binary, args.model_dir,
+ args.native_backend, texts)
+ low = native_int8(args.native_binary, args.native_bf16_dir,
+ args.native_backend, texts)
+ results.append((f"native/{args.native_backend}", "f32 vs bf16",
+ drift(ref, low)))
+ else:
+ print(f"unknown engine: {eng}", file=sys.stderr)
+ except Exception as exc: # one engine failing should not block the rest
+ print(f" [{eng}] SKIPPED: {type(exc).__name__}: {exc}",
+ file=sys.stderr)
+
+ print(f"\nf32 -> low-precision int8 drift ({len(texts)} texts)\n")
+ print(f"{'engine':14s} {'comparison':24s} {'int8_change':>11s} "
+ f"{'int8_cosine':>12s} {'rel_l2':>8s}")
+ print("-" * 74)
+ for name, comp, d in results:
+ print(f"{name:14s} {comp:24s} {d['int8_change_rate']*100:10.1f}% "
+ f"{d['worst_int8_cosine']:12.5f} {d['worst_rel_l2']*100:7.2f}%")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/rank_embeddings.py b/devtools/rank_embeddings.py
index ddc1e42..efc8aad 100755
--- a/tools/rank_embeddings.py
+++ b/devtools/rank_embeddings.py
@@ -2,7 +2,7 @@
"""Rank stdin JSON embeddings by cosine similarity to a query row.
Usage:
- ./embed ... --stream <<'EOF' | tools/rank_embeddings.py
+ ./tools/ffwd-cli ... --stream <<'EOF' | ./devtools/rank_embeddings.py
query: what is the capital of France?
document: Paris is the capital of France.
document: Berlin is the capital of Germany.