diff options
| -rwxr-xr-x | devtools/compare_model_embeddings.py | 130 | ||||
| -rwxr-xr-x | devtools/compare_reference_backends.py | 644 | ||||
| -rwxr-xr-x | devtools/convert_model_bf16.py (renamed from tools/convert_model_bf16.py) | 0 | ||||
| -rwxr-xr-x | devtools/gen_exports.py | 91 | ||||
| -rw-r--r-- | devtools/gen_test_vectors.py | 209 | ||||
| -rwxr-xr-x[-rw-r--r--] | devtools/gen_wordpiece_unicode.py (renamed from tools/gen_wordpiece_unicode.py) | 6 | ||||
| -rwxr-xr-x | devtools/inspect_model_artifacts.py | 260 | ||||
| -rw-r--r-- | devtools/openapi.yaml (renamed from tools/openapi.yaml) | 8 | ||||
| -rwxr-xr-x | devtools/quant_precision_drift.py | 197 | ||||
| -rwxr-xr-x | devtools/rank_embeddings.py (renamed from tools/rank_embeddings.py) | 2 | ||||
| -rwxr-xr-x | tools/inspect_model_artifacts.py | 88 |
11 files changed, 1539 insertions, 96 deletions
diff --git a/devtools/compare_model_embeddings.py b/devtools/compare_model_embeddings.py new file mode 100755 index 0000000..95154a2 --- /dev/null +++ b/devtools/compare_model_embeddings.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Compare embeddings from two model directories through the CLI.""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +def ensure_cli(binary: Path) -> None: + """Build the CLI if the default binary is missing.""" + if binary == DEFAULT_BINARY and not binary.exists(): + subprocess.run(["make", "-C", "tools/cli", "all"], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL) + + +TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "The quick brown fox jumps over the lazy dog near the riverbank", + ( + "Retrieval augmented generation combines embedding models with external " + "knowledge sources and ranks documents by semantic similarity." + ), +] + + +def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int, + backend: str, threads: int) -> list[list[float]]: + cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)] + if backend == "cpu" and threads > 0: + cmd.extend(["-t", str(threads)]) + + proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert proc.stdin is not None + assert proc.stdout is not None + proc.stdin.write("\n".join(texts) + "\n") + proc.stdin.close() + + embeddings = [] + for _ in texts: + line = proc.stdout.readline() + if not line: + stderr = proc.stderr.read() if proc.stderr else "" + raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}") + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin error for {model_dir}: {row}") + embeddings.append([float(x) for x in row["embedding"]]) + + rc = proc.wait(timeout=10) + if rc != 0: + stderr = proc.stderr.read() if proc.stderr else "" + raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}") + return embeddings + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--a", required=True, help="reference model directory") + ap.add_argument("--b", required=True, help="comparison model directory") + ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu") + ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"], + help="backend for --a; defaults to --backend") + ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"], + help="backend for --b; defaults to --backend") + ap.add_argument("--batch-size", type=int, default=4) + ap.add_argument("--threads", type=int, default=0) + ap.add_argument("--max-diff", type=float, default=0.01) + ap.add_argument("--min-cos", type=float, default=0.999) + args = ap.parse_args() + + if args.batch_size <= 0: + raise SystemExit("--batch-size must be > 0") + ensure_cli(Path(args.binary)) + if not Path(args.binary).exists(): + raise SystemExit(f"binary not found: {args.binary}") + + backend_a = args.backend_a or args.backend + backend_b = args.backend_b or args.backend + emb_a = run_model(args.binary, args.a, TEXTS, args.batch_size, + backend_a, args.threads) + emb_b = run_model(args.binary, args.b, TEXTS, args.batch_size, + backend_b, args.threads) + + worst_diff = 0.0 + worst_cos = 1.0 + for i, (a, b) in enumerate(zip(emb_a, emb_b)): + if len(a) != len(b): + raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}") + max_diff = max(abs(x - y) for x, y in zip(a, b)) + cos = cosine(a, b) + worst_diff = max(worst_diff, max_diff) + worst_cos = min(worst_cos, cos) + print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}") + + print( + f"ok: backend_a={backend_a} backend_b={backend_b} " + f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}" + ) + if worst_diff > args.max_diff or worst_cos < args.min_cos: + raise SystemExit( + f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/devtools/compare_reference_backends.py b/devtools/compare_reference_backends.py new file mode 100755 index 0000000..0f6d758 --- /dev/null +++ b/devtools/compare_reference_backends.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python3 +"""Compare ffwd output with offline reference implementations. + +Benchmarks the native CLI (stdin) or server (http) against sentence-transformers +and ONNX, reporting cosine parity, latency, and peak RSS. Needs the reference +stack, so run it with uv (Python 3.12 for the numpy pin): + + uv run --python 3.12 --with-requirements requirements-reference.txt \ + devtools/compare_reference_backends.py --model-dir DIR \ + --native-mode stdin --native-backend cpu --references sentence-transformers +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import signal +import statistics +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Callable + + +DEFAULT_STANDARD = [ + "Scientists explore the universe driven by curiosity.", + "Children learn through curious exploration.", + "Historical discoveries began with curious questions.", + "Redis is an in-memory data structure server often used for caching.", + "SQLite stores relational data in a single local database file.", +] + +DEFAULT_CONTEXTUAL = [ + [ + "Paris is the capital city of France.", + "The Eiffel Tower is a landmark in Paris.", + "France is a country in Western Europe.", + ], + [ + "Redis stores data in memory.", + "SQLite stores relational data in a local file.", + ], +] + +RESULT_PREFIX = "EMBED_REFERENCE_RESULT=" + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def percentile(values: list[float], pct: float) -> float: + ordered = sorted(values) + pos = max(0, math.ceil((pct / 100.0) * len(ordered)) - 1) + return ordered[pos] + + +def cosine(a: list[int], b: list[int]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + aa = sum(x * x for x in a) + bb = sum(x * x for x in b) + return dot / math.sqrt(aa * bb) if aa and bb else 0.0 + + +def int8_rows(value: Any) -> list[list[int]]: + rows = value.tolist() if hasattr(value, "tolist") else value + return [[int(round(float(x))) for x in row] for row in rows] + + +def flatten_contextual(value: Any) -> list[list[int]]: + rows: list[list[int]] = [] + for document in value: + rows.extend(int8_rows(document)) + return rows + + +def labels(kind: str, inputs: Any) -> list[str]: + if kind == "standard": + return [f"text[{i}]" for i in range(len(inputs))] + return [ + f"doc[{di}]/chunk[{ci}]" + for di, document in enumerate(inputs) + for ci in range(len(document)) + ] + + +def offline_environment(cache_dir: Path) -> dict[str, str]: + cache_dir.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env.update( + { + "HF_HOME": str(cache_dir), + "HF_HUB_CACHE": str(cache_dir / "hub"), + "HF_MODULES_CACHE": str(cache_dir / "modules"), + "TRANSFORMERS_CACHE": str(cache_dir / "transformers"), + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "HF_DATASETS_OFFLINE": "1", + "TOKENIZERS_PARALLELISM": "false", + } + ) + return env + + +def rss_mb(pid: int | None) -> float | None: + if not pid: + return None + proc = subprocess.run( + ["ps", "-o", "rss=", "-p", str(pid)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + try: + return int(proc.stdout.strip()) / 1024.0 + except ValueError: + return None + + +def post_json(url: str, payload: dict[str, Any], timeout: float) -> Any: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {exc.code}: {body}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"request failed for {url}: {exc.reason}") from exc + + +def decode_native(response: dict[str, Any], kind: str) -> list[list[int]]: + if kind == "standard": + entries = response["data"] + else: + entries = [chunk for document in response["data"] for chunk in document["data"]] + rows = [] + for entry in entries: + encoded = entry["embedding"] + raw = base64.b64decode(encoded) + rows.append([x - 256 if x > 127 else x for x in raw]) + return rows + + +def quantize_native(rows: list[dict[str, Any]]) -> list[list[int]]: + vectors = [] + for row in rows: + if "error" in row: + raise RuntimeError(f"native stdin returned an error: {row['error']}") + vectors.append( + [ + max(-128, min(127, round(math.tanh(float(x)) * 127.0))) + for x in row["embedding"] + ] + ) + return vectors + + +class RssGuard: + def __init__(self, proc: subprocess.Popen[str], ceiling_mb: float): + self.proc = proc + self.ceiling_mb = ceiling_mb + self.peak_mb = 0.0 + self.error: str | None = None + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.run, daemon=True) + + def start(self) -> None: + self.thread.start() + + def run(self) -> None: + while not self.stop_event.wait(0.025): + current = rss_mb(self.proc.pid) or 0.0 + self.peak_mb = max(self.peak_mb, current) + if current > self.ceiling_mb: + self.error = ( + f"native stdin exceeded RSS ceiling: " + f"{current:.1f} > {self.ceiling_mb} MiB" + ) + self.proc.kill() + return + if self.proc.poll() is not None: + return + + def stop(self) -> None: + self.stop_event.set() + self.thread.join() + self.peak_mb = max(self.peak_mb, rss_mb(self.proc.pid) or 0.0) + + +def benchmark_native_stdin(args: argparse.Namespace) -> dict[str, Any]: + if args.kind != "standard": + raise RuntimeError("native stdin comparison currently supports standard embeddings only") + cmd = [ + args.native_binary, + "-d", + args.model_dir, + "--stream", + "-b", + str(len(args.inputs)), + ] + if args.native_backend == "cpu": + cmd.extend(["-t", str(args.threads)]) + proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + guard = RssGuard(proc, args.max_native_rss_mb) + guard.start() + + def send() -> tuple[list[list[int]], float]: + if not proc.stdin or not proc.stdout: + raise RuntimeError("native stdin pipes are unavailable") + proc.stdin.write("\n".join(args.inputs) + "\n") + proc.stdin.flush() + rows = [] + for _ in args.inputs: + line = proc.stdout.readline() + if not line: + errors = proc.stderr.read() if proc.stderr else "" + raise RuntimeError(guard.error or f"native stdin exited early:\n{errors}") + rows.append(json.loads(line)) + return quantize_native(rows), float(rows[0]["ms"]) + + try: + start = time.perf_counter() + vectors, _ = send() + load_ms = (time.perf_counter() - start) * 1000.0 + for _ in range(args.warmups): + vectors, _ = send() + elapsed = [] + for _ in range(args.runs): + vectors, engine_ms = send() + elapsed.append(engine_ms) + finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + guard.stop() + if guard.error: + raise RuntimeError(guard.error) + if proc.returncode != 0: + errors = proc.stderr.read() if proc.stderr else "" + raise RuntimeError(f"native stdin failed ({proc.returncode}):\n{errors}") + return { + "name": f"native-stdin-{args.native_backend}", + "vectors": vectors, + "load_ms": load_ms, + "p50_ms": statistics.median(elapsed), + "p95_ms": percentile(elapsed, 95.0), + "rss_mb": guard.peak_mb, + } + + +def benchmark_http( + name: str, + url: str, + payload: dict[str, Any], + decode: Callable[[Any], list[list[int]]], + warmups: int, + runs: int, + timeout: float, + pid: int | None, +) -> dict[str, Any]: + for _ in range(warmups): + decode(post_json(url, payload, timeout)) + elapsed = [] + vectors = None + peak_rss = rss_mb(pid) + for _ in range(runs): + start = time.perf_counter() + vectors = decode(post_json(url, payload, timeout)) + elapsed.append((time.perf_counter() - start) * 1000.0) + current = rss_mb(pid) + peak_rss = max(peak_rss or 0.0, current or 0.0) or None + return { + "name": name, + "vectors": vectors, + "load_ms": None, + "p50_ms": statistics.median(elapsed), + "p95_ms": percentile(elapsed, 95.0), + "rss_mb": peak_rss, + } + + +def validate_artifacts(model_dir: Path, require_onnx: bool) -> None: + model_weights = model_dir / "model.safetensors" + model_index = model_dir / "model.safetensors.index.json" + required = [ + model_dir / "config.json", + model_dir / "tokenizer.json", + ] + if require_onnx: + required.append(model_dir / "onnx" / "model.onnx") + missing = [str(path) for path in required if not path.exists()] + if not model_weights.exists() and not model_index.exists(): + missing.append(f"{model_weights} or {model_index}") + if require_onnx and not list((model_dir / "onnx").glob("model.onnx_data*")): + missing.append(str(model_dir / "onnx" / "model.onnx_data*")) + if missing: + raise SystemExit("missing local model artifacts:\n " + "\n ".join(missing)) + + +def run_guarded_worker(args: argparse.Namespace, backend: str) -> dict[str, Any]: + cache_dir = Path(args.cache_dir).expanduser().resolve() + env = offline_environment(cache_dir) + cmd = [ + sys.executable, + str(Path(__file__).resolve()), + "--worker", + "--reference", + backend, + "--kind", + args.kind, + "--model-dir", + str(Path(args.model_dir).expanduser().resolve()), + "--runs", + str(args.runs), + "--warmups", + str(args.warmups), + "--threads", + str(args.threads), + "--input-json", + json.dumps(args.inputs), + ] + with tempfile.TemporaryFile(mode="w+") as stdout, tempfile.TemporaryFile( + mode="w+" + ) as stderr: + proc = subprocess.Popen( + cmd, + env=env, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + text=True, + ) + start = time.monotonic() + peak = 0.0 + error = None + while proc.poll() is None: + current = rss_mb(proc.pid) or 0.0 + peak = max(peak, current) + if peak > args.max_reference_rss_mb: + error = ( + f"{backend} exceeded RSS ceiling: " + f"{peak:.1f} > {args.max_reference_rss_mb} MiB" + ) + proc.send_signal(signal.SIGKILL) + break + if time.monotonic() - start > args.timeout: + error = f"{backend} exceeded timeout: {args.timeout:.1f}s" + proc.send_signal(signal.SIGKILL) + break + time.sleep(0.025) + proc.wait() + stdout.seek(0) + stderr.seek(0) + output = stdout.read() + errors = stderr.read() + if error: + raise RuntimeError(error) + if proc.returncode != 0: + raise RuntimeError(f"{backend} failed ({proc.returncode}):\n{errors or output}") + marker = output.rfind(RESULT_PREFIX) + if marker >= 0: + result = json.loads(output[marker + len(RESULT_PREFIX) :].strip()) + result["rss_mb"] = max(float(result["rss_mb"]), peak) + return result + raise RuntimeError(f"{backend} did not emit a result:\n{errors or output}") + + +def worker_sentencetransformers(args: argparse.Namespace, inputs: Any) -> Any: + import torch + + torch.set_num_threads(args.threads) + if args.kind == "standard": + from sentence_transformers import SentenceTransformer + + model = SentenceTransformer( + args.model_dir, trust_remote_code=True, local_files_only=True + ) + return model, lambda: int8_rows(model.encode(inputs, show_progress_bar=False)) + + from transformers import AutoModel + + model = AutoModel.from_pretrained( + args.model_dir, trust_remote_code=True, local_files_only=True + ) + return model, lambda: flatten_contextual(model.encode(inputs, show_progress_bar=False)) + + +def worker_onnx(args: argparse.Namespace, inputs: Any) -> Any: + import numpy as np + import onnxruntime as ort + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained( + args.model_dir, trust_remote_code=True, local_files_only=True + ) + options = ort.SessionOptions() + options.intra_op_num_threads = args.threads + session = ort.InferenceSession( + str(Path(args.model_dir) / "onnx" / "model.onnx"), + sess_options=options, + providers=["CPUExecutionProvider"], + ) + texts = inputs if args.kind == "standard" else [tokenizer.sep_token.join(x) for x in inputs] + tokenized = tokenizer(texts, padding=True, truncation=True, return_tensors="np") + ort_inputs = { + "input_ids": tokenized["input_ids"].astype(np.int64), + "attention_mask": tokenized["attention_mask"].astype(np.int64), + } + + def encode() -> list[list[int]]: + outputs = session.run(None, ort_inputs) + if args.kind == "standard": + return int8_rows(outputs[2]) + rows = [] + for input_ids, mask, hidden in zip( + ort_inputs["input_ids"], ort_inputs["attention_mask"], outputs[0] + ): + start = 0 + end = int(mask.sum()) + for sep in np.flatnonzero((input_ids == tokenizer.sep_token_id) & mask.astype(bool)): + rows.append(int8_rows([np.rint(np.tanh(hidden[start:sep].mean(axis=0)) * 127)])[0]) + start = int(sep) + 1 + rows.append(int8_rows([np.rint(np.tanh(hidden[start:end].mean(axis=0)) * 127)])[0]) + return rows + + return session, encode + + +def self_max_rss_mb() -> float: + import resource + + value = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + return value / (1024.0 * 1024.0) if sys.platform == "darwin" else value / 1024.0 + + +def worker_main(args: argparse.Namespace) -> int: + inputs = json.loads(args.input_json) + start = time.perf_counter() + if args.reference == "sentence-transformers": + owner, encode = worker_sentencetransformers(args, inputs) + elif args.reference == "onnx": + owner, encode = worker_onnx(args, inputs) + else: + raise SystemExit(f"unsupported reference: {args.reference}") + load_ms = (time.perf_counter() - start) * 1000.0 + for _ in range(args.warmups): + encode() + vectors = None + elapsed = [] + for _ in range(args.runs): + start = time.perf_counter() + vectors = encode() + elapsed.append((time.perf_counter() - start) * 1000.0) + del owner + result = { + "name": args.reference, + "vectors": vectors, + "load_ms": load_ms, + "p50_ms": statistics.median(elapsed), + "p95_ms": percentile(elapsed, 95.0), + "rss_mb": self_max_rss_mb(), + } + print(RESULT_PREFIX + json.dumps(result, separators=(",", ":"))) + return 0 + + +def print_matrix(name: str, vectors: list[list[int]], row_labels: list[str]) -> None: + print(f"\n{name} cosine similarity matrix") + print(" " * 16 + " ".join(f"{label:>14}" for label in row_labels)) + for label, row in zip(row_labels, vectors): + scores = " ".join(f"{cosine(row, other):14.6f}" for other in vectors) + print(f"{label:<16}{scores}") + + +def print_summary(results: list[dict[str, Any]], row_labels: list[str]) -> None: + print("\nPerformance") + print(f"{'backend':<24} {'load ms':>10} {'p50 ms':>10} {'p95 ms':>10} {'peak RSS MiB':>14}") + for result in results: + load = "-" if result["load_ms"] is None else f"{result['load_ms']:.1f}" + rss = "-" if result["rss_mb"] is None else f"{result['rss_mb']:.1f}" + print( + f"{result['name']:<24} {load:>10} {result['p50_ms']:>10.1f} " + f"{result['p95_ms']:>10.1f} {rss:>14}" + ) + + print("\nParity") + baseline = results[0] + for result in results[1:]: + if len(result["vectors"]) != len(baseline["vectors"]): + raise RuntimeError( + f"{result['name']} returned {len(result['vectors'])} vectors; " + f"expected {len(baseline['vectors'])}" + ) + pairs = list(zip(baseline["vectors"], result["vectors"])) + if any(len(a) != len(b) for a, b in pairs): + raise RuntimeError(f"{result['name']} returned an unexpected dimension") + scores = [cosine(a, b) for a, b in pairs] + diffs = [max(abs(x - y) for x, y in zip(a, b)) for a, b in pairs] + print( + f"{baseline['name']} vs {result['name']}: " + f"min_cosine={min(scores):.8f} mean_cosine={statistics.mean(scores):.8f} " + f"max_int8_diff={max(diffs)}" + ) + + +def load_report(path: str, kind: str, inputs: Any) -> list[dict[str, Any]]: + report = json.loads(Path(path).read_text()) + if report.get("kind") != kind or report.get("inputs") != inputs: + raise RuntimeError("baseline JSON does not match the selected kind and inputs") + results = report.get("results") + if not isinstance(results, list) or not results: + raise RuntimeError("baseline JSON does not contain results") + return results + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--model", default="pplx-embed-v1-0.6b") + ap.add_argument("--kind", choices=("standard", "contextual"), default="standard") + ap.add_argument("--base-url", default="http://127.0.0.1:8000") + ap.add_argument("--native-mode", choices=("http", "stdin", "none"), default="http") + ap.add_argument("--native-backend", choices=("cpu", "mlx"), default="mlx") + ap.add_argument("--native-binary", default=str(repo_root() / "tools/cli/ffwd-cli")) + ap.add_argument("--max-native-rss-mb", type=float, default=32768.0) + ap.add_argument("--skip-native", action="store_true") + ap.add_argument("--server-pid", type=int) + ap.add_argument("--tei-url", help="optional TEI base URL; standard mode only") + ap.add_argument("--tei-pid", type=int) + ap.add_argument("--references", default="sentence-transformers,onnx") + ap.add_argument("--runs", type=int, default=3) + ap.add_argument("--warmups", type=int, default=1) + ap.add_argument("--threads", type=int, default=4) + ap.add_argument("--timeout", type=float, default=300.0) + ap.add_argument("--max-reference-rss-mb", type=float, default=6144.0) + ap.add_argument( + "--cache-dir", default=str(repo_root() / ".venv" / "reference-cache") + ) + ap.add_argument("--input-json") + ap.add_argument("--baseline-json", help="prepend results from an earlier sequential run") + ap.add_argument("--json-out", help="write inputs and measured results for a later run") + ap.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + ap.add_argument("--reference", help=argparse.SUPPRESS) + args = ap.parse_args() + + if args.runs < 1 or args.warmups < 0 or args.threads < 1: + raise SystemExit("runs and threads must be positive; warmups must be non-negative") + if args.worker: + return worker_main(args) + if args.tei_url and args.kind != "standard": + raise SystemExit("TEI comparison currently supports --kind standard only") + if args.skip_native: + args.native_mode = "none" + + args.inputs = json.loads(args.input_json) if args.input_json else ( + DEFAULT_STANDARD if args.kind == "standard" else DEFAULT_CONTEXTUAL + ) + references = [x.strip() for x in args.references.split(",") if x.strip()] + validate_artifacts(Path(args.model_dir).expanduser().resolve(), "onnx" in references) + row_labels = labels(args.kind, args.inputs) + results = load_report(args.baseline_json, args.kind, args.inputs) if args.baseline_json else [] + + if args.native_mode == "stdin": + results.append(benchmark_native_stdin(args)) + elif args.native_mode == "http": + endpoint = "/v1/embeddings" if args.kind == "standard" else "/v1/contextualizedembeddings" + results.append( + benchmark_http( + "native-http", + args.base_url.rstrip("/") + endpoint, + {"model": args.model, "input": args.inputs}, + lambda response: decode_native(response, args.kind), + args.warmups, + args.runs, + args.timeout, + args.server_pid, + ) + ) + for backend in references: + if backend not in ("sentence-transformers", "onnx"): + raise SystemExit(f"unsupported reference backend: {backend}") + results.append(run_guarded_worker(args, backend)) + if args.tei_url: + results.append( + benchmark_http( + "tei-http", + args.tei_url.rstrip("/") + "/embed", + {"inputs": args.inputs, "normalize": False}, + int8_rows, + args.warmups, + args.runs, + args.timeout, + args.tei_pid, + ) + ) + if not results: + raise SystemExit("select at least one implementation to measure") + + print(f"kind={args.kind} vectors={len(row_labels)} runs={args.runs} warmups={args.warmups}") + for result in results: + print_matrix(result["name"], result["vectors"], row_labels) + print_summary(results, row_labels) + if args.json_out: + Path(args.json_out).write_text( + json.dumps( + {"kind": args.kind, "inputs": args.inputs, "results": results}, + indent=2, + ) + + "\n" + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/tools/convert_model_bf16.py b/devtools/convert_model_bf16.py index d59ccf2..d59ccf2 100755 --- a/tools/convert_model_bf16.py +++ b/devtools/convert_model_bf16.py diff --git a/devtools/gen_exports.py b/devtools/gen_exports.py new file mode 100755 index 0000000..215c45e --- /dev/null +++ b/devtools/gen_exports.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Generate the public-ABI linker export map for libffwd from the header. + +The shared library is built with -fvisibility=hidden, so the only symbols that +leave it are the declarations marked FFWD_API in libffwd/ffwd.h. This script +reads those annotations and writes the two linker inputs the build consumes: + + ffwd.exports Mach-O (-Wl,-exported_symbols_list): one _symbol per line + ffwd.map ELF (-Wl,--version-script): FFWD { global: ...; local: *; } + +Run it as a release step when the libffwd public API changes: + + python3 devtools/gen_exports.py # write ffwd.exports and ffwd.map + python3 devtools/gen_exports.py --check # verify they are in sync (CI), no write + python3 devtools/gen_exports.py --outdir D # write somewhere else +""" +import argparse +import os +import re +import sys + + +def find_symbols(header_text): + # Drop comments and preprocessor lines so the FFWD_API macro definition + # (#ifndef/#define FFWD_API ...) and doc comments are not mistaken for + # declarations. + text = re.sub(r"/\*.*?\*/", "", header_text, flags=re.S) + text = re.sub(r"//[^\n]*", "", text) + text = "\n".join(ln for ln in text.splitlines() if not ln.lstrip().startswith("#")) + + syms = set() + for m in re.finditer(r"\bFFWD_API\b(.*?)([(;])", text, flags=re.S): + ids = re.findall(r"[A-Za-z_][A-Za-z0-9_]*", m.group(1)) + if ids and ids[-1].startswith("ffwd_"): + syms.add(ids[-1]) + return sorted(syms) + + +def render_exports(syms): + return "".join("_%s\n" % s for s in syms) + + +def render_map(syms): + body = "\n".join(" %s;" % s for s in syms) + return "FFWD {\n global:\n%s\n local:\n *;\n};\n" % body + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + root = os.path.dirname(here) + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--header", default=os.path.join(root, "libffwd", "ffwd.h")) + ap.add_argument("--outdir", default=root, help="directory for ffwd.exports / ffwd.map") + ap.add_argument("--check", action="store_true", help="verify files are in sync; do not write") + args = ap.parse_args() + + with open(args.header) as f: + syms = find_symbols(f.read()) + if not syms: + sys.exit("error: no FFWD_API symbols found in %s" % args.header) + + targets = ( + (os.path.join(args.outdir, "ffwd.exports"), render_exports(syms)), + (os.path.join(args.outdir, "ffwd.map"), render_map(syms)), + ) + + if args.check: + stale = False + for path, want in targets: + try: + have = open(path).read() + except FileNotFoundError: + have = None + if have != want: + print("out of sync: %s" % path) + stale = True + if stale: + sys.exit("run: python3 %s" % os.path.relpath(__file__, os.getcwd())) + print("in sync (%d symbols)" % len(syms)) + return + + for path, want in targets: + with open(path, "w") as f: + f.write(want) + print("wrote ffwd.exports and ffwd.map (%d symbols)" % len(syms)) + + +if __name__ == "__main__": + main() diff --git a/devtools/gen_test_vectors.py b/devtools/gen_test_vectors.py new file mode 100644 index 0000000..44206ec --- /dev/null +++ b/devtools/gen_test_vectors.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Generate tokenizer parity vectors for the hermetic tokenizer checks. + +tests/check_tokenizer_parity.py and tests/check_wordpiece_parity.py read +the stored vectors generated by this script. Regenerate the vectors when +the test corpus changes or the reference tokenizer is bumped, then check +the result into git. + +For each tokenizer family it writes, under tests/test-vectors/<family>/: + - the vocab files the C driver loads (vocab.json + merges.txt for BPE; + vocab.txt + tokenizer_config.json for WordPiece), + - expected.json: provenance plus a list of {text, ids} cases. + +- The BPE reference is the fast Qwen tokenizer (what the C BPE backend +targets). +- The WordPiece reference is the slow BertTokenizer (the exact reference +for the C WordPiece backend). +- Both encode with add_special_tokens=False, matching the C drivers, +which emit core ids and leave [CLS]/[SEP] to the frontend. + + uv run --with transformers devtools/gen_test_vectors.py + uv run --with transformers devtools/gen_test_vectors.py --family bpe + uv run --with transformers devtools/gen_test_vectors.py --bpe-model DIR --fuzz 128 +""" + +import argparse +import json +import random +import re +import shutil +import sys +from pathlib import Path + +# BPE corpus: ASCII, accents, decomposed Unicode, emoji, digits, and long +# repeated strings that stress the merge loop. +BPE_TEXTS = [ + "", + "hello world", + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "punctuation: hello, world!!! (test)", + "repeated spaces\tand\nnewlines", + "unicode: cafe, caf\xe9, Istanbul, İstanbul", + "unicode decomposed: café, façade", + "emoji: embeddings \U0001f50d are useful", + "numbers: 12345 67.89 -0.12", + "a " * 256, + "antidisestablishmentarianism" * 64, + ("document: " + "dense retrieval systems need stable tokenization. " * 256), +] + +BPE_FUZZ_ALPHABET = ( + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + " \t\n\r" + ".,!?;:()[]{}'\"-/" + "éèêëáàâäíìîïóòôöúùûüñç" + "¿¡«»—…" + "中文测试Русскийعربيहिन्दी" + "🔍👍🏽🇺🇸" +) + +# WordPiece corpus: stays within the implemented coverage (ASCII, Latin-1 +# accents, common punctuation, CJK char-splitting) plus a multilingual set. +WORDPIECE_TEXTS = [ + "", + "hello world", + "The capital of France is Paris.", + "Tokenization, sub-words, and WordPiece!", + "unaffable antidisestablishmentarianism", + "café résumé naïve Zürich Málaga", + "COVID-19 affected 2020-2021 globally.", + "don't can't won't it's o'clock", + "e-mail user@example.com or http://test.org/path?q=1", + "numbers: 12345 67.89 -0.12 3.14159 (approx)", + "repeated spaces\tand\nnewlines\rhere", + "Hyphenated state-of-the-art well-known co-operate", + "MixedCase camelCase snake_case ALLCAPS", + "中 文 字 符", + "a " * 200, + "supercalifragilisticexpialidocious pneumonoultramicroscopicsilicovolcanoconiosis", + "Müller wohnt in Zürich, Straße.", + "L'élève préfère — n'est-ce pas?", + "¿Cómo estás? ¡Muy bien!", + "你好,世界。こんにちは", +] + +WORDPIECE_FUZZ_ALPHABET = ( + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + " \t\n\r" + ".,!?;:()[]{}'\"-/@#" + "éèêëáàâäíìîïóòôöúùûüñçßæøþ" +) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[1] + + +def transformers_version() -> str: + import transformers + return transformers.__version__ + + +def snapshot_revision(local_path: str) -> str: + """Best-effort: pull the commit hash from an HF cache path, else ''.""" + m = re.search(r"/snapshots/([0-9a-f]{8,})/", local_path + "/") + return m.group(1) if m else "" + + +def resolve_file(model: str, filename: str) -> Path: + """Return a local path to one of the model's files, from a local dir or the + HF cache (download only if absent).""" + local = Path(model) / filename + if local.exists(): + return local + from huggingface_hub import hf_hub_download + return Path(hf_hub_download(model, filename)) + + +def fuzz_cases(alphabet: str, count: int, seed: int) -> list[str]: + rng = random.Random(seed) + out = [] + for _ in range(count): + n = rng.randint(0, 96) + out.append("".join(rng.choice(alphabet) for _ in range(n))) + return out + + +def write_vectors(out_dir: Path, family: str, model: str, revision: str, + texts: list[str], encode) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + cases = [{"text": t, "ids": encode(t)} for t in texts] + payload = { + "family": family, + "model": model, + "revision": revision, + "transformers_version": transformers_version(), + "add_special_tokens": False, + "cases": cases, + } + (out_dir / "expected.json").write_text( + json.dumps(payload, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") + print(f"ok: wrote {len(cases)} {family} cases to {out_dir}") + + +def gen_bpe(root: Path, model: str, fuzz: int, seed: int) -> None: + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(model, trust_remote_code=True) + out_dir = root / "tests/test-vectors/bpe" + out_dir.mkdir(parents=True, exist_ok=True) + # Copy the shipped vocab.json + merges.txt the C BPE loader reads. The fast + # tokenizer's save_vocabulary does not re-emit them, so copy the originals + # the reference tokenizer was built from. + src = "" + for fn in ("vocab.json", "merges.txt"): + path = resolve_file(model, fn) + src = str(path) + shutil.copyfile(path, out_dir / fn) + texts = list(BPE_TEXTS) + fuzz_cases(BPE_FUZZ_ALPHABET, fuzz, seed) + revision = snapshot_revision(src) + write_vectors(out_dir, "bpe", model, revision, texts, + lambda t: tok.encode(t, add_special_tokens=False)) + + +def gen_wordpiece(root: Path, model: str, fuzz: int, seed: int) -> None: + from transformers import BertTokenizer + tok = BertTokenizer.from_pretrained(model) + out_dir = root / "tests/test-vectors/wordpiece" + out_dir.mkdir(parents=True, exist_ok=True) + # Copy vocab.txt and write the tokenizer_config.json the C loader reads for + # do_lower_case (save_pretrained would emit tokenizer.json, which the C + # side ignores). + src = resolve_file(model, "vocab.txt") + shutil.copyfile(src, out_dir / "vocab.txt") + lower = "true" if tok.do_lower_case else "false" + (out_dir / "tokenizer_config.json").write_text( + '{"do_lower_case": %s}\n' % lower, encoding="utf-8") + texts = list(WORDPIECE_TEXTS) + fuzz_cases(WORDPIECE_FUZZ_ALPHABET, fuzz, seed) + revision = snapshot_revision(str(src)) + write_vectors(out_dir, "wordpiece", model, revision, texts, + lambda t: tok.encode(t, add_special_tokens=False)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--family", choices=("bpe", "wordpiece", "both"), default="both") + ap.add_argument("--bpe-model", default="Qwen/Qwen3-Embedding-0.6B", + help="HF name or local dir for the BPE reference tokenizer") + ap.add_argument("--wordpiece-model", default="bert-base-uncased", + help="HF name or local dir for the WordPiece reference tokenizer") + ap.add_argument("--fuzz", type=int, default=64, + help="deterministic random cases to bake in per family") + ap.add_argument("--seed", type=int, default=1) + args = ap.parse_args() + + root = repo_root() + if args.family in ("bpe", "both"): + gen_bpe(root, args.bpe_model, args.fuzz, args.seed) + if args.family in ("wordpiece", "both"): + gen_wordpiece(root, args.wordpiece_model, args.fuzz, args.seed) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py index 24649f1..2f8642e 100644..100755 --- a/tools/gen_wordpiece_unicode.py +++ b/devtools/gen_wordpiece_unicode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece +"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece BasicTokenizer's punctuation and control classification, matching transformers' BertTokenizer._is_punctuation and _is_control exactly. @@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. Run when bumping the target Unicode version (the C side is data-only otherwise): - uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h + uv run python devtools/gen_wordpiece_unicode.py > unicode.h """ import unicodedata @@ -94,7 +94,7 @@ def main(): for f in [fold(cp)] if f != [cp] ] - print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata + print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata * (Unicode {unicodedata.unidata_version}). Do not edit by hand. * * Tables for the BasicTokenizer, matching transformers' BertTokenizer. diff --git a/devtools/inspect_model_artifacts.py b/devtools/inspect_model_artifacts.py new file mode 100755 index 0000000..0265832 --- /dev/null +++ b/devtools/inspect_model_artifacts.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Inspect and detect local model snapshots, across model families. + +For each model it detects: + - the architecture (from config.json: model_type / architectures / dims), + - the tokenizer family - BPE (vocab.json + merges.txt), WordPiece (vocab.txt), + or SentencePiece (sentencepiece.bpe.model / spiece.model / tokenizer.model), + - the weight layout - single safetensors, sharded safetensors, or legacy + pytorch_model.bin, + - the serving kind - base encoder, pooled sentence-embedding, or projected / + late-interaction (1_Dense) - plus the Sentence-Transformers pipeline. + +ONNX exports and other extras are reported when present. + +A MODEL_DIR argument may be a model directory (one with config.json) or any +directory to search. Point it at a single snapshot, a folder of models, or the +whole Hugging Face hub cache (~/.cache/huggingface/hub) and it finds each model. + +A snapshot is "ok" when it has config.json, some weights, and some tokenizer; +everything else is informational. + +Usage: inspect_model_artifacts.py MODEL_DIR [MODEL_DIR ...] +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +TOKENIZER_ARTIFACTS = ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "vocab.txt", + "sentencepiece.bpe.model", + "spiece.model", + "tokenizer.model", +) + +# The first family whose required files are all present wins. +TOKENIZER_FAMILIES = ( + ("BPE", ("vocab.json", "merges.txt")), + ("WordPiece", ("vocab.txt",)), + ("SentencePiece", ("sentencepiece.bpe.model",)), + ("SentencePiece", ("spiece.model",)), + ("SentencePiece", ("tokenizer.model",)), +) + +EXTRA_ARTIFACTS = ( + "config_sentence_transformers.json", + "sentence_bert_config.json", + "modules.json", + "1_Pooling", + "1_Dense", + "onnx", + "generation_config.json", +) + + +def human_bytes(size: int) -> str: + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if value < 1024.0 or unit == "TiB": + return f"{value:.1f} {unit}" + value /= 1024.0 + return f"{value:.1f} TiB" + + +def describe(path: Path) -> str: + if not path.exists(): + return "MISSING" + if path.is_dir(): + return f"{sum(1 for _ in path.iterdir())} entries" + return human_bytes(path.resolve().stat().st_size) + + +def fmt(label: str, path: Path, indent: int = 4, width: int = 32) -> str: + return f"{' ' * indent}{label:<{width}} {describe(path)}" + + +def is_model_dir(d: Path) -> bool: + return (d / "config.json").is_file() + + +def discover(root: Path) -> list[Path]: + """Return the model directories at or under `root`. + + A model directory has a config.json. If `root` is itself one, return it; + otherwise search descendants (e.g. a folder of models or the HF hub cache), + skipping onnx sub-exports and dropping any dir nested under another model. + """ + root = root.expanduser().resolve() + if is_model_dir(root): + return [root] + found = { + cfg.parent + for cfg in root.rglob("config.json") + if cfg.parent.name != "onnx" + } + out = [] + for d in sorted(found): + if not any(d != o and str(d).startswith(str(o) + os.sep) for o in found): + out.append(d) + return out + + +def tokenizer_family(d: Path) -> str: + for name, required in TOKENIZER_FAMILIES: + if all((d / f).exists() for f in required): + return name + if (d / "tokenizer.json").exists(): + return "fast-only (tokenizer.json)" + return "none" + + +def config_summary(d: Path) -> str | None: + cfg = d / "config.json" + if not cfg.exists(): + return None + try: + data = json.loads(cfg.read_text()) + except (OSError, ValueError): + return "config.json: unreadable" + parts = [] + if data.get("model_type"): + parts.append(str(data["model_type"])) + arch = data.get("architectures") or [] + if arch: + parts.append(str(arch[0])) + for key in ("hidden_size", "num_hidden_layers", "num_attention_heads", + "num_key_value_heads", "vocab_size", "max_position_embeddings"): + if key in data: + parts.append(f"{key}={data[key]}") + return " ".join(parts) if parts else "config.json: no recognized fields" + + +def modules_pipeline(d: Path) -> str | None: + mj = d / "modules.json" + if not mj.exists(): + return None + try: + mods = json.loads(mj.read_text()) + except (OSError, ValueError): + return None + names = [(m.get("type") or "").rsplit(".", 1)[-1] or m.get("name") or "?" + for m in mods] + return " -> ".join(n for n in names if n) or None + + +def model_kind(d: Path) -> str: + if (d / "1_Dense").is_dir(): + return "projected / late-interaction (1_Dense)" + if (d / "1_Pooling").is_dir() or (d / "modules.json").exists(): + return "pooled sentence-embedding" + return "base encoder / LM" + + +def weight_lines(d: Path) -> tuple[list[str], bool]: + """Return (formatted lines, whether any weights were found).""" + lines: list[str] = [] + st_index = d / "model.safetensors.index.json" + st_shards = sorted(d.glob("model-*-of-*.safetensors")) + pt_index = d / "pytorch_model.bin.index.json" + pt_shards = sorted(d.glob("pytorch_model-*-of-*.bin")) + onnx = sorted((d / "onnx").glob("*.onnx")) if (d / "onnx").is_dir() else [] + found = False + + if (d / "model.safetensors").exists(): + lines.append(fmt("model.safetensors", d / "model.safetensors")) + found = True + if st_index.exists() and st_shards: + lines.append(fmt("model.safetensors.index.json", st_index)) + lines += [fmt(s.name, s, indent=6) for s in st_shards] + found = True + if (d / "pytorch_model.bin").exists(): + lines.append(fmt("pytorch_model.bin", d / "pytorch_model.bin")) + found = True + if pt_index.exists() and pt_shards: + lines.append(fmt("pytorch_model.bin.index.json", pt_index)) + lines += [fmt(s.name, s, indent=6) for s in pt_shards] + found = True + if onnx: + lines += [fmt(f"onnx/{o.name}", o) for o in onnx] + found = True + if not found: + lines.append(" (no safetensors / pytorch / onnx weights)") + return lines, found + + +def inspect(model_dir: Path) -> bool: + d = model_dir.expanduser().resolve() + print(f"snapshot: {d}") + if not d.is_dir(): + print(" NOT A DIRECTORY") + return False + + print(f" model: {config_summary(d) or 'config.json MISSING'}") + print(f" kind: {model_kind(d)}") + pipeline = modules_pipeline(d) + if pipeline: + print(f" pipeline: {pipeline}") + + family = tokenizer_family(d) + print(f" tokenizer: {family}") + for name in TOKENIZER_ARTIFACTS: + if (d / name).exists(): + print(fmt(name, d / name)) + + print(" weights:") + wlines, has_weights = weight_lines(d) + print("\n".join(wlines)) + + extras = [name for name in EXTRA_ARTIFACTS if (d / name).exists()] + if extras: + print(" extras:") + for name in extras: + print(fmt(name, d / name)) + + ok = (d / "config.json").exists() and has_weights and family != "none" + print(f" verdict: {'ok' if ok else 'incomplete'}") + return ok + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR") + args = ap.parse_args() + + ok = True + first = True + for name in args.model_dirs: + root = Path(os.path.expandvars(name)).expanduser().resolve() + found = discover(root) + if not found: + if not first: + print() + print(f"snapshot: {root}\n no config.json found at or under this path") + ok = first = False + continue + if len(found) > 1: + if not first: + print() + print(f"# {root}: {len(found)} models") + first = False + for d in found: + if not first: + print() + ok = inspect(d) and ok + first = False + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/openapi.yaml b/devtools/openapi.yaml index 6dd3818..5140186 100644 --- a/tools/openapi.yaml +++ b/devtools/openapi.yaml @@ -1,9 +1,9 @@ openapi: 3.0.3 info: - title: embed.c server API + title: ffwd server API version: "1.0.0" description: > - HTTP API served by `embed-server`. Each loaded model has an operator-chosen + HTTP API served by `ffwd-server`. Each loaded model has an operator-chosen label; request `model` fields match that label. Embedding encoding follows the serving API selected at load time (`api=openai` or `api=perplexity`). Always compare embeddings with cosine similarity. @@ -12,7 +12,7 @@ info: servers: - url: http://127.0.0.1:8000 # Auth is optional: it applies only when the server is started with --api-key -# (or EMBED_API_KEY). The empty requirement allows unauthenticated calls. +# (or FFWD_API_KEY). The empty requirement allows unauthenticated calls. security: - {} - bearerAuth: [] @@ -86,7 +86,7 @@ components: properties: model: type: string - description: Served label chosen when starting `embed-server`. + description: Served label chosen when starting `ffwd-server`. example: qwen input: description: A single string or an array of up to 512 strings. diff --git a/devtools/quant_precision_drift.py b/devtools/quant_precision_drift.py new file mode 100755 index 0000000..896d2f8 --- /dev/null +++ b/devtools/quant_precision_drift.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Compare f32 -> low-precision int8-output drift across engines. + +The native BF16 path changes ~10-15% of int8 output bytes versus F32 (while +keeping cosine ~0.9999 and identical retrieval). This script checks whether that +is normal by measuring the *same* f32 -> low-precision drift in independent +implementations of the same model: + + - ONNX : released model.onnx (F32) vs model_quantized.onnx (official int8) + - HF : sentence-transformers / torch, loaded in F32 vs BF16 + - native : ffwd binary, an F32 model dir vs a BF16 model dir + +For every engine it reports, per low-precision variant against that engine's own +F32 output: the int8 byte-change rate, the int8-vector cosine, and the float +relative-L2 diff. If ONNX-int8 and HF-bf16 drift like ours, the drift is inherent +to low precision, not a backend defect. + +Reference engines output int8-scale embeddings (round); native outputs raw +floats quantized with tanh*127 (matching the server's base64_int8 encoder). + +Needs the reference stack, so run it with uv (Python 3.12 for the numpy pin): + + uv run --python 3.12 --with-requirements requirements-reference.txt \ + devtools/quant_precision_drift.py --model-dir DIR --engines hf +""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +DEFAULT_TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: The Eiffel Tower is a famous landmark located in Paris, France.", + "document: Photosynthesis converts sunlight into chemical energy in plants.", + ( + "document: Embedding models turn text into dense vectors for semantic " + "retrieval. Long document inference stresses bidirectional attention " + "because every token attends to every other token in the same packed " + "sequence. " + ) * 8, +] + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + if na == 0.0 or nb == 0.0: + return 1.0 + return dot / (na * nb) + + +def rel_l2(a: list[float], b: list[float]) -> float: + num = math.sqrt(sum((x - y) ** 2 for x, y in zip(a, b))) + den = math.sqrt(sum(x * x for x in a)) + return num / den if den else 0.0 + + +def drift(ref: list[list[int]], low: list[list[int]]) -> dict[str, float]: + if len(ref) != len(low): + raise RuntimeError(f"row count mismatch: {len(ref)} vs {len(low)}") + total = 0 + changed = 0 + worst_cos = 1.0 + worst_l2 = 0.0 + for a, b in zip(ref, low): + if len(a) != len(b): + raise RuntimeError(f"dim mismatch: {len(a)} vs {len(b)}") + total += len(a) + changed += sum(1 for x, y in zip(a, b) if x != y) + worst_cos = min(worst_cos, cosine([float(x) for x in a], + [float(y) for y in b])) + worst_l2 = max(worst_l2, rel_l2([float(x) for x in a], + [float(y) for y in b])) + return { + "int8_change_rate": changed / total if total else 0.0, + "worst_int8_cosine": worst_cos, + "worst_rel_l2": worst_l2, + } + + +# --- engines ------------------------------------------------------------- + +def onnx_int8(model_dir: str, onnx_file: str, texts: list[str]) -> list[list[int]]: + import numpy as np + import onnxruntime as ort + from transformers import AutoTokenizer + + tok = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, + local_files_only=True) + sess = ort.InferenceSession(str(Path(model_dir) / "onnx" / onnx_file), + providers=["CPUExecutionProvider"]) + t = tok(texts, padding=True, truncation=True, return_tensors="np") + feed = {"input_ids": t["input_ids"].astype(np.int64), + "attention_mask": t["attention_mask"].astype(np.int64)} + # outputs: [0]=hidden states, [2]=pooled int8-scale embedding (see + # compare_reference_backends.worker_onnx). + out = sess.run(None, feed)[2] + return [[int(round(float(x))) for x in row] for row in out] + + +def hf_int8(model_dir: str, texts: list[str], dtype: str) -> list[list[int]]: + import torch + from sentence_transformers import SentenceTransformer + + td = {"f32": torch.float32, "bf16": torch.bfloat16}[dtype] + model = SentenceTransformer(model_dir, trust_remote_code=True, + local_files_only=True, + model_kwargs={"torch_dtype": td}) + out = model.encode(texts, show_progress_bar=False) + return [[int(round(float(x))) for x in row] for row in out.tolist()] + + +def native_int8(binary: str, model_dir: str, backend: str, + texts: list[str]) -> list[list[int]]: + cmd = [binary, "-d", model_dir, f"--{backend}", "--stream", "-b", "8"] + proc = subprocess.run(cmd, input="\n".join(texts) + "\n", text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise RuntimeError(f"native failed: {' '.join(cmd)}\n{proc.stderr[-800:]}") + rows = [] + for line in proc.stdout.splitlines(): + if not line.strip(): + continue + emb = json.loads(line)["embedding"] + rows.append([max(-128, min(127, round(math.tanh(float(x)) * 127.0))) + for x in emb]) + return rows + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--model-dir", required=True, + help="F32 reference model dir (HF snapshot with onnx/)") + ap.add_argument("--engines", default="onnx,hf", + help="comma list: onnx,hf,native") + ap.add_argument("--native-binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--native-backend", default="mlx", choices=("cpu", "mlx")) + ap.add_argument("--native-bf16-dir", + help="BF16 model dir for the native engine (required for native)") + ap.add_argument("--onnx-low", default="model_quantized.onnx", + help="quantized ONNX file (vs model.onnx)") + args = ap.parse_args() + + texts = DEFAULT_TEXTS + engines = [e for e in args.engines.split(",") if e] + results: list[tuple[str, str, dict[str, float]]] = [] + + for eng in engines: + try: + if eng == "onnx": + ref = onnx_int8(args.model_dir, "model.onnx", texts) + low = onnx_int8(args.model_dir, args.onnx_low, texts) + results.append(("ONNX", f"f32 vs {args.onnx_low}", drift(ref, low))) + elif eng == "hf": + ref = hf_int8(args.model_dir, texts, "f32") + low = hf_int8(args.model_dir, texts, "bf16") + results.append(("HF/torch", "f32 vs bf16", drift(ref, low))) + elif eng == "native": + if not args.native_bf16_dir: + raise RuntimeError("--native-bf16-dir required for native") + ref = native_int8(args.native_binary, args.model_dir, + args.native_backend, texts) + low = native_int8(args.native_binary, args.native_bf16_dir, + args.native_backend, texts) + results.append((f"native/{args.native_backend}", "f32 vs bf16", + drift(ref, low))) + else: + print(f"unknown engine: {eng}", file=sys.stderr) + except Exception as exc: # one engine failing should not block the rest + print(f" [{eng}] SKIPPED: {type(exc).__name__}: {exc}", + file=sys.stderr) + + print(f"\nf32 -> low-precision int8 drift ({len(texts)} texts)\n") + print(f"{'engine':14s} {'comparison':24s} {'int8_change':>11s} " + f"{'int8_cosine':>12s} {'rel_l2':>8s}") + print("-" * 74) + for name, comp, d in results: + print(f"{name:14s} {comp:24s} {d['int8_change_rate']*100:10.1f}% " + f"{d['worst_int8_cosine']:12.5f} {d['worst_rel_l2']*100:7.2f}%") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rank_embeddings.py b/devtools/rank_embeddings.py index ddc1e42..efc8aad 100755 --- a/tools/rank_embeddings.py +++ b/devtools/rank_embeddings.py @@ -2,7 +2,7 @@ """Rank stdin JSON embeddings by cosine similarity to a query row. Usage: - ./embed ... --stream <<'EOF' | tools/rank_embeddings.py + ./tools/ffwd-cli ... --stream <<'EOF' | ./devtools/rank_embeddings.py query: what is the capital of France? document: Paris is the capital of France. document: Berlin is the capital of Germany. diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py deleted file mode 100755 index 2feabe2..0000000 --- a/tools/inspect_model_artifacts.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect local pplx-embed snapshots.""" - -from __future__ import annotations - -import argparse -import os -from pathlib import Path - - -PRIMARY_FILES = ( - "config.json", - "tokenizer_config.json", - "tokenizer.json", - "vocab.json", - "merges.txt", -) - - -def human_bytes(size: int) -> str: - units = ("B", "KiB", "MiB", "GiB", "TiB") - value = float(size) - for unit in units: - if value < 1024.0 or unit == units[-1]: - return f"{value:.1f} {unit}" - value /= 1024.0 - raise AssertionError("unreachable") - - -def describe(path: Path) -> str: - if not path.exists(): - return "MISSING" - resolved = path.resolve() - suffix = "" - if path.is_symlink(): - suffix = f" -> {resolved}" - return f"{human_bytes(resolved.stat().st_size):>10}{suffix}" - - -def inspect(model_dir: Path) -> bool: - model_dir = model_dir.expanduser().resolve() - print(f"snapshot: {model_dir}") - ok = True - for name in PRIMARY_FILES: - path = model_dir / name - print(f" {name:<30} {describe(path)}") - ok = path.exists() and ok - - model = model_dir / "model.safetensors" - index = model_dir / "model.safetensors.index.json" - shards = sorted(model_dir.glob("model-*-of-*.safetensors")) - if model.exists(): - print(f" {'model.safetensors':<30} {describe(model)}") - elif index.exists() and shards: - print(f" {'model.safetensors.index.json':<30} {describe(index)}") - for path in shards: - print(f" {path.name:<28} {describe(path)}") - else: - print(f" {'model.safetensors':<30} MISSING") - ok = False - - onnx_dir = model_dir / "onnx" - onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else [] - print(" onnx/") - if not onnx_files: - print(" MISSING") - return False - for path in onnx_files: - print(f" {path.name:<28} {describe(path)}") - ok = path.exists() and ok - return ok - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR") - args = ap.parse_args() - - ok = True - for i, name in enumerate(args.model_dirs): - if i: - print() - ok = inspect(Path(os.path.expandvars(name))) and ok - return 0 if ok else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) |
