From a09b41dc4824eb33e11b1e71d95ec94d2411ead8 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sun, 21 Jun 2026 13:36:12 +0200 Subject: build: source all.bash checks from tests, drop precision-drift tool --- all.bash | 12 +-- devtools/check_code_quality.sh | 182 ----------------------------------- devtools/compare_model_embeddings.py | 152 ----------------------------- tests/check_code_quality.sh | 182 +++++++++++++++++++++++++++++++++++ tests/check_embedding_parity.py | 152 +++++++++++++++++++++++++++++ 5 files changed, 337 insertions(+), 343 deletions(-) delete mode 100755 devtools/check_code_quality.sh delete mode 100755 devtools/compare_model_embeddings.py create mode 100755 tests/check_code_quality.sh create mode 100755 tests/check_embedding_parity.py diff --git a/all.bash b/all.bash index cd8cd72..3c8f7e1 100755 --- a/all.bash +++ b/all.bash @@ -35,7 +35,7 @@ err() { printf '%sERROR: %s%s\n' "$RED" "$*" "$NORMAL" >&2; } _ALL_CMDS="help checks check-code-quality check-sanitize check-kernel-golden check-batch-parity \ check-context-batch-parity check-late-interaction check-cuda-fast-gemm-parity \ check-embedding-parity check-mlx-model-limits check-mlx-quantized-parity \ -check-precision-drift check-reference-parity check-bert-parity check-e5-parity \ +check-reference-parity check-bert-parity check-e5-parity \ check-qwen3-parity check-tokenizer-parity check-wordpiece-parity check-model-matrix \ check-server-api check-cli check-workspace-api check-perplexity-sdk-compat \ check-openai-sdk-compat bench-contextual-api bench-late-rerank bench-late-api \ @@ -82,7 +82,7 @@ run_reference() { } check_code_quality() { - run ./devtools/check_code_quality.sh "$@" + run ./tests/check_code_quality.sh "$@" } # The dynamic-analysis counterpart to check-code-quality (which is static only: @@ -271,7 +271,7 @@ check_cuda_fast_gemm_parity() { } check_embedding_parity() { - run_uv ./devtools/compare_model_embeddings.py "$@" + run_uv ./tests/check_embedding_parity.py "$@" } check_mlx_model_limits() { @@ -290,10 +290,6 @@ check_mlx_quantized_parity() { run_uv ./tests/check_mlx_quantized_parity.py "$@" } -check_quant_precision_drift() { - run_reference ./devtools/quant_precision_drift.py "$@" -} - check_reference_parity() { run_reference ./tests/check_reference_parity.py "$@" } @@ -396,7 +392,6 @@ usage() { ┃ check-late-interaction Check late-interaction vectors ┃ ┃ check-mlx-model-limits Check MLX metadata rejection ┃ ┃ check-mlx-quantized-parity Compare quantized embeddings ┃ - ┃ check-precision-drift Compare low-precision drift ┃ ┃ check-reference-parity Compare server vs Python ref ┃ ┃ check-bert-parity Compare BERT embeddings ┃ ┃ check-e5-parity Compare XLM-R/E5 embeddings ┃ @@ -444,7 +439,6 @@ main() { check-embedding-parity) check_embedding_parity "$@" ;; check-mlx-model-limits) check_mlx_model_limits "$@" ;; check-mlx-quantized-parity) check_mlx_quantized_parity "$@" ;; - check-precision-drift) check_quant_precision_drift "$@" ;; check-reference-parity) check_reference_parity "$@" ;; check-bert-parity) check_bert_parity "$@" ;; check-e5-parity) check_e5_parity "$@" ;; diff --git a/devtools/check_code_quality.sh b/devtools/check_code_quality.sh deleted file mode 100755 index 3c81d92..0000000 --- a/devtools/check_code_quality.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" -cd "$ROOT" - -# Lint and analyze every C source under libffwd/, tests/, and bench/. Each -# platform keeps its own GPU backend and skips the other's, which it does not -# build (macOS builds MLX, Linux builds CUDA). -SYS="$(uname -s)" -SRC_SOURCES=() -for f in libffwd/*.c; do - [[ "$SYS" == Darwin && "$f" == libffwd/backend_cuda.c ]] && continue - [[ "$SYS" != Darwin && ( "$f" == libffwd/mlx.c || "$f" == libffwd/backend_mlx.c ) ]] && continue - SRC_SOURCES+=("$f") -done - -TEST_C_SOURCES=(tests/*.c) -BENCH_C_SOURCES=(bench/*.c) - -# CUDA backend sources are C++17 and only built on Linux. Parse them with the -# clang CUDA front end in host-only mode so no GPU or device toolchain is -# required; macOS has no CUDA toolkit, so skip them there. -CU_SOURCES=() -if [[ "$SYS" != Darwin ]]; then - for f in libffwd/*.cu; do - [[ -e "$f" ]] && CU_SOURCES+=("$f") - done -fi - -ANALYZE_SOURCES=("${SRC_SOURCES[@]}") -TIDY_SOURCES=("${SRC_SOURCES[@]}" "${TEST_C_SOURCES[@]}" "${BENCH_C_SOURCES[@]}") - -ANALYZE_COMMON=( - clang --analyze -Xanalyzer -analyzer-output=text - -Wall -Wextra - -I. -Ilibffwd -Ideps/ae -) - -PLATFORM_CFLAGS=() -if [[ "$(uname -s)" == "Darwin" ]]; then - PLATFORM_CFLAGS=( - -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX - -I/opt/homebrew/opt/mlx-c/include - -I/opt/homebrew/include - -I/usr/local/include - ) -else - PLATFORM_CFLAGS=( - -DUSE_BLAS -DUSE_OPENBLAS - -I/usr/include/openblas - ) -fi - -# Flags for the CUDA backend front end. CUDA_HOME matches the Makefile default. -# --cuda-host-only parses host code only so device codegen and libdevice are not -# needed; the macros mirror the nvcc command in the Makefile. -CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" -CUDA_CFLAGS=( - -x cuda --cuda-host-only "--cuda-path=$CUDA_HOME" - -std=c++17 - -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU - -I"$CUDA_HOME/include" - -I/usr/include/openblas -) - -print_cmd() { - printf '+' - printf ' %q' "$@" - printf '\n' -} - -run() { - print_cmd "$@" - "$@" -} - -run_analyze() { - local output - local status - - print_cmd "$@" - - set +e - output="$("$@" 2>&1)" - status=$? - set -e - - if [[ -n "$output" ]]; then - printf '%s\n' "$output" - fi - - if (( status != 0 )); then - return "$status" - fi - - if grep -q 'warning:' <<<"$output"; then - echo "clang analyzer warnings found" >&2 - return 1 - fi -} - -have() { - command -v "$1" >/dev/null 2>&1 -} - -find_clang_tidy() { - local brew_path="/opt/homebrew/opt/llvm@21/bin/clang-tidy" - - if command -v clang-tidy >/dev/null 2>&1; then - command -v clang-tidy - return 0 - fi - - if [[ -x "$brew_path" ]]; then - printf '%s\n' "$brew_path" - return 0 - fi - - return 1 -} - -main() { - local clang_tidy - - if ! have clang; then - echo "clang not found" >&2 - return 1 - fi - - run git --no-pager diff --check - run make cpu CC=clang - - run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ - "${ANALYZE_SOURCES[@]}" - run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ - -I. -Ilibffwd -Itests "${TEST_C_SOURCES[@]}" - run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ - -I. -Ilibffwd -Ibench -Itools/server "${BENCH_C_SOURCES[@]}" - - if (( ${#CU_SOURCES[@]} > 0 )); then - run_analyze "${ANALYZE_COMMON[@]}" "${CUDA_CFLAGS[@]}" \ - "${CU_SOURCES[@]}" - fi - - if clang_tidy="$(find_clang_tidy)"; then - run \ - "$clang_tidy" \ - --config-file=.clang-tidy \ - "${TIDY_SOURCES[@]}" \ - -- \ - -std=c17 \ - "${PLATFORM_CFLAGS[@]}" \ - -I. \ - -Ilibffwd \ - -Itests \ - -Ibench \ - -Itools/server \ - -Ideps/ae - - if (( ${#CU_SOURCES[@]} > 0 )); then - run \ - "$clang_tidy" \ - --config-file=.clang-tidy \ - "${CU_SOURCES[@]}" \ - -- \ - "${CUDA_CFLAGS[@]}" \ - -I. \ - -Ilibffwd \ - -Ideps/ae - fi - else - echo "clang-tidy not found; skipping" - fi - - if [[ "$(uname -s)" == "Darwin" ]]; then - run make gpu CC=clang - fi -} - -main "$@" diff --git a/devtools/compare_model_embeddings.py b/devtools/compare_model_embeddings.py deleted file mode 100755 index bfda788..0000000 --- a/devtools/compare_model_embeddings.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -"""Compare embeddings from two model directories through the CLI.""" - -from __future__ import annotations - -import argparse -import json -import math -import subprocess -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -# Each backend is its own binary; the backend label alone does not change the -# CLI's behaviour, so cpu-vs-cuda comparisons must run two different binaries. -BUILD_DIR = {"cpu": "blas", "mlx": "mlx", "cuda": "cuda"} - - -def default_binary(backend: str) -> Path: - return ROOT / "build" / "release" / BUILD_DIR[backend] / "ffwd-cli" - - -def resolve_binary(backend: str, explicit: str, shared: str) -> Path: - """Pick the CLI for one side: an explicit --binary-{a,b}, then the shared - --binary override, then the per-backend build-tree default.""" - if explicit: - return Path(explicit) - if shared: - return Path(shared) - return default_binary(backend) - - -def ensure_binary(backend: str, binary: Path) -> None: - """Build the backend's CLI on demand when using the build-tree default.""" - if binary == default_binary(backend) and not binary.exists(): - subprocess.run(["make", backend], cwd=ROOT, check=True, - stdout=subprocess.DEVNULL) - if not binary.exists(): - raise SystemExit(f"binary not found for backend {backend}: {binary}") - - -TEXTS = [ - "query: what is the capital of France?", - "document: Paris is the capital of France.", - "document: Berlin is the capital of Germany.", - "The quick brown fox jumps over the lazy dog near the riverbank", - ( - "Retrieval augmented generation combines embedding models with external " - "knowledge sources and ranks documents by semantic similarity." - ), -] - - -def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int, - backend: str, threads: int) -> list[list[float]]: - cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)] - if backend == "cpu" and threads > 0: - cmd.extend(["-t", str(threads)]) - - proc = subprocess.Popen( - cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - assert proc.stdin is not None - assert proc.stdout is not None - proc.stdin.write("\n".join(texts) + "\n") - proc.stdin.close() - - embeddings = [] - for _ in texts: - line = proc.stdout.readline() - if not line: - stderr = proc.stderr.read() if proc.stderr else "" - raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}") - row = json.loads(line) - if "error" in row: - raise RuntimeError(f"stdin error for {model_dir}: {row}") - embeddings.append([float(x) for x in row["embedding"]]) - - rc = proc.wait(timeout=10) - if rc != 0: - stderr = proc.stderr.read() if proc.stderr else "" - raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}") - return embeddings - - -def cosine(a: list[float], b: list[float]) -> float: - dot = sum(x * y for x, y in zip(a, b)) - na = math.sqrt(sum(x * x for x in a)) - nb = math.sqrt(sum(y * y for y in b)) - return dot / (na * nb) - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--binary", default="", - help="shared CLI override for both sides; default resolves per backend") - ap.add_argument("--binary-a", default="", help="CLI for --a (overrides --binary)") - ap.add_argument("--binary-b", default="", help="CLI for --b (overrides --binary)") - ap.add_argument("--a", required=True, help="reference model directory") - ap.add_argument("--b", required=True, help="comparison model directory") - ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu") - ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"], - help="backend for --a; defaults to --backend") - ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"], - help="backend for --b; defaults to --backend") - ap.add_argument("--batch-size", type=int, default=4) - ap.add_argument("--threads", type=int, default=0) - ap.add_argument("--max-diff", type=float, default=0.01) - ap.add_argument("--min-cos", type=float, default=0.999) - args = ap.parse_args() - - if args.batch_size <= 0: - raise SystemExit("--batch-size must be > 0") - - backend_a = args.backend_a or args.backend - backend_b = args.backend_b or args.backend - binary_a = resolve_binary(backend_a, args.binary_a, args.binary) - binary_b = resolve_binary(backend_b, args.binary_b, args.binary) - ensure_binary(backend_a, binary_a) - ensure_binary(backend_b, binary_b) - emb_a = run_model(str(binary_a), args.a, TEXTS, args.batch_size, - backend_a, args.threads) - emb_b = run_model(str(binary_b), args.b, TEXTS, args.batch_size, - backend_b, args.threads) - - worst_diff = 0.0 - worst_cos = 1.0 - for i, (a, b) in enumerate(zip(emb_a, emb_b)): - if len(a) != len(b): - raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}") - max_diff = max(abs(x - y) for x, y in zip(a, b)) - cos = cosine(a, b) - worst_diff = max(worst_diff, max_diff) - worst_cos = min(worst_cos, cos) - print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}") - - print( - f"ok: backend_a={backend_a} backend_b={backend_b} " - f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}" - ) - if worst_diff > args.max_diff or worst_cos < args.min_cos: - raise SystemExit( - f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/check_code_quality.sh b/tests/check_code_quality.sh new file mode 100755 index 0000000..3c81d92 --- /dev/null +++ b/tests/check_code_quality.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" +cd "$ROOT" + +# Lint and analyze every C source under libffwd/, tests/, and bench/. Each +# platform keeps its own GPU backend and skips the other's, which it does not +# build (macOS builds MLX, Linux builds CUDA). +SYS="$(uname -s)" +SRC_SOURCES=() +for f in libffwd/*.c; do + [[ "$SYS" == Darwin && "$f" == libffwd/backend_cuda.c ]] && continue + [[ "$SYS" != Darwin && ( "$f" == libffwd/mlx.c || "$f" == libffwd/backend_mlx.c ) ]] && continue + SRC_SOURCES+=("$f") +done + +TEST_C_SOURCES=(tests/*.c) +BENCH_C_SOURCES=(bench/*.c) + +# CUDA backend sources are C++17 and only built on Linux. Parse them with the +# clang CUDA front end in host-only mode so no GPU or device toolchain is +# required; macOS has no CUDA toolkit, so skip them there. +CU_SOURCES=() +if [[ "$SYS" != Darwin ]]; then + for f in libffwd/*.cu; do + [[ -e "$f" ]] && CU_SOURCES+=("$f") + done +fi + +ANALYZE_SOURCES=("${SRC_SOURCES[@]}") +TIDY_SOURCES=("${SRC_SOURCES[@]}" "${TEST_C_SOURCES[@]}" "${BENCH_C_SOURCES[@]}") + +ANALYZE_COMMON=( + clang --analyze -Xanalyzer -analyzer-output=text + -Wall -Wextra + -I. -Ilibffwd -Ideps/ae +) + +PLATFORM_CFLAGS=() +if [[ "$(uname -s)" == "Darwin" ]]; then + PLATFORM_CFLAGS=( + -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX + -I/opt/homebrew/opt/mlx-c/include + -I/opt/homebrew/include + -I/usr/local/include + ) +else + PLATFORM_CFLAGS=( + -DUSE_BLAS -DUSE_OPENBLAS + -I/usr/include/openblas + ) +fi + +# Flags for the CUDA backend front end. CUDA_HOME matches the Makefile default. +# --cuda-host-only parses host code only so device codegen and libdevice are not +# needed; the macros mirror the nvcc command in the Makefile. +CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" +CUDA_CFLAGS=( + -x cuda --cuda-host-only "--cuda-path=$CUDA_HOME" + -std=c++17 + -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU + -I"$CUDA_HOME/include" + -I/usr/include/openblas +) + +print_cmd() { + printf '+' + printf ' %q' "$@" + printf '\n' +} + +run() { + print_cmd "$@" + "$@" +} + +run_analyze() { + local output + local status + + print_cmd "$@" + + set +e + output="$("$@" 2>&1)" + status=$? + set -e + + if [[ -n "$output" ]]; then + printf '%s\n' "$output" + fi + + if (( status != 0 )); then + return "$status" + fi + + if grep -q 'warning:' <<<"$output"; then + echo "clang analyzer warnings found" >&2 + return 1 + fi +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +find_clang_tidy() { + local brew_path="/opt/homebrew/opt/llvm@21/bin/clang-tidy" + + if command -v clang-tidy >/dev/null 2>&1; then + command -v clang-tidy + return 0 + fi + + if [[ -x "$brew_path" ]]; then + printf '%s\n' "$brew_path" + return 0 + fi + + return 1 +} + +main() { + local clang_tidy + + if ! have clang; then + echo "clang not found" >&2 + return 1 + fi + + run git --no-pager diff --check + run make cpu CC=clang + + run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ + "${ANALYZE_SOURCES[@]}" + run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ + -I. -Ilibffwd -Itests "${TEST_C_SOURCES[@]}" + run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ + -I. -Ilibffwd -Ibench -Itools/server "${BENCH_C_SOURCES[@]}" + + if (( ${#CU_SOURCES[@]} > 0 )); then + run_analyze "${ANALYZE_COMMON[@]}" "${CUDA_CFLAGS[@]}" \ + "${CU_SOURCES[@]}" + fi + + if clang_tidy="$(find_clang_tidy)"; then + run \ + "$clang_tidy" \ + --config-file=.clang-tidy \ + "${TIDY_SOURCES[@]}" \ + -- \ + -std=c17 \ + "${PLATFORM_CFLAGS[@]}" \ + -I. \ + -Ilibffwd \ + -Itests \ + -Ibench \ + -Itools/server \ + -Ideps/ae + + if (( ${#CU_SOURCES[@]} > 0 )); then + run \ + "$clang_tidy" \ + --config-file=.clang-tidy \ + "${CU_SOURCES[@]}" \ + -- \ + "${CUDA_CFLAGS[@]}" \ + -I. \ + -Ilibffwd \ + -Ideps/ae + fi + else + echo "clang-tidy not found; skipping" + fi + + if [[ "$(uname -s)" == "Darwin" ]]; then + run make gpu CC=clang + fi +} + +main "$@" diff --git a/tests/check_embedding_parity.py b/tests/check_embedding_parity.py new file mode 100755 index 0000000..bfda788 --- /dev/null +++ b/tests/check_embedding_parity.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Compare embeddings from two model directories through the CLI.""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +# Each backend is its own binary; the backend label alone does not change the +# CLI's behaviour, so cpu-vs-cuda comparisons must run two different binaries. +BUILD_DIR = {"cpu": "blas", "mlx": "mlx", "cuda": "cuda"} + + +def default_binary(backend: str) -> Path: + return ROOT / "build" / "release" / BUILD_DIR[backend] / "ffwd-cli" + + +def resolve_binary(backend: str, explicit: str, shared: str) -> Path: + """Pick the CLI for one side: an explicit --binary-{a,b}, then the shared + --binary override, then the per-backend build-tree default.""" + if explicit: + return Path(explicit) + if shared: + return Path(shared) + return default_binary(backend) + + +def ensure_binary(backend: str, binary: Path) -> None: + """Build the backend's CLI on demand when using the build-tree default.""" + if binary == default_binary(backend) and not binary.exists(): + subprocess.run(["make", backend], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL) + if not binary.exists(): + raise SystemExit(f"binary not found for backend {backend}: {binary}") + + +TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "The quick brown fox jumps over the lazy dog near the riverbank", + ( + "Retrieval augmented generation combines embedding models with external " + "knowledge sources and ranks documents by semantic similarity." + ), +] + + +def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int, + backend: str, threads: int) -> list[list[float]]: + cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)] + if backend == "cpu" and threads > 0: + cmd.extend(["-t", str(threads)]) + + proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert proc.stdin is not None + assert proc.stdout is not None + proc.stdin.write("\n".join(texts) + "\n") + proc.stdin.close() + + embeddings = [] + for _ in texts: + line = proc.stdout.readline() + if not line: + stderr = proc.stderr.read() if proc.stderr else "" + raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}") + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin error for {model_dir}: {row}") + embeddings.append([float(x) for x in row["embedding"]]) + + rc = proc.wait(timeout=10) + if rc != 0: + stderr = proc.stderr.read() if proc.stderr else "" + raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}") + return embeddings + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--binary", default="", + help="shared CLI override for both sides; default resolves per backend") + ap.add_argument("--binary-a", default="", help="CLI for --a (overrides --binary)") + ap.add_argument("--binary-b", default="", help="CLI for --b (overrides --binary)") + ap.add_argument("--a", required=True, help="reference model directory") + ap.add_argument("--b", required=True, help="comparison model directory") + ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu") + ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"], + help="backend for --a; defaults to --backend") + ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"], + help="backend for --b; defaults to --backend") + ap.add_argument("--batch-size", type=int, default=4) + ap.add_argument("--threads", type=int, default=0) + ap.add_argument("--max-diff", type=float, default=0.01) + ap.add_argument("--min-cos", type=float, default=0.999) + args = ap.parse_args() + + if args.batch_size <= 0: + raise SystemExit("--batch-size must be > 0") + + backend_a = args.backend_a or args.backend + backend_b = args.backend_b or args.backend + binary_a = resolve_binary(backend_a, args.binary_a, args.binary) + binary_b = resolve_binary(backend_b, args.binary_b, args.binary) + ensure_binary(backend_a, binary_a) + ensure_binary(backend_b, binary_b) + emb_a = run_model(str(binary_a), args.a, TEXTS, args.batch_size, + backend_a, args.threads) + emb_b = run_model(str(binary_b), args.b, TEXTS, args.batch_size, + backend_b, args.threads) + + worst_diff = 0.0 + worst_cos = 1.0 + for i, (a, b) in enumerate(zip(emb_a, emb_b)): + if len(a) != len(b): + raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}") + max_diff = max(abs(x - y) for x, y in zip(a, b)) + cos = cosine(a, b) + worst_diff = max(worst_diff, max_diff) + worst_cos = min(worst_cos, cos) + print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}") + + print( + f"ok: backend_a={backend_a} backend_b={backend_b} " + f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}" + ) + if worst_diff > args.max_diff or worst_cos < args.min_cos: + raise SystemExit( + f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- cgit v1.3.1