aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rwxr-xr-xtests/check_code_quality.sh182
-rwxr-xr-xtests/check_embedding_parity.py152
2 files changed, 334 insertions, 0 deletions
diff --git a/tests/check_code_quality.sh b/tests/check_code_quality.sh
new file mode 100755
index 0000000..3c81d92
--- /dev/null
+++ b/tests/check_code_quality.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$ROOT"
+
+# Lint and analyze every C source under libffwd/, tests/, and bench/. Each
+# platform keeps its own GPU backend and skips the other's, which it does not
+# build (macOS builds MLX, Linux builds CUDA).
+SYS="$(uname -s)"
+SRC_SOURCES=()
+for f in libffwd/*.c; do
+ [[ "$SYS" == Darwin && "$f" == libffwd/backend_cuda.c ]] && continue
+ [[ "$SYS" != Darwin && ( "$f" == libffwd/mlx.c || "$f" == libffwd/backend_mlx.c ) ]] && continue
+ SRC_SOURCES+=("$f")
+done
+
+TEST_C_SOURCES=(tests/*.c)
+BENCH_C_SOURCES=(bench/*.c)
+
+# CUDA backend sources are C++17 and only built on Linux. Parse them with the
+# clang CUDA front end in host-only mode so no GPU or device toolchain is
+# required; macOS has no CUDA toolkit, so skip them there.
+CU_SOURCES=()
+if [[ "$SYS" != Darwin ]]; then
+ for f in libffwd/*.cu; do
+ [[ -e "$f" ]] && CU_SOURCES+=("$f")
+ done
+fi
+
+ANALYZE_SOURCES=("${SRC_SOURCES[@]}")
+TIDY_SOURCES=("${SRC_SOURCES[@]}" "${TEST_C_SOURCES[@]}" "${BENCH_C_SOURCES[@]}")
+
+ANALYZE_COMMON=(
+ clang --analyze -Xanalyzer -analyzer-output=text
+ -Wall -Wextra
+ -I. -Ilibffwd -Ideps/ae
+)
+
+PLATFORM_CFLAGS=()
+if [[ "$(uname -s)" == "Darwin" ]]; then
+ PLATFORM_CFLAGS=(
+ -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX
+ -I/opt/homebrew/opt/mlx-c/include
+ -I/opt/homebrew/include
+ -I/usr/local/include
+ )
+else
+ PLATFORM_CFLAGS=(
+ -DUSE_BLAS -DUSE_OPENBLAS
+ -I/usr/include/openblas
+ )
+fi
+
+# Flags for the CUDA backend front end. CUDA_HOME matches the Makefile default.
+# --cuda-host-only parses host code only so device codegen and libdevice are not
+# needed; the macros mirror the nvcc command in the Makefile.
+CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
+CUDA_CFLAGS=(
+ -x cuda --cuda-host-only "--cuda-path=$CUDA_HOME"
+ -std=c++17
+ -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU
+ -I"$CUDA_HOME/include"
+ -I/usr/include/openblas
+)
+
+print_cmd() {
+ printf '+'
+ printf ' %q' "$@"
+ printf '\n'
+}
+
+run() {
+ print_cmd "$@"
+ "$@"
+}
+
+run_analyze() {
+ local output
+ local status
+
+ print_cmd "$@"
+
+ set +e
+ output="$("$@" 2>&1)"
+ status=$?
+ set -e
+
+ if [[ -n "$output" ]]; then
+ printf '%s\n' "$output"
+ fi
+
+ if (( status != 0 )); then
+ return "$status"
+ fi
+
+ if grep -q 'warning:' <<<"$output"; then
+ echo "clang analyzer warnings found" >&2
+ return 1
+ fi
+}
+
+have() {
+ command -v "$1" >/dev/null 2>&1
+}
+
+find_clang_tidy() {
+ local brew_path="/opt/homebrew/opt/llvm@21/bin/clang-tidy"
+
+ if command -v clang-tidy >/dev/null 2>&1; then
+ command -v clang-tidy
+ return 0
+ fi
+
+ if [[ -x "$brew_path" ]]; then
+ printf '%s\n' "$brew_path"
+ return 0
+ fi
+
+ return 1
+}
+
+main() {
+ local clang_tidy
+
+ if ! have clang; then
+ echo "clang not found" >&2
+ return 1
+ fi
+
+ run git --no-pager diff --check
+ run make cpu CC=clang
+
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ "${ANALYZE_SOURCES[@]}"
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ -I. -Ilibffwd -Itests "${TEST_C_SOURCES[@]}"
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ -I. -Ilibffwd -Ibench -Itools/server "${BENCH_C_SOURCES[@]}"
+
+ if (( ${#CU_SOURCES[@]} > 0 )); then
+ run_analyze "${ANALYZE_COMMON[@]}" "${CUDA_CFLAGS[@]}" \
+ "${CU_SOURCES[@]}"
+ fi
+
+ if clang_tidy="$(find_clang_tidy)"; then
+ run \
+ "$clang_tidy" \
+ --config-file=.clang-tidy \
+ "${TIDY_SOURCES[@]}" \
+ -- \
+ -std=c17 \
+ "${PLATFORM_CFLAGS[@]}" \
+ -I. \
+ -Ilibffwd \
+ -Itests \
+ -Ibench \
+ -Itools/server \
+ -Ideps/ae
+
+ if (( ${#CU_SOURCES[@]} > 0 )); then
+ run \
+ "$clang_tidy" \
+ --config-file=.clang-tidy \
+ "${CU_SOURCES[@]}" \
+ -- \
+ "${CUDA_CFLAGS[@]}" \
+ -I. \
+ -Ilibffwd \
+ -Ideps/ae
+ fi
+ else
+ echo "clang-tidy not found; skipping"
+ fi
+
+ if [[ "$(uname -s)" == "Darwin" ]]; then
+ run make gpu CC=clang
+ fi
+}
+
+main "$@"
diff --git a/tests/check_embedding_parity.py b/tests/check_embedding_parity.py
new file mode 100755
index 0000000..bfda788
--- /dev/null
+++ b/tests/check_embedding_parity.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Compare embeddings from two model directories through the CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import subprocess
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+# Each backend is its own binary; the backend label alone does not change the
+# CLI's behaviour, so cpu-vs-cuda comparisons must run two different binaries.
+BUILD_DIR = {"cpu": "blas", "mlx": "mlx", "cuda": "cuda"}
+
+
+def default_binary(backend: str) -> Path:
+ return ROOT / "build" / "release" / BUILD_DIR[backend] / "ffwd-cli"
+
+
+def resolve_binary(backend: str, explicit: str, shared: str) -> Path:
+ """Pick the CLI for one side: an explicit --binary-{a,b}, then the shared
+ --binary override, then the per-backend build-tree default."""
+ if explicit:
+ return Path(explicit)
+ if shared:
+ return Path(shared)
+ return default_binary(backend)
+
+
+def ensure_binary(backend: str, binary: Path) -> None:
+ """Build the backend's CLI on demand when using the build-tree default."""
+ if binary == default_binary(backend) and not binary.exists():
+ subprocess.run(["make", backend], cwd=ROOT, check=True,
+ stdout=subprocess.DEVNULL)
+ if not binary.exists():
+ raise SystemExit(f"binary not found for backend {backend}: {binary}")
+
+
+TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "The quick brown fox jumps over the lazy dog near the riverbank",
+ (
+ "Retrieval augmented generation combines embedding models with external "
+ "knowledge sources and ranks documents by semantic similarity."
+ ),
+]
+
+
+def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int,
+ backend: str, threads: int) -> list[list[float]]:
+ cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
+ if backend == "cpu" and threads > 0:
+ cmd.extend(["-t", str(threads)])
+
+ proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ )
+ assert proc.stdin is not None
+ assert proc.stdout is not None
+ proc.stdin.write("\n".join(texts) + "\n")
+ proc.stdin.close()
+
+ embeddings = []
+ for _ in texts:
+ line = proc.stdout.readline()
+ if not line:
+ stderr = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}")
+ row = json.loads(line)
+ if "error" in row:
+ raise RuntimeError(f"stdin error for {model_dir}: {row}")
+ embeddings.append([float(x) for x in row["embedding"]])
+
+ rc = proc.wait(timeout=10)
+ if rc != 0:
+ stderr = proc.stderr.read() if proc.stderr else ""
+ raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}")
+ return embeddings
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--binary", default="",
+ help="shared CLI override for both sides; default resolves per backend")
+ ap.add_argument("--binary-a", default="", help="CLI for --a (overrides --binary)")
+ ap.add_argument("--binary-b", default="", help="CLI for --b (overrides --binary)")
+ ap.add_argument("--a", required=True, help="reference model directory")
+ ap.add_argument("--b", required=True, help="comparison model directory")
+ ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu")
+ ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"],
+ help="backend for --a; defaults to --backend")
+ ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"],
+ help="backend for --b; defaults to --backend")
+ ap.add_argument("--batch-size", type=int, default=4)
+ ap.add_argument("--threads", type=int, default=0)
+ ap.add_argument("--max-diff", type=float, default=0.01)
+ ap.add_argument("--min-cos", type=float, default=0.999)
+ args = ap.parse_args()
+
+ if args.batch_size <= 0:
+ raise SystemExit("--batch-size must be > 0")
+
+ backend_a = args.backend_a or args.backend
+ backend_b = args.backend_b or args.backend
+ binary_a = resolve_binary(backend_a, args.binary_a, args.binary)
+ binary_b = resolve_binary(backend_b, args.binary_b, args.binary)
+ ensure_binary(backend_a, binary_a)
+ ensure_binary(backend_b, binary_b)
+ emb_a = run_model(str(binary_a), args.a, TEXTS, args.batch_size,
+ backend_a, args.threads)
+ emb_b = run_model(str(binary_b), args.b, TEXTS, args.batch_size,
+ backend_b, args.threads)
+
+ worst_diff = 0.0
+ worst_cos = 1.0
+ for i, (a, b) in enumerate(zip(emb_a, emb_b)):
+ if len(a) != len(b):
+ raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}")
+ max_diff = max(abs(x - y) for x, y in zip(a, b))
+ cos = cosine(a, b)
+ worst_diff = max(worst_diff, max_diff)
+ worst_cos = min(worst_cos, cos)
+ print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}")
+
+ print(
+ f"ok: backend_a={backend_a} backend_b={backend_b} "
+ f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}"
+ )
+ if worst_diff > args.max_diff or worst_cos < args.min_cos:
+ raise SystemExit(
+ f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())