aboutsummaryrefslogtreecommitdiff
path: root/devtools
diff options
context:
space:
mode:
Diffstat (limited to 'devtools')
-rwxr-xr-xdevtools/check_code_quality.sh182
-rwxr-xr-xdevtools/compare_model_embeddings.py152
2 files changed, 0 insertions, 334 deletions
diff --git a/devtools/check_code_quality.sh b/devtools/check_code_quality.sh
deleted file mode 100755
index 3c81d92..0000000
--- a/devtools/check_code_quality.sh
+++ /dev/null
@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
-cd "$ROOT"
-
-# Lint and analyze every C source under libffwd/, tests/, and bench/. Each
-# platform keeps its own GPU backend and skips the other's, which it does not
-# build (macOS builds MLX, Linux builds CUDA).
-SYS="$(uname -s)"
-SRC_SOURCES=()
-for f in libffwd/*.c; do
- [[ "$SYS" == Darwin && "$f" == libffwd/backend_cuda.c ]] && continue
- [[ "$SYS" != Darwin && ( "$f" == libffwd/mlx.c || "$f" == libffwd/backend_mlx.c ) ]] && continue
- SRC_SOURCES+=("$f")
-done
-
-TEST_C_SOURCES=(tests/*.c)
-BENCH_C_SOURCES=(bench/*.c)
-
-# CUDA backend sources are C++17 and only built on Linux. Parse them with the
-# clang CUDA front end in host-only mode so no GPU or device toolchain is
-# required; macOS has no CUDA toolkit, so skip them there.
-CU_SOURCES=()
-if [[ "$SYS" != Darwin ]]; then
- for f in libffwd/*.cu; do
- [[ -e "$f" ]] && CU_SOURCES+=("$f")
- done
-fi
-
-ANALYZE_SOURCES=("${SRC_SOURCES[@]}")
-TIDY_SOURCES=("${SRC_SOURCES[@]}" "${TEST_C_SOURCES[@]}" "${BENCH_C_SOURCES[@]}")
-
-ANALYZE_COMMON=(
- clang --analyze -Xanalyzer -analyzer-output=text
- -Wall -Wextra
- -I. -Ilibffwd -Ideps/ae
-)
-
-PLATFORM_CFLAGS=()
-if [[ "$(uname -s)" == "Darwin" ]]; then
- PLATFORM_CFLAGS=(
- -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX
- -I/opt/homebrew/opt/mlx-c/include
- -I/opt/homebrew/include
- -I/usr/local/include
- )
-else
- PLATFORM_CFLAGS=(
- -DUSE_BLAS -DUSE_OPENBLAS
- -I/usr/include/openblas
- )
-fi
-
-# Flags for the CUDA backend front end. CUDA_HOME matches the Makefile default.
-# --cuda-host-only parses host code only so device codegen and libdevice are not
-# needed; the macros mirror the nvcc command in the Makefile.
-CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
-CUDA_CFLAGS=(
- -x cuda --cuda-host-only "--cuda-path=$CUDA_HOME"
- -std=c++17
- -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU
- -I"$CUDA_HOME/include"
- -I/usr/include/openblas
-)
-
-print_cmd() {
- printf '+'
- printf ' %q' "$@"
- printf '\n'
-}
-
-run() {
- print_cmd "$@"
- "$@"
-}
-
-run_analyze() {
- local output
- local status
-
- print_cmd "$@"
-
- set +e
- output="$("$@" 2>&1)"
- status=$?
- set -e
-
- if [[ -n "$output" ]]; then
- printf '%s\n' "$output"
- fi
-
- if (( status != 0 )); then
- return "$status"
- fi
-
- if grep -q 'warning:' <<<"$output"; then
- echo "clang analyzer warnings found" >&2
- return 1
- fi
-}
-
-have() {
- command -v "$1" >/dev/null 2>&1
-}
-
-find_clang_tidy() {
- local brew_path="/opt/homebrew/opt/llvm@21/bin/clang-tidy"
-
- if command -v clang-tidy >/dev/null 2>&1; then
- command -v clang-tidy
- return 0
- fi
-
- if [[ -x "$brew_path" ]]; then
- printf '%s\n' "$brew_path"
- return 0
- fi
-
- return 1
-}
-
-main() {
- local clang_tidy
-
- if ! have clang; then
- echo "clang not found" >&2
- return 1
- fi
-
- run git --no-pager diff --check
- run make cpu CC=clang
-
- run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
- "${ANALYZE_SOURCES[@]}"
- run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
- -I. -Ilibffwd -Itests "${TEST_C_SOURCES[@]}"
- run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
- -I. -Ilibffwd -Ibench -Itools/server "${BENCH_C_SOURCES[@]}"
-
- if (( ${#CU_SOURCES[@]} > 0 )); then
- run_analyze "${ANALYZE_COMMON[@]}" "${CUDA_CFLAGS[@]}" \
- "${CU_SOURCES[@]}"
- fi
-
- if clang_tidy="$(find_clang_tidy)"; then
- run \
- "$clang_tidy" \
- --config-file=.clang-tidy \
- "${TIDY_SOURCES[@]}" \
- -- \
- -std=c17 \
- "${PLATFORM_CFLAGS[@]}" \
- -I. \
- -Ilibffwd \
- -Itests \
- -Ibench \
- -Itools/server \
- -Ideps/ae
-
- if (( ${#CU_SOURCES[@]} > 0 )); then
- run \
- "$clang_tidy" \
- --config-file=.clang-tidy \
- "${CU_SOURCES[@]}" \
- -- \
- "${CUDA_CFLAGS[@]}" \
- -I. \
- -Ilibffwd \
- -Ideps/ae
- fi
- else
- echo "clang-tidy not found; skipping"
- fi
-
- if [[ "$(uname -s)" == "Darwin" ]]; then
- run make gpu CC=clang
- fi
-}
-
-main "$@"
diff --git a/devtools/compare_model_embeddings.py b/devtools/compare_model_embeddings.py
deleted file mode 100755
index bfda788..0000000
--- a/devtools/compare_model_embeddings.py
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/env python3
-"""Compare embeddings from two model directories through the CLI."""
-
-from __future__ import annotations
-
-import argparse
-import json
-import math
-import subprocess
-from pathlib import Path
-
-ROOT = Path(__file__).resolve().parents[1]
-# Each backend is its own binary; the backend label alone does not change the
-# CLI's behaviour, so cpu-vs-cuda comparisons must run two different binaries.
-BUILD_DIR = {"cpu": "blas", "mlx": "mlx", "cuda": "cuda"}
-
-
-def default_binary(backend: str) -> Path:
- return ROOT / "build" / "release" / BUILD_DIR[backend] / "ffwd-cli"
-
-
-def resolve_binary(backend: str, explicit: str, shared: str) -> Path:
- """Pick the CLI for one side: an explicit --binary-{a,b}, then the shared
- --binary override, then the per-backend build-tree default."""
- if explicit:
- return Path(explicit)
- if shared:
- return Path(shared)
- return default_binary(backend)
-
-
-def ensure_binary(backend: str, binary: Path) -> None:
- """Build the backend's CLI on demand when using the build-tree default."""
- if binary == default_binary(backend) and not binary.exists():
- subprocess.run(["make", backend], cwd=ROOT, check=True,
- stdout=subprocess.DEVNULL)
- if not binary.exists():
- raise SystemExit(f"binary not found for backend {backend}: {binary}")
-
-
-TEXTS = [
- "query: what is the capital of France?",
- "document: Paris is the capital of France.",
- "document: Berlin is the capital of Germany.",
- "The quick brown fox jumps over the lazy dog near the riverbank",
- (
- "Retrieval augmented generation combines embedding models with external "
- "knowledge sources and ranks documents by semantic similarity."
- ),
-]
-
-
-def run_model(binary: str, model_dir: str, texts: list[str], batch_size: int,
- backend: str, threads: int) -> list[list[float]]:
- cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
- if backend == "cpu" and threads > 0:
- cmd.extend(["-t", str(threads)])
-
- proc = subprocess.Popen(
- cmd,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
- )
- assert proc.stdin is not None
- assert proc.stdout is not None
- proc.stdin.write("\n".join(texts) + "\n")
- proc.stdin.close()
-
- embeddings = []
- for _ in texts:
- line = proc.stdout.readline()
- if not line:
- stderr = proc.stderr.read() if proc.stderr else ""
- raise RuntimeError(f"stdin exited early for {model_dir}\n{stderr}")
- row = json.loads(line)
- if "error" in row:
- raise RuntimeError(f"stdin error for {model_dir}: {row}")
- embeddings.append([float(x) for x in row["embedding"]])
-
- rc = proc.wait(timeout=10)
- if rc != 0:
- stderr = proc.stderr.read() if proc.stderr else ""
- raise RuntimeError(f"stdin failed for {model_dir}: rc={rc}\n{stderr}")
- return embeddings
-
-
-def cosine(a: list[float], b: list[float]) -> float:
- dot = sum(x * y for x, y in zip(a, b))
- na = math.sqrt(sum(x * x for x in a))
- nb = math.sqrt(sum(y * y for y in b))
- return dot / (na * nb)
-
-
-def main() -> int:
- ap = argparse.ArgumentParser(description=__doc__)
- ap.add_argument("--binary", default="",
- help="shared CLI override for both sides; default resolves per backend")
- ap.add_argument("--binary-a", default="", help="CLI for --a (overrides --binary)")
- ap.add_argument("--binary-b", default="", help="CLI for --b (overrides --binary)")
- ap.add_argument("--a", required=True, help="reference model directory")
- ap.add_argument("--b", required=True, help="comparison model directory")
- ap.add_argument("--backend", choices=["cpu", "mlx", "cuda"], default="cpu")
- ap.add_argument("--backend-a", choices=["cpu", "mlx", "cuda"],
- help="backend for --a; defaults to --backend")
- ap.add_argument("--backend-b", choices=["cpu", "mlx", "cuda"],
- help="backend for --b; defaults to --backend")
- ap.add_argument("--batch-size", type=int, default=4)
- ap.add_argument("--threads", type=int, default=0)
- ap.add_argument("--max-diff", type=float, default=0.01)
- ap.add_argument("--min-cos", type=float, default=0.999)
- args = ap.parse_args()
-
- if args.batch_size <= 0:
- raise SystemExit("--batch-size must be > 0")
-
- backend_a = args.backend_a or args.backend
- backend_b = args.backend_b or args.backend
- binary_a = resolve_binary(backend_a, args.binary_a, args.binary)
- binary_b = resolve_binary(backend_b, args.binary_b, args.binary)
- ensure_binary(backend_a, binary_a)
- ensure_binary(backend_b, binary_b)
- emb_a = run_model(str(binary_a), args.a, TEXTS, args.batch_size,
- backend_a, args.threads)
- emb_b = run_model(str(binary_b), args.b, TEXTS, args.batch_size,
- backend_b, args.threads)
-
- worst_diff = 0.0
- worst_cos = 1.0
- for i, (a, b) in enumerate(zip(emb_a, emb_b)):
- if len(a) != len(b):
- raise SystemExit(f"dimension mismatch at text {i}: {len(a)} vs {len(b)}")
- max_diff = max(abs(x - y) for x, y in zip(a, b))
- cos = cosine(a, b)
- worst_diff = max(worst_diff, max_diff)
- worst_cos = min(worst_cos, cos)
- print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}")
-
- print(
- f"ok: backend_a={backend_a} backend_b={backend_b} "
- f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}"
- )
- if worst_diff > args.max_diff or worst_cos < args.min_cos:
- raise SystemExit(
- f"model parity outside threshold: diff={worst_diff:.8g}, cos={worst_cos:.8f}"
- )
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())