aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-21 08:26:31 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 08:26:31 +0200
commit445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf (patch)
tree09c7e853df4dd6fece5e3758709aad7bd6ea9503
parenta5f67f2f828a7ac5c541ec54abf758597efcfbb6 (diff)
downloadffwd-445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf.tar.gz
ffwd-445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf.zip
build: rehome check scripts and modernize all.bash
-rwxr-xr-xall.bash278
-rwxr-xr-xdevtools/check_code_quality.sh181
-rwxr-xr-xdevtools/container-build-test.sh87
-rwxr-xr-xdevtools/verify_builds.sh330
-rwxr-xr-xtests/check_cuda_fast_gemm_parity.py188
-rwxr-xr-xtests/check_mlx_model_limits.py60
-rwxr-xr-xtests/check_mlx_quantized_parity.py104
7 files changed, 1228 insertions, 0 deletions
diff --git a/all.bash b/all.bash
new file mode 100755
index 0000000..5f3dbcc
--- /dev/null
+++ b/all.bash
@@ -0,0 +1,278 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+export UV_CACHE_DIR="${UV_CACHE_DIR:-$ROOT/.venv/uv-cache}"
+
+RED=""
+GREEN=""
+YELLOW=""
+NORMAL=""
+MAXWIDTH=0
+
+if [[ -t 2 ]] && command -v tput >/dev/null 2>&1; then
+ RED="$(tput setaf 1 2>/dev/null || true)"
+ GREEN="$(tput setaf 2 2>/dev/null || true)"
+ YELLOW="$(tput setaf 3 2>/dev/null || true)"
+ NORMAL="$(tput sgr0 2>/dev/null || true)"
+ MAXWIDTH="$(tput cols 2>/dev/null || printf '0')"
+ if (( MAXWIDTH > 2 )); then
+ MAXWIDTH=$((MAXWIDTH - 2))
+ else
+ MAXWIDTH=0
+ fi
+fi
+
+info() { printf '%sINFO: %s%s\n' "$GREEN" "$*" "$NORMAL" >&2; }
+warn() { printf '%sWARN: %s%s\n' "$YELLOW" "$*" "$NORMAL" >&2; }
+err() { printf '%sERROR: %s%s\n' "$RED" "$*" "$NORMAL" >&2; }
+
+run() {
+ local rendered
+ printf -v rendered '%q ' "$@"
+ rendered="${rendered% }"
+ if (( MAXWIDTH > 0 && ${#rendered} > MAXWIDTH )); then
+ rendered="${rendered:0:$((MAXWIDTH - 3))}..."
+ fi
+ info "\$ $rendered"
+ (cd "$ROOT" && "$@")
+}
+
+run_uv() {
+ run uv run "$@"
+}
+
+# Run a Python check that needs the pinned SentenceTransformers reference stack
+# (torch, sentence-transformers); the pins live in requirements-reference.txt.
+run_reference() {
+ run uv run --with-requirements requirements-reference.txt "$@"
+}
+
+check_code_quality() {
+ run ./devtools/check_code_quality.sh "$@"
+}
+
+run_checks() {
+ run ./devtools/verify_builds.sh "$@"
+}
+
+check_kernel_golden() {
+ # Build and run the math-kernel golden tests straight from the test Makefile.
+ # test_kernels exercises the BLAS-built kernels; test_kernels_generic the
+ # scalar/SIMD path with USE_BLAS off. Extra args (e.g. --blas, kept for
+ # backward compatibility) are accepted but ignored; both variants always run.
+ run make -C tests test_kernels test_kernels_generic
+ run ./tests/test_kernels
+ run ./tests/test_kernels_generic
+}
+
+check_batch_parity() {
+ run_uv ./tests/check_batch_parity.py "$@"
+}
+
+check_contextual_batch_parity() {
+ run_uv ./tests/check_contextual_batch_parity.py "$@"
+}
+
+check_late_interaction() {
+ run_uv ./tests/check_late_interaction.py "$@"
+}
+
+check_cuda_fast_gemm_parity() {
+ if [[ "$(uname -s)" != "Linux" ]]; then
+ warn "CUDA fast GEMM parity requires Linux; skipping on $(uname -s)"
+ return 0
+ fi
+ if ! command -v nvidia-smi >/dev/null 2>&1; then
+ warn "CUDA fast GEMM parity requires an NVIDIA GPU; nvidia-smi not found"
+ return 0
+ fi
+ run_uv ./tests/check_cuda_fast_gemm_parity.py "$@"
+}
+
+check_embedding_parity() {
+ run_uv ./devtools/compare_model_embeddings.py "$@"
+}
+
+check_mlx_model_limits() {
+ if [[ "$(uname -s)" != "Darwin" ]]; then
+ warn "MLX metadata limits require macOS; skipping on $(uname -s)"
+ return 0
+ fi
+ run_uv ./tests/check_mlx_model_limits.py "$@"
+}
+
+check_mlx_quantized_parity() {
+ if [[ "$(uname -s)" != "Darwin" ]]; then
+ warn "MLX quantized parity requires macOS; skipping on $(uname -s)"
+ return 0
+ fi
+ run_uv ./tests/check_mlx_quantized_parity.py "$@"
+}
+
+check_quant_precision_drift() {
+ run_reference ./devtools/quant_precision_drift.py "$@"
+}
+
+check_reference_parity() {
+ run_reference ./tests/check_reference_parity.py "$@"
+}
+
+check_bert_parity() {
+ run_reference ./tests/check_bert_parity.py "$@"
+}
+
+check_e5_parity() {
+ run_reference ./tests/check_e5_parity.py "$@"
+}
+
+check_qwen3_parity() {
+ run uv run --with sentence-transformers --with torch --with numpy \
+ ./tests/check_qwen3_parity.py "$@"
+}
+
+check_tokenizer_parity() {
+ run_uv ./tests/check_tokenizer_parity.py "$@"
+}
+
+check_wordpiece_parity() {
+ run_uv ./tests/check_wordpiece_parity.py "$@"
+}
+
+check_model_matrix() {
+ run_uv ./tests/check_model_matrix.py "$@"
+}
+
+check_perplexity_sdk_compat() {
+ run uv run --with perplexityai --with numpy \
+ ./tests/check_perplexity_sdk_compat.py "$@"
+}
+
+check_openai_sdk_compat() {
+ run uv run --with openai ./tests/check_openai_sdk_compat.py "$@"
+}
+
+check_server_api() {
+ run_uv ./tests/check_server_api.py "$@"
+}
+
+check_cli() {
+ run_uv ./tests/check_cli.py "$@"
+}
+
+check_workspace_api() {
+ run_uv ./tests/check_workspace_api.py "$@"
+}
+
+bench_contextual_api() {
+ run_uv ./bench/bench_contextual_api.py "$@"
+}
+
+bench_late_rerank() {
+ run_uv ./bench/bench_late_rerank.py "$@"
+}
+
+bench_late_api() {
+ run_uv ./bench/bench_late_api.py "$@"
+}
+
+bench_matrix() {
+ run_uv ./bench/bench_matrix.py "$@"
+}
+
+bench_qwen_tokenizer() {
+ run_uv ./bench/bench_qwen_tokenizer.py "$@"
+}
+
+bench_server_api() {
+ run_uv ./bench/bench_server_api.py "$@"
+}
+
+usage() {
+ cat <<'EOF'
+Usage: ./all.bash COMMAND [ARGS...]
+
+Aggregate:
+ checks Build and run the comprehensive local check suite
+
+Checks:
+ check-code-quality Run clang builds, analysis, and linting
+ check-kernel-golden Run math-kernel golden tests
+ check-batch-parity Compare singleton and true batched stdin execution
+ check-context-batch-parity Compare contextual batch and singleton execution
+ check-cuda-fast-gemm-parity Gate CUDA reduced-precision modes against exact F32
+ check-embedding-parity Compare embeddings from two model directories
+ check-late-interaction Check late-interaction vectors and MaxSim ranking
+ check-mlx-model-limits Check MLX model metadata rejection
+ check-mlx-quantized-parity Compare normal and quantized MLX embeddings
+ check-precision-drift Compare low-precision drift across engines
+ check-reference-parity Compare a running server with the Python reference
+ check-bert-parity Compare BERT embeddings with SentenceTransformers
+ check-e5-parity Compare XLM-R/E5 embeddings with SentenceTransformers
+ check-qwen3-parity Compare Qwen3 embeddings with SentenceTransformers
+ check-tokenizer-parity Compare the C BPE tokenizer with stored vectors
+ check-wordpiece-parity Compare the C WordPiece tokenizer with stored vectors
+ check-model-matrix Smoke-check supported model variants
+ check-server-api Verify a running local HTTP server
+ check-cli Drive the CLI binary through its modes and flags
+ check-workspace-api Check the model/workspace API with real weights
+ check-perplexity-sdk-compat Verify a running server through the Perplexity SDK
+ check-openai-sdk-compat Verify a running server through the OpenAI SDK
+
+Benchmarks:
+ bench-contextual-api Benchmark contextual inference through HTTP
+ bench-late-rerank Benchmark late-interaction MaxSim reranking
+ bench-late-api Benchmark HTTP reranking; optionally compare PyLate
+ bench-matrix Run the batch-aware stdin benchmark matrix
+ bench-qwen-tokenizer Benchmark tokenizer encode paths
+ bench-server-api Benchmark concurrent HTTP embedding requests
+EOF
+}
+
+main() {
+ local command="${1:-help}"
+ if (($#)); then
+ shift
+ fi
+
+ case "$command" in
+ help|-h|--help) usage ;;
+ checks) run_checks "$@" ;;
+ check-code-quality) check_code_quality "$@" ;;
+ check-kernel-golden) check_kernel_golden "$@" ;;
+ check-batch-parity) check_batch_parity "$@" ;;
+ check-context-batch-parity) check_contextual_batch_parity "$@" ;;
+ check-late-interaction) check_late_interaction "$@" ;;
+ check-cuda-fast-gemm-parity) check_cuda_fast_gemm_parity "$@" ;;
+ check-embedding-parity) check_embedding_parity "$@" ;;
+ check-mlx-model-limits) check_mlx_model_limits "$@" ;;
+ check-mlx-quantized-parity) check_mlx_quantized_parity "$@" ;;
+ check-precision-drift) check_quant_precision_drift "$@" ;;
+ check-reference-parity) check_reference_parity "$@" ;;
+ check-bert-parity) check_bert_parity "$@" ;;
+ check-e5-parity) check_e5_parity "$@" ;;
+ check-qwen3-parity) check_qwen3_parity "$@" ;;
+ check-tokenizer-parity) check_tokenizer_parity "$@" ;;
+ check-wordpiece-parity) check_wordpiece_parity "$@" ;;
+ check-model-matrix) check_model_matrix "$@" ;;
+ check-perplexity-sdk-compat) check_perplexity_sdk_compat "$@" ;;
+ check-openai-sdk-compat) check_openai_sdk_compat "$@" ;;
+ check-server-api) check_server_api "$@" ;;
+ check-cli) check_cli "$@" ;;
+ check-workspace-api) check_workspace_api "$@" ;;
+ bench-contextual-api) bench_contextual_api "$@" ;;
+ bench-late-rerank) bench_late_rerank "$@" ;;
+ bench-late-api) bench_late_api "$@" ;;
+ bench-matrix) bench_matrix "$@" ;;
+ bench-qwen-tokenizer) bench_qwen_tokenizer "$@" ;;
+ bench-server-api) bench_server_api "$@" ;;
+ *)
+ err "unknown command: $command"
+ usage >&2
+ return 2
+ ;;
+ esac
+}
+
+main "$@"
diff --git a/devtools/check_code_quality.sh b/devtools/check_code_quality.sh
new file mode 100755
index 0000000..305ba50
--- /dev/null
+++ b/devtools/check_code_quality.sh
@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$ROOT"
+
+# Lint and analyze every C source under libffwd/, tests/, and bench/. Each
+# platform keeps its own GPU backend and skips the other's, which it does not
+# build (macOS builds MLX, Linux builds CUDA).
+SYS="$(uname -s)"
+SRC_SOURCES=()
+for f in libffwd/*.c; do
+ [[ "$SYS" == Darwin && "$f" == libffwd/backend_cuda.c ]] && continue
+ [[ "$SYS" != Darwin && ( "$f" == libffwd/mlx.c || "$f" == libffwd/backend_mlx.c ) ]] && continue
+ SRC_SOURCES+=("$f")
+done
+
+TEST_C_SOURCES=(tests/*.c)
+BENCH_C_SOURCES=(bench/*.c)
+
+# CUDA backend sources are C++17 and only built on Linux. Parse them with the
+# clang CUDA front end in host-only mode so no GPU or device toolchain is
+# required; macOS has no CUDA toolkit, so skip them there.
+CU_SOURCES=()
+if [[ "$SYS" != Darwin ]]; then
+ for f in libffwd/*.cu; do
+ [[ -e "$f" ]] && CU_SOURCES+=("$f")
+ done
+fi
+
+ANALYZE_SOURCES=("${SRC_SOURCES[@]}")
+TIDY_SOURCES=("${SRC_SOURCES[@]}" "${TEST_C_SOURCES[@]}" "${BENCH_C_SOURCES[@]}")
+
+ANALYZE_COMMON=(
+ clang --analyze -Xanalyzer -analyzer-output=text
+ -Wall -Wextra
+ -I. -Ilibffwd -Ideps/ae
+)
+
+PLATFORM_CFLAGS=()
+if [[ "$(uname -s)" == "Darwin" ]]; then
+ PLATFORM_CFLAGS=(
+ -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX
+ -I/opt/homebrew/opt/mlx-c/include
+ -I/opt/homebrew/include
+ -I/usr/local/include
+ )
+else
+ PLATFORM_CFLAGS=(
+ -DUSE_BLAS -DUSE_OPENBLAS
+ -I/usr/include/openblas
+ )
+fi
+
+# Flags for the CUDA backend front end. CUDA_HOME matches the Makefile default.
+# --cuda-host-only parses host code only so device codegen and libdevice are not
+# needed; the macros mirror the nvcc command in the Makefile.
+CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
+CUDA_CFLAGS=(
+ -x cuda --cuda-host-only "--cuda-path=$CUDA_HOME"
+ -std=c++17
+ -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU
+ -I"$CUDA_HOME/include"
+ -I/usr/include/openblas
+)
+
+print_cmd() {
+ printf '+'
+ printf ' %q' "$@"
+ printf '\n'
+}
+
+run() {
+ print_cmd "$@"
+ "$@"
+}
+
+run_analyze() {
+ local output
+ local status
+
+ print_cmd "$@"
+
+ set +e
+ output="$("$@" 2>&1)"
+ status=$?
+ set -e
+
+ if [[ -n "$output" ]]; then
+ printf '%s\n' "$output"
+ fi
+
+ if (( status != 0 )); then
+ return "$status"
+ fi
+
+ if grep -q 'warning:' <<<"$output"; then
+ echo "clang analyzer warnings found" >&2
+ return 1
+ fi
+}
+
+have() {
+ command -v "$1" >/dev/null 2>&1
+}
+
+find_clang_tidy() {
+ local brew_path="/opt/homebrew/opt/llvm@21/bin/clang-tidy"
+
+ if command -v clang-tidy >/dev/null 2>&1; then
+ command -v clang-tidy
+ return 0
+ fi
+
+ if [[ -x "$brew_path" ]]; then
+ printf '%s\n' "$brew_path"
+ return 0
+ fi
+
+ return 1
+}
+
+main() {
+ local clang_tidy
+
+ if ! have clang; then
+ echo "clang not found" >&2
+ return 1
+ fi
+
+ run git --no-pager diff --check
+ run make cpu CC=clang
+
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ "${ANALYZE_SOURCES[@]}"
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ -I. -Ilibffwd -Itests "${TEST_C_SOURCES[@]}"
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ -I. -Ilibffwd -Ibench "${BENCH_C_SOURCES[@]}"
+
+ if (( ${#CU_SOURCES[@]} > 0 )); then
+ run_analyze "${ANALYZE_COMMON[@]}" "${CUDA_CFLAGS[@]}" \
+ "${CU_SOURCES[@]}"
+ fi
+
+ if clang_tidy="$(find_clang_tidy)"; then
+ run \
+ "$clang_tidy" \
+ --config-file=.clang-tidy \
+ "${TIDY_SOURCES[@]}" \
+ -- \
+ -std=c17 \
+ "${PLATFORM_CFLAGS[@]}" \
+ -I. \
+ -Ilibffwd \
+ -Itests \
+ -Ibench \
+ -Ideps/ae
+
+ if (( ${#CU_SOURCES[@]} > 0 )); then
+ run \
+ "$clang_tidy" \
+ --config-file=.clang-tidy \
+ "${CU_SOURCES[@]}" \
+ -- \
+ "${CUDA_CFLAGS[@]}" \
+ -I. \
+ -Ilibffwd \
+ -Ideps/ae
+ fi
+ else
+ echo "clang-tidy not found; skipping"
+ fi
+
+ if [[ "$(uname -s)" == "Darwin" ]]; then
+ run make gpu CC=clang
+ fi
+}
+
+main "$@"
diff --git a/devtools/container-build-test.sh b/devtools/container-build-test.sh
new file mode 100755
index 0000000..4dcc627
--- /dev/null
+++ b/devtools/container-build-test.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Build, test and bench the OpenBLAS backend in a throwaway Ubuntu container to
+# catch Linux portability breakage a macOS/Accelerate build hides. Needs Apple's
+# `container` CLI running. Run from anywhere in the repo: it finds the repo root
+# via git, bind-mounts it at /work, installs the Runpod toolchain, re-execs
+# itself inside with --in-container, and runs make cpu/test/bench/bench-server-
+# utils/debug, then `make clean` (skip with KEEP=1). Override the image with
+# IMAGE=ubuntu:22.04. Exits non-zero at the first failing step.
+set -euo pipefail
+
+IMAGE="${IMAGE:-ubuntu:24.04}"
+
+# --- inside the container: bootstrap toolchain, then build/test ---------------
+run_in_container() {
+ export DEBIAN_FRONTEND=noninteractive
+
+ echo "==================== apt bootstrap ===================="
+ apt-get update -qq
+ apt-get install -y --no-install-recommends \
+ build-essential ca-certificates libcjson-dev libopenblas-dev \
+ pkg-config git >/dev/null
+
+ echo "==================== system / toolchain ===================="
+ uname -a
+ cc --version | sed -n '1p'
+ echo "libcjson $(pkg-config --modversion libcjson 2>/dev/null || echo MISSING) :: $(pkg-config --cflags --libs libcjson 2>/dev/null)"
+ echo "openblas $(pkg-config --modversion openblas 2>/dev/null || echo MISSING) :: $(pkg-config --cflags --libs openblas 2>/dev/null)"
+
+ cd /work
+ # /work is a bind mount; git refuses to operate on it without this opt-in.
+ git config --global --add safe.directory /work 2>/dev/null || true
+ make clean >/dev/null 2>&1 || true
+
+ echo "==================== make cpu ===================="
+ make cpu
+ ls -la ffwd-cli ffwd-server
+ ./ffwd-cli --version || true
+
+ echo "==================== make test ===================="
+ make test
+
+ echo "==================== make bench ===================="
+ make bench
+
+ echo "==================== make bench-server-utils ===================="
+ make bench-server-utils
+
+ echo "==================== make debug (sanitizers) ===================="
+ make debug
+
+ if [ -n "${KEEP:-}" ]; then
+ echo "==================== KEEP set: leaving artifacts in tree ===================="
+ else
+ echo "==================== make clean ===================="
+ make clean
+ residual=$(ls -d build ffwd-cli ffwd-server 2>/dev/null | wc -l | tr -d ' ')
+ echo "residual build artifacts: $residual (want 0)"
+ fi
+
+ echo "==================== ALL STEPS COMPLETED ===================="
+}
+
+# --- host side: launch the container and re-exec this script inside it --------
+launch_on_host() {
+ command -v container >/dev/null 2>&1 || {
+ echo "error: Apple 'container' CLI not found on PATH" >&2; exit 1; }
+
+ local repo_root script_rel
+ repo_root="$(git rev-parse --show-toplevel)"
+ # Path of this script relative to the repo root, so it resolves under /work.
+ script_rel="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")"
+ script_rel="${script_rel#"$repo_root"/}"
+
+ echo "repo: $repo_root"
+ echo "image: $IMAGE"
+ exec container run --rm \
+ --env "KEEP=${KEEP:-}" \
+ --mount "source=$repo_root,target=/work" \
+ "$IMAGE" \
+ bash "/work/$script_rel" --in-container
+}
+
+if [ "${1:-}" = "--in-container" ]; then
+ run_in_container
+else
+ launch_on_host
+fi
diff --git a/devtools/verify_builds.sh b/devtools/verify_builds.sh
new file mode 100755
index 0000000..25347b5
--- /dev/null
+++ b/devtools/verify_builds.sh
@@ -0,0 +1,330 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
+
+DEFAULT_MODEL_GLOB='models--perplexity-ai--pplx-embed-v1-0.6b/snapshots/*'
+DEFAULT_CONTEXT_MODEL_GLOB='models--perplexity-ai--pplx-embed-context-v1-0.6B/snapshots/*'
+
+model_dir=""
+context_model_dir=""
+binary="./ffwd-cli"
+batch_size=4
+skip_build=0
+skip_mlx=0
+platform_skip_mlx=0
+skip_contextual=0
+context_runs=0
+late_model_dir=""
+long=0
+skip_code_quality=0
+skip_hermetic=0
+skip_tokenizer=0
+skip_mlx_quantized=0
+
+die() {
+ echo "error: $*" >&2
+ exit 2
+}
+
+usage() {
+ cat <<EOF
+Usage: $(basename "$0") [options]
+
+Options:
+ --model-dir DIR 0.6B model snapshot; defaults to the HF cache
+ --context-model-dir DIR context 0.6B snapshot; defaults to the HF cache
+ --binary PATH binary to test, default: ./ffwd-cli
+ --batch-size N batch size, default: 4
+ --skip-build skip make cpu / make gpu
+ --skip-hermetic skip the model-free make test suite
+ --skip-mlx stop after the CPU build and CPU checks
+ --skip-contextual skip contextual parity checks
+ --skip-tokenizer skip Hugging Face tokenizer parity
+ --skip-mlx-quantized skip the MLX Q8 parity check
+ --context-runs N timed contextual batches per backend after parity
+ --late-model-dir DIR optional late-interaction 0.6B snapshot to verify
+ --long also run the long CPU attention parity check
+ --skip-code-quality skip the final compiler/analyzer/tidy CI gate
+ -h, --help show this help
+EOF
+}
+
+find_cached_snapshot() {
+ local glob="$1"
+ local label="$2"
+ local hub="$HOME/.cache/huggingface/hub"
+ local last=""
+ local p
+
+ while IFS= read -r p; do
+ if [[ -d "$p" && -f "$p/config.json" ]]; then
+ last="$p"
+ fi
+ done < <({ compgen -G "$hub/$glob" || true; } | sort)
+
+ if [[ -z "$last" ]]; then
+ echo "no cached $label snapshot found; pass the model dir" >&2
+ exit 1
+ fi
+
+ printf '%s\n' "$last"
+}
+
+find_default_model() {
+ find_cached_snapshot "$DEFAULT_MODEL_GLOB" "pplx-embed-v1-0.6b"
+}
+
+find_default_context_model() {
+ find_cached_snapshot "$DEFAULT_CONTEXT_MODEL_GLOB" "pplx-embed-context-v1-0.6b"
+}
+
+run() {
+ printf '+'
+ printf ' %q' "$@"
+ printf '\n'
+ (cd "$ROOT" && "$@")
+}
+
+run_parity() {
+ local binary_arg="$1"
+ local model_dir_arg="$2"
+ local backend="$3"
+ local batch_size_arg="$4"
+ local use_long="$5"
+
+ local cmd=(
+ "$ROOT/all.bash" check-batch-parity
+ --binary "$binary_arg"
+ --model-dir "$model_dir_arg"
+ --backend "$backend"
+ --batch-size "$batch_size_arg"
+ )
+
+ if (( use_long )); then
+ cmd+=(--long)
+ fi
+
+ run "${cmd[@]}"
+}
+
+run_contextual() {
+ local model_dir_arg="$1"
+ local backend="$2"
+ local runs="$3"
+
+ run \
+ "$ROOT/all.bash" check-context-batch-parity \
+ --model-dir "$model_dir_arg" \
+ --backend "$backend" \
+ --runs "$runs"
+}
+
+run_late() {
+ local model_dir_arg="$1"
+ local backend="$2"
+
+ run \
+ "$ROOT/all.bash" check-late-interaction \
+ --model-dir "$model_dir_arg" \
+ --backend "$backend"
+}
+
+while (($#)); do
+ case "$1" in
+ --model-dir)
+ (($# >= 2)) || die "--model-dir requires a value"
+ model_dir="$2"
+ shift 2
+ ;;
+ --model-dir=*)
+ model_dir="${1#*=}"
+ shift
+ ;;
+
+ --context-model-dir)
+ (($# >= 2)) || die "--context-model-dir requires a value"
+ context_model_dir="$2"
+ shift 2
+ ;;
+ --context-model-dir=*)
+ context_model_dir="${1#*=}"
+ shift
+ ;;
+
+ --binary)
+ (($# >= 2)) || die "--binary requires a value"
+ binary="$2"
+ shift 2
+ ;;
+ --binary=*)
+ binary="${1#*=}"
+ shift
+ ;;
+
+ --batch-size)
+ (($# >= 2)) || die "--batch-size requires a value"
+ batch_size="$2"
+ shift 2
+ ;;
+ --batch-size=*)
+ batch_size="${1#*=}"
+ shift
+ ;;
+
+ --skip-build)
+ skip_build=1
+ shift
+ ;;
+ --skip-hermetic)
+ skip_hermetic=1
+ shift
+ ;;
+ --skip-mlx)
+ skip_mlx=1
+ shift
+ ;;
+ --skip-contextual)
+ skip_contextual=1
+ shift
+ ;;
+ --skip-tokenizer)
+ skip_tokenizer=1
+ shift
+ ;;
+ --skip-mlx-quantized)
+ skip_mlx_quantized=1
+ shift
+ ;;
+
+ --context-runs)
+ (($# >= 2)) || die "--context-runs requires a value"
+ context_runs="$2"
+ shift 2
+ ;;
+ --context-runs=*)
+ context_runs="${1#*=}"
+ shift
+ ;;
+
+ --late-model-dir)
+ (($# >= 2)) || die "--late-model-dir requires a value"
+ late_model_dir="$2"
+ shift 2
+ ;;
+ --late-model-dir=*)
+ late_model_dir="${1#*=}"
+ shift
+ ;;
+
+ --long)
+ long=1
+ shift
+ ;;
+ --skip-code-quality)
+ skip_code_quality=1
+ shift
+ ;;
+
+ -h|--help)
+ usage
+ exit 0
+ ;;
+
+ *)
+ die "unknown argument: $1"
+ ;;
+ esac
+done
+
+if [[ "$(uname -s)" != "Darwin" && $skip_mlx -eq 0 ]]; then
+ echo "info: MLX checks require macOS; skipping on $(uname -s)"
+ skip_mlx=1
+ platform_skip_mlx=1
+fi
+
+if [[ -z "$model_dir" ]]; then
+ model_dir="$(find_default_model)"
+fi
+
+if (( skip_contextual )); then
+ context_model_dir=""
+elif [[ -z "$context_model_dir" ]]; then
+ context_model_dir="$(find_default_context_model)"
+fi
+
+if (( ! skip_hermetic )); then
+ run make test
+fi
+
+if (( ! skip_build )); then
+ run make cpu
+fi
+
+run "$ROOT/all.bash" check-kernel-golden --blas
+
+if (( ! skip_tokenizer )); then
+ run "$ROOT/all.bash" check-tokenizer-parity --model-dir "$model_dir"
+fi
+
+run "$ROOT/all.bash" check-workspace-api --model-dir "$model_dir"
+
+run_parity "$binary" "$model_dir" cpu "$batch_size" 0
+
+if (( long )); then
+ run_parity "$binary" "$model_dir" cpu "$batch_size" 1
+fi
+
+if [[ -n "$context_model_dir" ]]; then
+ run_contextual "$context_model_dir" cpu "$context_runs"
+fi
+
+if [[ -n "$late_model_dir" ]]; then
+ run_late "$late_model_dir" cpu
+fi
+
+if (( skip_mlx )); then
+ if (( platform_skip_mlx && ! skip_code_quality )); then
+ run "$ROOT/all.bash" check-code-quality
+ fi
+ if (( skip_build )); then
+ echo "ok: CPU checks passed; builds and MLX checks skipped"
+ else
+ echo "ok: CPU build verified; MLX build skipped"
+ fi
+ exit 0
+fi
+
+if (( ! skip_build )); then
+ run make gpu
+fi
+
+run_parity "$binary" "$model_dir" mlx "$batch_size" 0
+
+run \
+ "$ROOT/all.bash" check-mlx-model-limits \
+ --binary "$binary" \
+ --model-dir "$model_dir"
+
+if (( ! skip_mlx_quantized )); then
+ run \
+ "$ROOT/all.bash" check-mlx-quantized-parity \
+ --binary "$binary" \
+ --model-dir "$model_dir" \
+ --batch-size "$batch_size"
+fi
+
+if [[ -n "$context_model_dir" ]]; then
+ run_contextual "$context_model_dir" mlx "$context_runs"
+fi
+
+if [[ -n "$late_model_dir" ]]; then
+ run_late "$late_model_dir" mlx
+fi
+
+if (( ! skip_code_quality )); then
+ run "$ROOT/all.bash" check-code-quality
+fi
+
+echo "verify ok"
diff --git a/tests/check_cuda_fast_gemm_parity.py b/tests/check_cuda_fast_gemm_parity.py
new file mode 100755
index 0000000..06ee1f1
--- /dev/null
+++ b/tests/check_cuda_fast_gemm_parity.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""Check CUDA reduced-precision quality against exact CUDA F32.
+
+Covers both reduced-precision axes: compute mode (--mode tf32|bf16|16f,
+selecting --gpu-gemm-mode) and weight storage (--weights bf16, selecting
+--gpu-weight-dtype). The reference run always uses exact F32 weights and compute.
+For BF16 weight storage the int8 byte-change rate is inherently large
+(~7-15%) and informational only; cosine is the quality gate, so the int8
+threshold defaults to 1.0 in that case unless overridden."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
+
+
+DEFAULT_TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ (
+ "document: Embedding models turn text into dense vectors for semantic "
+ "retrieval. Long document inference stresses bidirectional attention "
+ "because every token attends to every other token in the same packed "
+ "sequence. "
+ ) * 3,
+ (
+ "document: Embedding models turn text into dense vectors for semantic "
+ "retrieval. Long document inference stresses bidirectional attention "
+ "because every token attends to every other token in the same packed "
+ "sequence. "
+ ) * 6,
+]
+# The longest passage stays under ~512 tokens so short-context encoders
+# (BERT/MiniLM/BGE max_position 512, XLM-R 514) accept it; larger 8x/14x inputs
+# overflowed their position tables and were (correctly) rejected, which looked
+# like a CUDA failure. Still long enough to stress packed-sequence attention.
+
+
+def run_cuda(binary: str, model_dir: str, mode: str, weights: str,
+ batch_size: int, texts: list[str]) -> list[list[float]]:
+ cmd = [
+ binary,
+ "-d", model_dir,
+ "--gpu-gemm-mode", mode,
+ "--gpu-weight-dtype", weights,
+ "--stream",
+ "-b", str(batch_size),
+ ]
+ proc = subprocess.run(
+ cmd,
+ input="\n".join(texts) + "\n",
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=False,
+ )
+ if proc.returncode != 0:
+ raise RuntimeError(
+ f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}"
+ )
+
+ rows: list[list[float]] = []
+ for line in proc.stdout.splitlines():
+ if not line.strip():
+ continue
+ row = json.loads(line)
+ if "error" in row:
+ raise RuntimeError(f"stdin returned error: {row}")
+ rows.append([float(x) for x in row["embedding"]])
+ if len(rows) != len(texts):
+ raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}")
+ return rows
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def quantize_int8_tanh(x: float) -> int:
+ v = round(math.tanh(x) * 127.0)
+ if v < -128:
+ return -128
+ if v > 127:
+ return 127
+ return int(v)
+
+
+def compare(name: str, ref: list[list[float]], got: list[list[float]]) -> tuple[float, float, int]:
+ worst_diff = 0.0
+ worst_cos = 1.0
+ int8_changes = 0
+ for i, (a, b) in enumerate(zip(ref, got)):
+ if len(a) != len(b):
+ raise RuntimeError(f"dimension mismatch at row {i}: {len(a)} vs {len(b)}")
+ max_diff = max(abs(x - y) for x, y in zip(a, b))
+ cos = cosine(a, b)
+ changes = sum(
+ quantize_int8_tanh(x) != quantize_int8_tanh(y)
+ for x, y in zip(a, b)
+ )
+ worst_diff = max(worst_diff, max_diff)
+ worst_cos = min(worst_cos, cos)
+ int8_changes += changes
+ print(
+ f"{name} [{i}] cosine={cos:.8f} "
+ f"max_abs_diff={max_diff:.8g} int8_changes={changes}"
+ )
+ return worst_diff, worst_cos, int8_changes
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--binary", default=str(DEFAULT_BINARY))
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--mode", choices=("f32", "tf32", "bf16", "16f"),
+ default="tf32")
+ ap.add_argument("--weights", choices=("f32", "bf16"), default="f32")
+ ap.add_argument("--batch-size", type=int, default=8)
+ ap.add_argument("--max-diff", type=float, default=None)
+ ap.add_argument("--min-cosine", type=float, default=0.999)
+ ap.add_argument("--max-int8-change-rate", type=float, default=None)
+ ap.add_argument("texts", nargs="*")
+ args = ap.parse_args()
+
+ if args.batch_size < 2:
+ raise SystemExit("--batch-size must be >= 2")
+ if args.mode == "f32" and args.weights == "f32":
+ raise SystemExit("nothing to check: --mode and --weights are both f32")
+ if args.max_diff is None:
+ args.max_diff = 0.05 if args.weights == "bf16" else 0.01
+ if args.max_int8_change_rate is None:
+ args.max_int8_change_rate = 1.0 if args.weights == "bf16" else 0.02
+
+ texts = args.texts or DEFAULT_TEXTS
+ ref_batch = run_cuda(args.binary, args.model_dir, "f32", "f32",
+ args.batch_size, texts)
+ fast_batch = run_cuda(args.binary, args.model_dir, args.mode,
+ args.weights, args.batch_size, texts)
+ fast_single = run_cuda(args.binary, args.model_dir, args.mode,
+ args.weights, 1, texts)
+
+ qdiff, qcos, qchanges = compare("quality", ref_batch, fast_batch)
+ sdiff, scos, schanges = compare("shape", fast_single, fast_batch)
+
+ total_values = len(texts) * len(ref_batch[0])
+ qrate = qchanges / total_values
+ srate = schanges / total_values
+ print(
+ f"ok: mode={args.mode} weights={args.weights} "
+ f"batch_size={args.batch_size} "
+ f"quality_diff={qdiff:.8g} quality_cos={qcos:.8f} "
+ f"quality_int8_change_rate={qrate:.6f} "
+ f"shape_diff={sdiff:.8g} shape_cos={scos:.8f} "
+ f"shape_int8_change_rate={srate:.6f}"
+ )
+
+ failed = (
+ qdiff > args.max_diff or qcos < args.min_cosine or
+ sdiff > args.max_diff or scos < args.min_cosine or
+ qrate > args.max_int8_change_rate or
+ srate > args.max_int8_change_rate
+ )
+ if failed:
+ raise SystemExit(
+ f"cuda parity failed (mode={args.mode} weights={args.weights}): "
+ f"quality diff={qdiff:g} cos={qcos:g} int8_rate={qrate:g}; "
+ f"shape diff={sdiff:g} cos={scos:g} int8_rate={srate:g}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except Exception as exc:
+ print(f"error: {exc}", file=sys.stderr)
+ raise SystemExit(1)
diff --git a/tests/check_mlx_model_limits.py b/tests/check_mlx_model_limits.py
new file mode 100755
index 0000000..08eb7ba
--- /dev/null
+++ b/tests/check_mlx_model_limits.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Verify that MLX rejects mismatched model metadata before touching Metal."""
+
+import argparse
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--binary", default=str(DEFAULT_BINARY))
+ ap.add_argument("--model-dir", required=True)
+ args = ap.parse_args()
+
+ binary = Path(args.binary).resolve()
+ model_dir = Path(args.model_dir).resolve()
+ with (model_dir / "config.json").open() as f:
+ config = json.load(f)
+
+ vocab_size = config.get("vocab_size")
+ if not isinstance(vocab_size, int) or vocab_size <= 1:
+ raise SystemExit("config.json has no usable vocab_size")
+ config["vocab_size"] = vocab_size - 1
+
+ with tempfile.TemporaryDirectory(prefix="ffwd-mlx-invalid-") as td:
+ tmp = Path(td)
+ (tmp / "config.json").write_text(json.dumps(config))
+ (tmp / "vocab.json").symlink_to(model_dir / "vocab.json")
+ tensors = list(model_dir.glob("*.safetensors"))
+ if not tensors:
+ raise SystemExit("model directory has no safetensor files")
+ for tensor in tensors:
+ (tmp / tensor.name).symlink_to(tensor)
+
+ proc = subprocess.run(
+ [str(binary), "-d", str(tmp), "--stream"],
+ input="",
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+
+ expected = "mlx: bad shape for embed_tokens.weight at dim 0"
+ if proc.returncode == 0:
+ raise SystemExit("MLX unexpectedly accepted mismatched vocab_size")
+ if expected not in proc.stderr:
+ raise SystemExit(f"unexpected stderr:\n{proc.stderr}")
+ if "MLX error:" in proc.stderr:
+ raise SystemExit(f"MLX touched Metal before rejecting the model:\n{proc.stderr}")
+ print("ok: MLX rejected mismatched tensor shape before Metal initialization")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/check_mlx_quantized_parity.py b/tests/check_mlx_quantized_parity.py
new file mode 100755
index 0000000..2cf0add
--- /dev/null
+++ b/tests/check_mlx_quantized_parity.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Compare normal MLX embeddings against opt-in quantized MLX weights."""
+
+import argparse
+import json
+import math
+import subprocess
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli"
+
+
+TEXTS = [
+ "query: what is the capital of France?",
+ "document: Paris is the capital of France.",
+ "document: Berlin is the capital of Germany.",
+ "document: Istanbul is a major city in Turkey.",
+ "The quick brown fox jumps over the lazy dog near the riverbank.",
+ (
+ "Retrieval augmented generation combines embedding models with "
+ "external knowledge sources and ranks documents by semantic similarity."
+ ),
+]
+
+
+def run(binary: str, model_dir: str, texts: list[str], batch_size: int,
+ bits: int, group_size: int) -> list[list[float]]:
+ cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)]
+ if bits:
+ cmd.extend([
+ "--gpu-quant-bits", str(bits),
+ "--gpu-quant-group-size", str(group_size),
+ ])
+ proc = subprocess.run(
+ cmd,
+ input="\n".join(texts) + "\n",
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=False,
+ )
+ if proc.returncode != 0:
+ raise RuntimeError(
+ f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}"
+ )
+ rows = []
+ for line in proc.stdout.splitlines():
+ if not line.strip():
+ continue
+ row = json.loads(line)
+ if "error" in row:
+ raise RuntimeError(f"stdin returned error: {row}")
+ rows.append(row["embedding"])
+ if len(rows) != len(texts):
+ raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}")
+ return rows
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ return dot / (na * nb)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--binary", default=str(DEFAULT_BINARY))
+ ap.add_argument("--model-dir", required=True)
+ ap.add_argument("--batch-size", type=int, default=4)
+ ap.add_argument("--bits", type=int, default=8, choices=(8,))
+ ap.add_argument("--group-size", type=int, default=64)
+ ap.add_argument("--max-diff", type=float, default=0.08)
+ ap.add_argument("--min-cosine", type=float, default=0.995)
+ args = ap.parse_args()
+
+ ref = run(args.binary, args.model_dir, TEXTS, args.batch_size, 0,
+ args.group_size)
+ got = run(args.binary, args.model_dir, TEXTS, args.batch_size, args.bits,
+ args.group_size)
+
+ worst_diff = 0.0
+ worst_cos = 1.0
+ for i, (a, b) in enumerate(zip(ref, got)):
+ max_diff = max(abs(x - y) for x, y in zip(a, b))
+ cos = cosine(a, b)
+ worst_diff = max(worst_diff, max_diff)
+ worst_cos = min(worst_cos, cos)
+ print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}")
+
+ print(
+ f"ok: bits={args.bits} group_size={args.group_size} "
+ f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}"
+ )
+ if worst_diff > args.max_diff or worst_cos < args.min_cosine:
+ raise SystemExit(
+ f"quantized parity failed: diff={worst_diff:g}, cos={worst_cos:g}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())