diff options
| -rwxr-xr-x | all.bash | 278 | ||||
| -rwxr-xr-x | devtools/check_code_quality.sh | 181 | ||||
| -rwxr-xr-x | devtools/container-build-test.sh | 87 | ||||
| -rwxr-xr-x | devtools/verify_builds.sh | 330 | ||||
| -rwxr-xr-x | tests/check_cuda_fast_gemm_parity.py | 188 | ||||
| -rwxr-xr-x | tests/check_mlx_model_limits.py | 60 | ||||
| -rwxr-xr-x | tests/check_mlx_quantized_parity.py | 104 |
7 files changed, 1228 insertions, 0 deletions
diff --git a/all.bash b/all.bash new file mode 100755 index 0000000..5f3dbcc --- /dev/null +++ b/all.bash @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +set -uo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +export UV_CACHE_DIR="${UV_CACHE_DIR:-$ROOT/.venv/uv-cache}" + +RED="" +GREEN="" +YELLOW="" +NORMAL="" +MAXWIDTH=0 + +if [[ -t 2 ]] && command -v tput >/dev/null 2>&1; then + RED="$(tput setaf 1 2>/dev/null || true)" + GREEN="$(tput setaf 2 2>/dev/null || true)" + YELLOW="$(tput setaf 3 2>/dev/null || true)" + NORMAL="$(tput sgr0 2>/dev/null || true)" + MAXWIDTH="$(tput cols 2>/dev/null || printf '0')" + if (( MAXWIDTH > 2 )); then + MAXWIDTH=$((MAXWIDTH - 2)) + else + MAXWIDTH=0 + fi +fi + +info() { printf '%sINFO: %s%s\n' "$GREEN" "$*" "$NORMAL" >&2; } +warn() { printf '%sWARN: %s%s\n' "$YELLOW" "$*" "$NORMAL" >&2; } +err() { printf '%sERROR: %s%s\n' "$RED" "$*" "$NORMAL" >&2; } + +run() { + local rendered + printf -v rendered '%q ' "$@" + rendered="${rendered% }" + if (( MAXWIDTH > 0 && ${#rendered} > MAXWIDTH )); then + rendered="${rendered:0:$((MAXWIDTH - 3))}..." + fi + info "\$ $rendered" + (cd "$ROOT" && "$@") +} + +run_uv() { + run uv run "$@" +} + +# Run a Python check that needs the pinned SentenceTransformers reference stack +# (torch, sentence-transformers); the pins live in requirements-reference.txt. +run_reference() { + run uv run --with-requirements requirements-reference.txt "$@" +} + +check_code_quality() { + run ./devtools/check_code_quality.sh "$@" +} + +run_checks() { + run ./devtools/verify_builds.sh "$@" +} + +check_kernel_golden() { + # Build and run the math-kernel golden tests straight from the test Makefile. + # test_kernels exercises the BLAS-built kernels; test_kernels_generic the + # scalar/SIMD path with USE_BLAS off. Extra args (e.g. --blas, kept for + # backward compatibility) are accepted but ignored; both variants always run. + run make -C tests test_kernels test_kernels_generic + run ./tests/test_kernels + run ./tests/test_kernels_generic +} + +check_batch_parity() { + run_uv ./tests/check_batch_parity.py "$@" +} + +check_contextual_batch_parity() { + run_uv ./tests/check_contextual_batch_parity.py "$@" +} + +check_late_interaction() { + run_uv ./tests/check_late_interaction.py "$@" +} + +check_cuda_fast_gemm_parity() { + if [[ "$(uname -s)" != "Linux" ]]; then + warn "CUDA fast GEMM parity requires Linux; skipping on $(uname -s)" + return 0 + fi + if ! command -v nvidia-smi >/dev/null 2>&1; then + warn "CUDA fast GEMM parity requires an NVIDIA GPU; nvidia-smi not found" + return 0 + fi + run_uv ./tests/check_cuda_fast_gemm_parity.py "$@" +} + +check_embedding_parity() { + run_uv ./devtools/compare_model_embeddings.py "$@" +} + +check_mlx_model_limits() { + if [[ "$(uname -s)" != "Darwin" ]]; then + warn "MLX metadata limits require macOS; skipping on $(uname -s)" + return 0 + fi + run_uv ./tests/check_mlx_model_limits.py "$@" +} + +check_mlx_quantized_parity() { + if [[ "$(uname -s)" != "Darwin" ]]; then + warn "MLX quantized parity requires macOS; skipping on $(uname -s)" + return 0 + fi + run_uv ./tests/check_mlx_quantized_parity.py "$@" +} + +check_quant_precision_drift() { + run_reference ./devtools/quant_precision_drift.py "$@" +} + +check_reference_parity() { + run_reference ./tests/check_reference_parity.py "$@" +} + +check_bert_parity() { + run_reference ./tests/check_bert_parity.py "$@" +} + +check_e5_parity() { + run_reference ./tests/check_e5_parity.py "$@" +} + +check_qwen3_parity() { + run uv run --with sentence-transformers --with torch --with numpy \ + ./tests/check_qwen3_parity.py "$@" +} + +check_tokenizer_parity() { + run_uv ./tests/check_tokenizer_parity.py "$@" +} + +check_wordpiece_parity() { + run_uv ./tests/check_wordpiece_parity.py "$@" +} + +check_model_matrix() { + run_uv ./tests/check_model_matrix.py "$@" +} + +check_perplexity_sdk_compat() { + run uv run --with perplexityai --with numpy \ + ./tests/check_perplexity_sdk_compat.py "$@" +} + +check_openai_sdk_compat() { + run uv run --with openai ./tests/check_openai_sdk_compat.py "$@" +} + +check_server_api() { + run_uv ./tests/check_server_api.py "$@" +} + +check_cli() { + run_uv ./tests/check_cli.py "$@" +} + +check_workspace_api() { + run_uv ./tests/check_workspace_api.py "$@" +} + +bench_contextual_api() { + run_uv ./bench/bench_contextual_api.py "$@" +} + +bench_late_rerank() { + run_uv ./bench/bench_late_rerank.py "$@" +} + +bench_late_api() { + run_uv ./bench/bench_late_api.py "$@" +} + +bench_matrix() { + run_uv ./bench/bench_matrix.py "$@" +} + +bench_qwen_tokenizer() { + run_uv ./bench/bench_qwen_tokenizer.py "$@" +} + +bench_server_api() { + run_uv ./bench/bench_server_api.py "$@" +} + +usage() { + cat <<'EOF' +Usage: ./all.bash COMMAND [ARGS...] + +Aggregate: + checks Build and run the comprehensive local check suite + +Checks: + check-code-quality Run clang builds, analysis, and linting + check-kernel-golden Run math-kernel golden tests + check-batch-parity Compare singleton and true batched stdin execution + check-context-batch-parity Compare contextual batch and singleton execution + check-cuda-fast-gemm-parity Gate CUDA reduced-precision modes against exact F32 + check-embedding-parity Compare embeddings from two model directories + check-late-interaction Check late-interaction vectors and MaxSim ranking + check-mlx-model-limits Check MLX model metadata rejection + check-mlx-quantized-parity Compare normal and quantized MLX embeddings + check-precision-drift Compare low-precision drift across engines + check-reference-parity Compare a running server with the Python reference + check-bert-parity Compare BERT embeddings with SentenceTransformers + check-e5-parity Compare XLM-R/E5 embeddings with SentenceTransformers + check-qwen3-parity Compare Qwen3 embeddings with SentenceTransformers + check-tokenizer-parity Compare the C BPE tokenizer with stored vectors + check-wordpiece-parity Compare the C WordPiece tokenizer with stored vectors + check-model-matrix Smoke-check supported model variants + check-server-api Verify a running local HTTP server + check-cli Drive the CLI binary through its modes and flags + check-workspace-api Check the model/workspace API with real weights + check-perplexity-sdk-compat Verify a running server through the Perplexity SDK + check-openai-sdk-compat Verify a running server through the OpenAI SDK + +Benchmarks: + bench-contextual-api Benchmark contextual inference through HTTP + bench-late-rerank Benchmark late-interaction MaxSim reranking + bench-late-api Benchmark HTTP reranking; optionally compare PyLate + bench-matrix Run the batch-aware stdin benchmark matrix + bench-qwen-tokenizer Benchmark tokenizer encode paths + bench-server-api Benchmark concurrent HTTP embedding requests +EOF +} + +main() { + local command="${1:-help}" + if (($#)); then + shift + fi + + case "$command" in + help|-h|--help) usage ;; + checks) run_checks "$@" ;; + check-code-quality) check_code_quality "$@" ;; + check-kernel-golden) check_kernel_golden "$@" ;; + check-batch-parity) check_batch_parity "$@" ;; + check-context-batch-parity) check_contextual_batch_parity "$@" ;; + check-late-interaction) check_late_interaction "$@" ;; + check-cuda-fast-gemm-parity) check_cuda_fast_gemm_parity "$@" ;; + check-embedding-parity) check_embedding_parity "$@" ;; + check-mlx-model-limits) check_mlx_model_limits "$@" ;; + check-mlx-quantized-parity) check_mlx_quantized_parity "$@" ;; + check-precision-drift) check_quant_precision_drift "$@" ;; + check-reference-parity) check_reference_parity "$@" ;; + check-bert-parity) check_bert_parity "$@" ;; + check-e5-parity) check_e5_parity "$@" ;; + check-qwen3-parity) check_qwen3_parity "$@" ;; + check-tokenizer-parity) check_tokenizer_parity "$@" ;; + check-wordpiece-parity) check_wordpiece_parity "$@" ;; + check-model-matrix) check_model_matrix "$@" ;; + check-perplexity-sdk-compat) check_perplexity_sdk_compat "$@" ;; + check-openai-sdk-compat) check_openai_sdk_compat "$@" ;; + check-server-api) check_server_api "$@" ;; + check-cli) check_cli "$@" ;; + check-workspace-api) check_workspace_api "$@" ;; + bench-contextual-api) bench_contextual_api "$@" ;; + bench-late-rerank) bench_late_rerank "$@" ;; + bench-late-api) bench_late_api "$@" ;; + bench-matrix) bench_matrix "$@" ;; + bench-qwen-tokenizer) bench_qwen_tokenizer "$@" ;; + bench-server-api) bench_server_api "$@" ;; + *) + err "unknown command: $command" + usage >&2 + return 2 + ;; + esac +} + +main "$@" diff --git a/devtools/check_code_quality.sh b/devtools/check_code_quality.sh new file mode 100755 index 0000000..305ba50 --- /dev/null +++ b/devtools/check_code_quality.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" +cd "$ROOT" + +# Lint and analyze every C source under libffwd/, tests/, and bench/. Each +# platform keeps its own GPU backend and skips the other's, which it does not +# build (macOS builds MLX, Linux builds CUDA). +SYS="$(uname -s)" +SRC_SOURCES=() +for f in libffwd/*.c; do + [[ "$SYS" == Darwin && "$f" == libffwd/backend_cuda.c ]] && continue + [[ "$SYS" != Darwin && ( "$f" == libffwd/mlx.c || "$f" == libffwd/backend_mlx.c ) ]] && continue + SRC_SOURCES+=("$f") +done + +TEST_C_SOURCES=(tests/*.c) +BENCH_C_SOURCES=(bench/*.c) + +# CUDA backend sources are C++17 and only built on Linux. Parse them with the +# clang CUDA front end in host-only mode so no GPU or device toolchain is +# required; macOS has no CUDA toolkit, so skip them there. +CU_SOURCES=() +if [[ "$SYS" != Darwin ]]; then + for f in libffwd/*.cu; do + [[ -e "$f" ]] && CU_SOURCES+=("$f") + done +fi + +ANALYZE_SOURCES=("${SRC_SOURCES[@]}") +TIDY_SOURCES=("${SRC_SOURCES[@]}" "${TEST_C_SOURCES[@]}" "${BENCH_C_SOURCES[@]}") + +ANALYZE_COMMON=( + clang --analyze -Xanalyzer -analyzer-output=text + -Wall -Wextra + -I. -Ilibffwd -Ideps/ae +) + +PLATFORM_CFLAGS=() +if [[ "$(uname -s)" == "Darwin" ]]; then + PLATFORM_CFLAGS=( + -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX + -I/opt/homebrew/opt/mlx-c/include + -I/opt/homebrew/include + -I/usr/local/include + ) +else + PLATFORM_CFLAGS=( + -DUSE_BLAS -DUSE_OPENBLAS + -I/usr/include/openblas + ) +fi + +# Flags for the CUDA backend front end. CUDA_HOME matches the Makefile default. +# --cuda-host-only parses host code only so device codegen and libdevice are not +# needed; the macros mirror the nvcc command in the Makefile. +CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" +CUDA_CFLAGS=( + -x cuda --cuda-host-only "--cuda-path=$CUDA_HOME" + -std=c++17 + -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU + -I"$CUDA_HOME/include" + -I/usr/include/openblas +) + +print_cmd() { + printf '+' + printf ' %q' "$@" + printf '\n' +} + +run() { + print_cmd "$@" + "$@" +} + +run_analyze() { + local output + local status + + print_cmd "$@" + + set +e + output="$("$@" 2>&1)" + status=$? + set -e + + if [[ -n "$output" ]]; then + printf '%s\n' "$output" + fi + + if (( status != 0 )); then + return "$status" + fi + + if grep -q 'warning:' <<<"$output"; then + echo "clang analyzer warnings found" >&2 + return 1 + fi +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +find_clang_tidy() { + local brew_path="/opt/homebrew/opt/llvm@21/bin/clang-tidy" + + if command -v clang-tidy >/dev/null 2>&1; then + command -v clang-tidy + return 0 + fi + + if [[ -x "$brew_path" ]]; then + printf '%s\n' "$brew_path" + return 0 + fi + + return 1 +} + +main() { + local clang_tidy + + if ! have clang; then + echo "clang not found" >&2 + return 1 + fi + + run git --no-pager diff --check + run make cpu CC=clang + + run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ + "${ANALYZE_SOURCES[@]}" + run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ + -I. -Ilibffwd -Itests "${TEST_C_SOURCES[@]}" + run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \ + -I. -Ilibffwd -Ibench "${BENCH_C_SOURCES[@]}" + + if (( ${#CU_SOURCES[@]} > 0 )); then + run_analyze "${ANALYZE_COMMON[@]}" "${CUDA_CFLAGS[@]}" \ + "${CU_SOURCES[@]}" + fi + + if clang_tidy="$(find_clang_tidy)"; then + run \ + "$clang_tidy" \ + --config-file=.clang-tidy \ + "${TIDY_SOURCES[@]}" \ + -- \ + -std=c17 \ + "${PLATFORM_CFLAGS[@]}" \ + -I. \ + -Ilibffwd \ + -Itests \ + -Ibench \ + -Ideps/ae + + if (( ${#CU_SOURCES[@]} > 0 )); then + run \ + "$clang_tidy" \ + --config-file=.clang-tidy \ + "${CU_SOURCES[@]}" \ + -- \ + "${CUDA_CFLAGS[@]}" \ + -I. \ + -Ilibffwd \ + -Ideps/ae + fi + else + echo "clang-tidy not found; skipping" + fi + + if [[ "$(uname -s)" == "Darwin" ]]; then + run make gpu CC=clang + fi +} + +main "$@" diff --git a/devtools/container-build-test.sh b/devtools/container-build-test.sh new file mode 100755 index 0000000..4dcc627 --- /dev/null +++ b/devtools/container-build-test.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Build, test and bench the OpenBLAS backend in a throwaway Ubuntu container to +# catch Linux portability breakage a macOS/Accelerate build hides. Needs Apple's +# `container` CLI running. Run from anywhere in the repo: it finds the repo root +# via git, bind-mounts it at /work, installs the Runpod toolchain, re-execs +# itself inside with --in-container, and runs make cpu/test/bench/bench-server- +# utils/debug, then `make clean` (skip with KEEP=1). Override the image with +# IMAGE=ubuntu:22.04. Exits non-zero at the first failing step. +set -euo pipefail + +IMAGE="${IMAGE:-ubuntu:24.04}" + +# --- inside the container: bootstrap toolchain, then build/test --------------- +run_in_container() { + export DEBIAN_FRONTEND=noninteractive + + echo "==================== apt bootstrap ====================" + apt-get update -qq + apt-get install -y --no-install-recommends \ + build-essential ca-certificates libcjson-dev libopenblas-dev \ + pkg-config git >/dev/null + + echo "==================== system / toolchain ====================" + uname -a + cc --version | sed -n '1p' + echo "libcjson $(pkg-config --modversion libcjson 2>/dev/null || echo MISSING) :: $(pkg-config --cflags --libs libcjson 2>/dev/null)" + echo "openblas $(pkg-config --modversion openblas 2>/dev/null || echo MISSING) :: $(pkg-config --cflags --libs openblas 2>/dev/null)" + + cd /work + # /work is a bind mount; git refuses to operate on it without this opt-in. + git config --global --add safe.directory /work 2>/dev/null || true + make clean >/dev/null 2>&1 || true + + echo "==================== make cpu ====================" + make cpu + ls -la ffwd-cli ffwd-server + ./ffwd-cli --version || true + + echo "==================== make test ====================" + make test + + echo "==================== make bench ====================" + make bench + + echo "==================== make bench-server-utils ====================" + make bench-server-utils + + echo "==================== make debug (sanitizers) ====================" + make debug + + if [ -n "${KEEP:-}" ]; then + echo "==================== KEEP set: leaving artifacts in tree ====================" + else + echo "==================== make clean ====================" + make clean + residual=$(ls -d build ffwd-cli ffwd-server 2>/dev/null | wc -l | tr -d ' ') + echo "residual build artifacts: $residual (want 0)" + fi + + echo "==================== ALL STEPS COMPLETED ====================" +} + +# --- host side: launch the container and re-exec this script inside it -------- +launch_on_host() { + command -v container >/dev/null 2>&1 || { + echo "error: Apple 'container' CLI not found on PATH" >&2; exit 1; } + + local repo_root script_rel + repo_root="$(git rev-parse --show-toplevel)" + # Path of this script relative to the repo root, so it resolves under /work. + script_rel="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" + script_rel="${script_rel#"$repo_root"/}" + + echo "repo: $repo_root" + echo "image: $IMAGE" + exec container run --rm \ + --env "KEEP=${KEEP:-}" \ + --mount "source=$repo_root,target=/work" \ + "$IMAGE" \ + bash "/work/$script_rel" --in-container +} + +if [ "${1:-}" = "--in-container" ]; then + run_in_container +else + launch_on_host +fi diff --git a/devtools/verify_builds.sh b/devtools/verify_builds.sh new file mode 100755 index 0000000..25347b5 --- /dev/null +++ b/devtools/verify_builds.sh @@ -0,0 +1,330 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" + +DEFAULT_MODEL_GLOB='models--perplexity-ai--pplx-embed-v1-0.6b/snapshots/*' +DEFAULT_CONTEXT_MODEL_GLOB='models--perplexity-ai--pplx-embed-context-v1-0.6B/snapshots/*' + +model_dir="" +context_model_dir="" +binary="./ffwd-cli" +batch_size=4 +skip_build=0 +skip_mlx=0 +platform_skip_mlx=0 +skip_contextual=0 +context_runs=0 +late_model_dir="" +long=0 +skip_code_quality=0 +skip_hermetic=0 +skip_tokenizer=0 +skip_mlx_quantized=0 + +die() { + echo "error: $*" >&2 + exit 2 +} + +usage() { + cat <<EOF +Usage: $(basename "$0") [options] + +Options: + --model-dir DIR 0.6B model snapshot; defaults to the HF cache + --context-model-dir DIR context 0.6B snapshot; defaults to the HF cache + --binary PATH binary to test, default: ./ffwd-cli + --batch-size N batch size, default: 4 + --skip-build skip make cpu / make gpu + --skip-hermetic skip the model-free make test suite + --skip-mlx stop after the CPU build and CPU checks + --skip-contextual skip contextual parity checks + --skip-tokenizer skip Hugging Face tokenizer parity + --skip-mlx-quantized skip the MLX Q8 parity check + --context-runs N timed contextual batches per backend after parity + --late-model-dir DIR optional late-interaction 0.6B snapshot to verify + --long also run the long CPU attention parity check + --skip-code-quality skip the final compiler/analyzer/tidy CI gate + -h, --help show this help +EOF +} + +find_cached_snapshot() { + local glob="$1" + local label="$2" + local hub="$HOME/.cache/huggingface/hub" + local last="" + local p + + while IFS= read -r p; do + if [[ -d "$p" && -f "$p/config.json" ]]; then + last="$p" + fi + done < <({ compgen -G "$hub/$glob" || true; } | sort) + + if [[ -z "$last" ]]; then + echo "no cached $label snapshot found; pass the model dir" >&2 + exit 1 + fi + + printf '%s\n' "$last" +} + +find_default_model() { + find_cached_snapshot "$DEFAULT_MODEL_GLOB" "pplx-embed-v1-0.6b" +} + +find_default_context_model() { + find_cached_snapshot "$DEFAULT_CONTEXT_MODEL_GLOB" "pplx-embed-context-v1-0.6b" +} + +run() { + printf '+' + printf ' %q' "$@" + printf '\n' + (cd "$ROOT" && "$@") +} + +run_parity() { + local binary_arg="$1" + local model_dir_arg="$2" + local backend="$3" + local batch_size_arg="$4" + local use_long="$5" + + local cmd=( + "$ROOT/all.bash" check-batch-parity + --binary "$binary_arg" + --model-dir "$model_dir_arg" + --backend "$backend" + --batch-size "$batch_size_arg" + ) + + if (( use_long )); then + cmd+=(--long) + fi + + run "${cmd[@]}" +} + +run_contextual() { + local model_dir_arg="$1" + local backend="$2" + local runs="$3" + + run \ + "$ROOT/all.bash" check-context-batch-parity \ + --model-dir "$model_dir_arg" \ + --backend "$backend" \ + --runs "$runs" +} + +run_late() { + local model_dir_arg="$1" + local backend="$2" + + run \ + "$ROOT/all.bash" check-late-interaction \ + --model-dir "$model_dir_arg" \ + --backend "$backend" +} + +while (($#)); do + case "$1" in + --model-dir) + (($# >= 2)) || die "--model-dir requires a value" + model_dir="$2" + shift 2 + ;; + --model-dir=*) + model_dir="${1#*=}" + shift + ;; + + --context-model-dir) + (($# >= 2)) || die "--context-model-dir requires a value" + context_model_dir="$2" + shift 2 + ;; + --context-model-dir=*) + context_model_dir="${1#*=}" + shift + ;; + + --binary) + (($# >= 2)) || die "--binary requires a value" + binary="$2" + shift 2 + ;; + --binary=*) + binary="${1#*=}" + shift + ;; + + --batch-size) + (($# >= 2)) || die "--batch-size requires a value" + batch_size="$2" + shift 2 + ;; + --batch-size=*) + batch_size="${1#*=}" + shift + ;; + + --skip-build) + skip_build=1 + shift + ;; + --skip-hermetic) + skip_hermetic=1 + shift + ;; + --skip-mlx) + skip_mlx=1 + shift + ;; + --skip-contextual) + skip_contextual=1 + shift + ;; + --skip-tokenizer) + skip_tokenizer=1 + shift + ;; + --skip-mlx-quantized) + skip_mlx_quantized=1 + shift + ;; + + --context-runs) + (($# >= 2)) || die "--context-runs requires a value" + context_runs="$2" + shift 2 + ;; + --context-runs=*) + context_runs="${1#*=}" + shift + ;; + + --late-model-dir) + (($# >= 2)) || die "--late-model-dir requires a value" + late_model_dir="$2" + shift 2 + ;; + --late-model-dir=*) + late_model_dir="${1#*=}" + shift + ;; + + --long) + long=1 + shift + ;; + --skip-code-quality) + skip_code_quality=1 + shift + ;; + + -h|--help) + usage + exit 0 + ;; + + *) + die "unknown argument: $1" + ;; + esac +done + +if [[ "$(uname -s)" != "Darwin" && $skip_mlx -eq 0 ]]; then + echo "info: MLX checks require macOS; skipping on $(uname -s)" + skip_mlx=1 + platform_skip_mlx=1 +fi + +if [[ -z "$model_dir" ]]; then + model_dir="$(find_default_model)" +fi + +if (( skip_contextual )); then + context_model_dir="" +elif [[ -z "$context_model_dir" ]]; then + context_model_dir="$(find_default_context_model)" +fi + +if (( ! skip_hermetic )); then + run make test +fi + +if (( ! skip_build )); then + run make cpu +fi + +run "$ROOT/all.bash" check-kernel-golden --blas + +if (( ! skip_tokenizer )); then + run "$ROOT/all.bash" check-tokenizer-parity --model-dir "$model_dir" +fi + +run "$ROOT/all.bash" check-workspace-api --model-dir "$model_dir" + +run_parity "$binary" "$model_dir" cpu "$batch_size" 0 + +if (( long )); then + run_parity "$binary" "$model_dir" cpu "$batch_size" 1 +fi + +if [[ -n "$context_model_dir" ]]; then + run_contextual "$context_model_dir" cpu "$context_runs" +fi + +if [[ -n "$late_model_dir" ]]; then + run_late "$late_model_dir" cpu +fi + +if (( skip_mlx )); then + if (( platform_skip_mlx && ! skip_code_quality )); then + run "$ROOT/all.bash" check-code-quality + fi + if (( skip_build )); then + echo "ok: CPU checks passed; builds and MLX checks skipped" + else + echo "ok: CPU build verified; MLX build skipped" + fi + exit 0 +fi + +if (( ! skip_build )); then + run make gpu +fi + +run_parity "$binary" "$model_dir" mlx "$batch_size" 0 + +run \ + "$ROOT/all.bash" check-mlx-model-limits \ + --binary "$binary" \ + --model-dir "$model_dir" + +if (( ! skip_mlx_quantized )); then + run \ + "$ROOT/all.bash" check-mlx-quantized-parity \ + --binary "$binary" \ + --model-dir "$model_dir" \ + --batch-size "$batch_size" +fi + +if [[ -n "$context_model_dir" ]]; then + run_contextual "$context_model_dir" mlx "$context_runs" +fi + +if [[ -n "$late_model_dir" ]]; then + run_late "$late_model_dir" mlx +fi + +if (( ! skip_code_quality )); then + run "$ROOT/all.bash" check-code-quality +fi + +echo "verify ok" diff --git a/tests/check_cuda_fast_gemm_parity.py b/tests/check_cuda_fast_gemm_parity.py new file mode 100755 index 0000000..06ee1f1 --- /dev/null +++ b/tests/check_cuda_fast_gemm_parity.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Check CUDA reduced-precision quality against exact CUDA F32. + +Covers both reduced-precision axes: compute mode (--mode tf32|bf16|16f, +selecting --gpu-gemm-mode) and weight storage (--weights bf16, selecting +--gpu-weight-dtype). The reference run always uses exact F32 weights and compute. +For BF16 weight storage the int8 byte-change rate is inherently large +(~7-15%) and informational only; cosine is the quality gate, so the int8 +threshold defaults to 1.0 in that case unless overridden.""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +DEFAULT_TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + ( + "document: Embedding models turn text into dense vectors for semantic " + "retrieval. Long document inference stresses bidirectional attention " + "because every token attends to every other token in the same packed " + "sequence. " + ) * 3, + ( + "document: Embedding models turn text into dense vectors for semantic " + "retrieval. Long document inference stresses bidirectional attention " + "because every token attends to every other token in the same packed " + "sequence. " + ) * 6, +] +# The longest passage stays under ~512 tokens so short-context encoders +# (BERT/MiniLM/BGE max_position 512, XLM-R 514) accept it; larger 8x/14x inputs +# overflowed their position tables and were (correctly) rejected, which looked +# like a CUDA failure. Still long enough to stress packed-sequence attention. + + +def run_cuda(binary: str, model_dir: str, mode: str, weights: str, + batch_size: int, texts: list[str]) -> list[list[float]]: + cmd = [ + binary, + "-d", model_dir, + "--gpu-gemm-mode", mode, + "--gpu-weight-dtype", weights, + "--stream", + "-b", str(batch_size), + ] + proc = subprocess.run( + cmd, + input="\n".join(texts) + "\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}" + ) + + rows: list[list[float]] = [] + for line in proc.stdout.splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin returned error: {row}") + rows.append([float(x) for x in row["embedding"]]) + if len(rows) != len(texts): + raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}") + return rows + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def quantize_int8_tanh(x: float) -> int: + v = round(math.tanh(x) * 127.0) + if v < -128: + return -128 + if v > 127: + return 127 + return int(v) + + +def compare(name: str, ref: list[list[float]], got: list[list[float]]) -> tuple[float, float, int]: + worst_diff = 0.0 + worst_cos = 1.0 + int8_changes = 0 + for i, (a, b) in enumerate(zip(ref, got)): + if len(a) != len(b): + raise RuntimeError(f"dimension mismatch at row {i}: {len(a)} vs {len(b)}") + max_diff = max(abs(x - y) for x, y in zip(a, b)) + cos = cosine(a, b) + changes = sum( + quantize_int8_tanh(x) != quantize_int8_tanh(y) + for x, y in zip(a, b) + ) + worst_diff = max(worst_diff, max_diff) + worst_cos = min(worst_cos, cos) + int8_changes += changes + print( + f"{name} [{i}] cosine={cos:.8f} " + f"max_abs_diff={max_diff:.8g} int8_changes={changes}" + ) + return worst_diff, worst_cos, int8_changes + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--mode", choices=("f32", "tf32", "bf16", "16f"), + default="tf32") + ap.add_argument("--weights", choices=("f32", "bf16"), default="f32") + ap.add_argument("--batch-size", type=int, default=8) + ap.add_argument("--max-diff", type=float, default=None) + ap.add_argument("--min-cosine", type=float, default=0.999) + ap.add_argument("--max-int8-change-rate", type=float, default=None) + ap.add_argument("texts", nargs="*") + args = ap.parse_args() + + if args.batch_size < 2: + raise SystemExit("--batch-size must be >= 2") + if args.mode == "f32" and args.weights == "f32": + raise SystemExit("nothing to check: --mode and --weights are both f32") + if args.max_diff is None: + args.max_diff = 0.05 if args.weights == "bf16" else 0.01 + if args.max_int8_change_rate is None: + args.max_int8_change_rate = 1.0 if args.weights == "bf16" else 0.02 + + texts = args.texts or DEFAULT_TEXTS + ref_batch = run_cuda(args.binary, args.model_dir, "f32", "f32", + args.batch_size, texts) + fast_batch = run_cuda(args.binary, args.model_dir, args.mode, + args.weights, args.batch_size, texts) + fast_single = run_cuda(args.binary, args.model_dir, args.mode, + args.weights, 1, texts) + + qdiff, qcos, qchanges = compare("quality", ref_batch, fast_batch) + sdiff, scos, schanges = compare("shape", fast_single, fast_batch) + + total_values = len(texts) * len(ref_batch[0]) + qrate = qchanges / total_values + srate = schanges / total_values + print( + f"ok: mode={args.mode} weights={args.weights} " + f"batch_size={args.batch_size} " + f"quality_diff={qdiff:.8g} quality_cos={qcos:.8f} " + f"quality_int8_change_rate={qrate:.6f} " + f"shape_diff={sdiff:.8g} shape_cos={scos:.8f} " + f"shape_int8_change_rate={srate:.6f}" + ) + + failed = ( + qdiff > args.max_diff or qcos < args.min_cosine or + sdiff > args.max_diff or scos < args.min_cosine or + qrate > args.max_int8_change_rate or + srate > args.max_int8_change_rate + ) + if failed: + raise SystemExit( + f"cuda parity failed (mode={args.mode} weights={args.weights}): " + f"quality diff={qdiff:g} cos={qcos:g} int8_rate={qrate:g}; " + f"shape diff={sdiff:g} cos={scos:g} int8_rate={srate:g}" + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/check_mlx_model_limits.py b/tests/check_mlx_model_limits.py new file mode 100755 index 0000000..08eb7ba --- /dev/null +++ b/tests/check_mlx_model_limits.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Verify that MLX rejects mismatched model metadata before touching Metal.""" + +import argparse +import json +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--model-dir", required=True) + args = ap.parse_args() + + binary = Path(args.binary).resolve() + model_dir = Path(args.model_dir).resolve() + with (model_dir / "config.json").open() as f: + config = json.load(f) + + vocab_size = config.get("vocab_size") + if not isinstance(vocab_size, int) or vocab_size <= 1: + raise SystemExit("config.json has no usable vocab_size") + config["vocab_size"] = vocab_size - 1 + + with tempfile.TemporaryDirectory(prefix="ffwd-mlx-invalid-") as td: + tmp = Path(td) + (tmp / "config.json").write_text(json.dumps(config)) + (tmp / "vocab.json").symlink_to(model_dir / "vocab.json") + tensors = list(model_dir.glob("*.safetensors")) + if not tensors: + raise SystemExit("model directory has no safetensor files") + for tensor in tensors: + (tmp / tensor.name).symlink_to(tensor) + + proc = subprocess.run( + [str(binary), "-d", str(tmp), "--stream"], + input="", + text=True, + capture_output=True, + check=False, + ) + + expected = "mlx: bad shape for embed_tokens.weight at dim 0" + if proc.returncode == 0: + raise SystemExit("MLX unexpectedly accepted mismatched vocab_size") + if expected not in proc.stderr: + raise SystemExit(f"unexpected stderr:\n{proc.stderr}") + if "MLX error:" in proc.stderr: + raise SystemExit(f"MLX touched Metal before rejecting the model:\n{proc.stderr}") + print("ok: MLX rejected mismatched tensor shape before Metal initialization") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/check_mlx_quantized_parity.py b/tests/check_mlx_quantized_parity.py new file mode 100755 index 0000000..2cf0add --- /dev/null +++ b/tests/check_mlx_quantized_parity.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Compare normal MLX embeddings against opt-in quantized MLX weights.""" + +import argparse +import json +import math +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_BINARY = ROOT / "tools/cli/ffwd-cli" + + +TEXTS = [ + "query: what is the capital of France?", + "document: Paris is the capital of France.", + "document: Berlin is the capital of Germany.", + "document: Istanbul is a major city in Turkey.", + "The quick brown fox jumps over the lazy dog near the riverbank.", + ( + "Retrieval augmented generation combines embedding models with " + "external knowledge sources and ranks documents by semantic similarity." + ), +] + + +def run(binary: str, model_dir: str, texts: list[str], batch_size: int, + bits: int, group_size: int) -> list[list[float]]: + cmd = [binary, "-d", model_dir, "--stream", "-b", str(batch_size)] + if bits: + cmd.extend([ + "--gpu-quant-bits", str(bits), + "--gpu-quant-group-size", str(group_size), + ]) + proc = subprocess.run( + cmd, + input="\n".join(texts) + "\n", + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}" + ) + rows = [] + for line in proc.stdout.splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if "error" in row: + raise RuntimeError(f"stdin returned error: {row}") + rows.append(row["embedding"]) + if len(rows) != len(texts): + raise RuntimeError(f"expected {len(texts)} rows, got {len(rows)}") + return rows + + +def cosine(a: list[float], b: list[float]) -> float: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--binary", default=str(DEFAULT_BINARY)) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--batch-size", type=int, default=4) + ap.add_argument("--bits", type=int, default=8, choices=(8,)) + ap.add_argument("--group-size", type=int, default=64) + ap.add_argument("--max-diff", type=float, default=0.08) + ap.add_argument("--min-cosine", type=float, default=0.995) + args = ap.parse_args() + + ref = run(args.binary, args.model_dir, TEXTS, args.batch_size, 0, + args.group_size) + got = run(args.binary, args.model_dir, TEXTS, args.batch_size, args.bits, + args.group_size) + + worst_diff = 0.0 + worst_cos = 1.0 + for i, (a, b) in enumerate(zip(ref, got)): + max_diff = max(abs(x - y) for x, y in zip(a, b)) + cos = cosine(a, b) + worst_diff = max(worst_diff, max_diff) + worst_cos = min(worst_cos, cos) + print(f"[{i}] cosine={cos:.8f} max_abs_diff={max_diff:.8g}") + + print( + f"ok: bits={args.bits} group_size={args.group_size} " + f"worst_diff={worst_diff:.8g} worst_cos={worst_cos:.8f}" + ) + if worst_diff > args.max_diff or worst_cos < args.min_cosine: + raise SystemExit( + f"quantized parity failed: diff={worst_diff:g}, cos={worst_cos:g}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
