aboutsummaryrefslogtreecommitdiff
path: root/devtools/verify_builds.sh
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-21 08:26:31 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 08:26:31 +0200
commit445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf (patch)
tree09c7e853df4dd6fece5e3758709aad7bd6ea9503 /devtools/verify_builds.sh
parenta5f67f2f828a7ac5c541ec54abf758597efcfbb6 (diff)
downloadffwd-445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf.tar.gz
ffwd-445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf.zip
build: rehome check scripts and modernize all.bash
Diffstat (limited to 'devtools/verify_builds.sh')
-rwxr-xr-xdevtools/verify_builds.sh330
1 files changed, 330 insertions, 0 deletions
diff --git a/devtools/verify_builds.sh b/devtools/verify_builds.sh
new file mode 100755
index 0000000..25347b5
--- /dev/null
+++ b/devtools/verify_builds.sh
@@ -0,0 +1,330 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
+
+DEFAULT_MODEL_GLOB='models--perplexity-ai--pplx-embed-v1-0.6b/snapshots/*'
+DEFAULT_CONTEXT_MODEL_GLOB='models--perplexity-ai--pplx-embed-context-v1-0.6B/snapshots/*'
+
+model_dir=""
+context_model_dir=""
+binary="./ffwd-cli"
+batch_size=4
+skip_build=0
+skip_mlx=0
+platform_skip_mlx=0
+skip_contextual=0
+context_runs=0
+late_model_dir=""
+long=0
+skip_code_quality=0
+skip_hermetic=0
+skip_tokenizer=0
+skip_mlx_quantized=0
+
+die() {
+ echo "error: $*" >&2
+ exit 2
+}
+
+usage() {
+ cat <<EOF
+Usage: $(basename "$0") [options]
+
+Options:
+ --model-dir DIR 0.6B model snapshot; defaults to the HF cache
+ --context-model-dir DIR context 0.6B snapshot; defaults to the HF cache
+ --binary PATH binary to test, default: ./ffwd-cli
+ --batch-size N batch size, default: 4
+ --skip-build skip make cpu / make gpu
+ --skip-hermetic skip the model-free make test suite
+ --skip-mlx stop after the CPU build and CPU checks
+ --skip-contextual skip contextual parity checks
+ --skip-tokenizer skip Hugging Face tokenizer parity
+ --skip-mlx-quantized skip the MLX Q8 parity check
+ --context-runs N timed contextual batches per backend after parity
+ --late-model-dir DIR optional late-interaction 0.6B snapshot to verify
+ --long also run the long CPU attention parity check
+ --skip-code-quality skip the final compiler/analyzer/tidy CI gate
+ -h, --help show this help
+EOF
+}
+
+find_cached_snapshot() {
+ local glob="$1"
+ local label="$2"
+ local hub="$HOME/.cache/huggingface/hub"
+ local last=""
+ local p
+
+ while IFS= read -r p; do
+ if [[ -d "$p" && -f "$p/config.json" ]]; then
+ last="$p"
+ fi
+ done < <({ compgen -G "$hub/$glob" || true; } | sort)
+
+ if [[ -z "$last" ]]; then
+ echo "no cached $label snapshot found; pass the model dir" >&2
+ exit 1
+ fi
+
+ printf '%s\n' "$last"
+}
+
+find_default_model() {
+ find_cached_snapshot "$DEFAULT_MODEL_GLOB" "pplx-embed-v1-0.6b"
+}
+
+find_default_context_model() {
+ find_cached_snapshot "$DEFAULT_CONTEXT_MODEL_GLOB" "pplx-embed-context-v1-0.6b"
+}
+
+run() {
+ printf '+'
+ printf ' %q' "$@"
+ printf '\n'
+ (cd "$ROOT" && "$@")
+}
+
+run_parity() {
+ local binary_arg="$1"
+ local model_dir_arg="$2"
+ local backend="$3"
+ local batch_size_arg="$4"
+ local use_long="$5"
+
+ local cmd=(
+ "$ROOT/all.bash" check-batch-parity
+ --binary "$binary_arg"
+ --model-dir "$model_dir_arg"
+ --backend "$backend"
+ --batch-size "$batch_size_arg"
+ )
+
+ if (( use_long )); then
+ cmd+=(--long)
+ fi
+
+ run "${cmd[@]}"
+}
+
+run_contextual() {
+ local model_dir_arg="$1"
+ local backend="$2"
+ local runs="$3"
+
+ run \
+ "$ROOT/all.bash" check-context-batch-parity \
+ --model-dir "$model_dir_arg" \
+ --backend "$backend" \
+ --runs "$runs"
+}
+
+run_late() {
+ local model_dir_arg="$1"
+ local backend="$2"
+
+ run \
+ "$ROOT/all.bash" check-late-interaction \
+ --model-dir "$model_dir_arg" \
+ --backend "$backend"
+}
+
+while (($#)); do
+ case "$1" in
+ --model-dir)
+ (($# >= 2)) || die "--model-dir requires a value"
+ model_dir="$2"
+ shift 2
+ ;;
+ --model-dir=*)
+ model_dir="${1#*=}"
+ shift
+ ;;
+
+ --context-model-dir)
+ (($# >= 2)) || die "--context-model-dir requires a value"
+ context_model_dir="$2"
+ shift 2
+ ;;
+ --context-model-dir=*)
+ context_model_dir="${1#*=}"
+ shift
+ ;;
+
+ --binary)
+ (($# >= 2)) || die "--binary requires a value"
+ binary="$2"
+ shift 2
+ ;;
+ --binary=*)
+ binary="${1#*=}"
+ shift
+ ;;
+
+ --batch-size)
+ (($# >= 2)) || die "--batch-size requires a value"
+ batch_size="$2"
+ shift 2
+ ;;
+ --batch-size=*)
+ batch_size="${1#*=}"
+ shift
+ ;;
+
+ --skip-build)
+ skip_build=1
+ shift
+ ;;
+ --skip-hermetic)
+ skip_hermetic=1
+ shift
+ ;;
+ --skip-mlx)
+ skip_mlx=1
+ shift
+ ;;
+ --skip-contextual)
+ skip_contextual=1
+ shift
+ ;;
+ --skip-tokenizer)
+ skip_tokenizer=1
+ shift
+ ;;
+ --skip-mlx-quantized)
+ skip_mlx_quantized=1
+ shift
+ ;;
+
+ --context-runs)
+ (($# >= 2)) || die "--context-runs requires a value"
+ context_runs="$2"
+ shift 2
+ ;;
+ --context-runs=*)
+ context_runs="${1#*=}"
+ shift
+ ;;
+
+ --late-model-dir)
+ (($# >= 2)) || die "--late-model-dir requires a value"
+ late_model_dir="$2"
+ shift 2
+ ;;
+ --late-model-dir=*)
+ late_model_dir="${1#*=}"
+ shift
+ ;;
+
+ --long)
+ long=1
+ shift
+ ;;
+ --skip-code-quality)
+ skip_code_quality=1
+ shift
+ ;;
+
+ -h|--help)
+ usage
+ exit 0
+ ;;
+
+ *)
+ die "unknown argument: $1"
+ ;;
+ esac
+done
+
+if [[ "$(uname -s)" != "Darwin" && $skip_mlx -eq 0 ]]; then
+ echo "info: MLX checks require macOS; skipping on $(uname -s)"
+ skip_mlx=1
+ platform_skip_mlx=1
+fi
+
+if [[ -z "$model_dir" ]]; then
+ model_dir="$(find_default_model)"
+fi
+
+if (( skip_contextual )); then
+ context_model_dir=""
+elif [[ -z "$context_model_dir" ]]; then
+ context_model_dir="$(find_default_context_model)"
+fi
+
+if (( ! skip_hermetic )); then
+ run make test
+fi
+
+if (( ! skip_build )); then
+ run make cpu
+fi
+
+run "$ROOT/all.bash" check-kernel-golden --blas
+
+if (( ! skip_tokenizer )); then
+ run "$ROOT/all.bash" check-tokenizer-parity --model-dir "$model_dir"
+fi
+
+run "$ROOT/all.bash" check-workspace-api --model-dir "$model_dir"
+
+run_parity "$binary" "$model_dir" cpu "$batch_size" 0
+
+if (( long )); then
+ run_parity "$binary" "$model_dir" cpu "$batch_size" 1
+fi
+
+if [[ -n "$context_model_dir" ]]; then
+ run_contextual "$context_model_dir" cpu "$context_runs"
+fi
+
+if [[ -n "$late_model_dir" ]]; then
+ run_late "$late_model_dir" cpu
+fi
+
+if (( skip_mlx )); then
+ if (( platform_skip_mlx && ! skip_code_quality )); then
+ run "$ROOT/all.bash" check-code-quality
+ fi
+ if (( skip_build )); then
+ echo "ok: CPU checks passed; builds and MLX checks skipped"
+ else
+ echo "ok: CPU build verified; MLX build skipped"
+ fi
+ exit 0
+fi
+
+if (( ! skip_build )); then
+ run make gpu
+fi
+
+run_parity "$binary" "$model_dir" mlx "$batch_size" 0
+
+run \
+ "$ROOT/all.bash" check-mlx-model-limits \
+ --binary "$binary" \
+ --model-dir "$model_dir"
+
+if (( ! skip_mlx_quantized )); then
+ run \
+ "$ROOT/all.bash" check-mlx-quantized-parity \
+ --binary "$binary" \
+ --model-dir "$model_dir" \
+ --batch-size "$batch_size"
+fi
+
+if [[ -n "$context_model_dir" ]]; then
+ run_contextual "$context_model_dir" mlx "$context_runs"
+fi
+
+if [[ -n "$late_model_dir" ]]; then
+ run_late "$late_model_dir" mlx
+fi
+
+if (( ! skip_code_quality )); then
+ run "$ROOT/all.bash" check-code-quality
+fi
+
+echo "verify ok"