diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 09:56:10 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 09:56:10 +0200 |
| commit | e381024cca6fd1e0815c1b8dc72a09f493d57378 (patch) | |
| tree | f443323a6f6de51e9265948fabd6d731b3346cbd | |
| parent | 3787740e1bea4d071b5c15d77728c9c09d12119d (diff) | |
| download | ffwd-e381024cca6fd1e0815c1b8dc72a09f493d57378.tar.gz ffwd-e381024cca6fd1e0815c1b8dc72a09f493d57378.zip | |
build: fold verify_builds.sh into all.bash checks
| -rwxr-xr-x | all.bash | 115 | ||||
| -rwxr-xr-x | devtools/verify_builds.sh | 330 |
2 files changed, 114 insertions, 331 deletions
@@ -85,8 +85,117 @@ check_code_quality() { run ./devtools/check_code_quality.sh "$@" } +# The comprehensive local suite, folded in from the old verify_builds.sh so +# all.bash is the single entry point. It builds, runs the hermetic C tests and +# the math-kernel goldens and the hermetic tokenizer checks (all model-free), +# then the CPU model checks and - on macOS - the MLX model checks, and finishes +# with the compiler/analyzer/tidy gate. Model-dependent steps run only when +# --model-dir is given. CUDA validation stays a separate RunPod step. run_checks() { - run ./devtools/verify_builds.sh "$@" + local model_dir="" context_model_dir="" late_model_dir="" + local batch_size=4 context_runs=0 + local long=0 skip_build=0 skip_hermetic=0 skip_mlx=0 + local skip_code_quality=0 skip_tokenizer=0 skip_mlx_quantized=0 + local platform_skip_mlx=0 + + while (($#)); do + case "$1" in + --model-dir) model_dir="$2"; shift 2 ;; + --model-dir=*) model_dir="${1#*=}"; shift ;; + --context-model-dir) context_model_dir="$2"; shift 2 ;; + --context-model-dir=*) context_model_dir="${1#*=}"; shift ;; + --late-model-dir) late_model_dir="$2"; shift 2 ;; + --late-model-dir=*) late_model_dir="${1#*=}"; shift ;; + --batch-size) batch_size="$2"; shift 2 ;; + --batch-size=*) batch_size="${1#*=}"; shift ;; + --context-runs) context_runs="$2"; shift 2 ;; + --context-runs=*) context_runs="${1#*=}"; shift ;; + --long) long=1; shift ;; + --skip-build) skip_build=1; shift ;; + --skip-hermetic) skip_hermetic=1; shift ;; + --skip-mlx) skip_mlx=1; shift ;; + --skip-code-quality) skip_code_quality=1; shift ;; + --skip-tokenizer) skip_tokenizer=1; shift ;; + --skip-mlx-quantized) skip_mlx_quantized=1; shift ;; + -h|--help) + cat <<'USAGE' +Usage: ./all.bash checks [options] + --model-dir DIR model for the workspace/batch parity checks + --context-model-dir DIR contextual model (enables contextual parity) + --late-model-dir DIR late-interaction model (enables late checks) + --batch-size N batch size for the parity checks (default 4) + --context-runs N timed contextual batches per backend + --long also run the long CPU attention parity check + --skip-build skip make cpu / make gpu + --skip-hermetic skip the model-free make test suite + --skip-mlx stop after the CPU build and CPU checks + --skip-tokenizer skip the hermetic tokenizer parity checks + --skip-mlx-quantized skip the MLX Q8 parity check + --skip-code-quality skip the compiler/analyzer/tidy gate +USAGE + return 0 ;; + *) err "checks: unknown option: $1"; return 2 ;; + esac + done + + if [[ "$(uname -s)" != "Darwin" && $skip_mlx -eq 0 ]]; then + warn "MLX checks require macOS; skipping the MLX phase on $(uname -s)" + skip_mlx=1 + platform_skip_mlx=1 + fi + + # Any failing step aborts the suite with a non-zero exit, as the old script did. + _step() { "$@" || { err "checks: step failed -> $*"; exit 1; }; } + + local long_args=() + (( long )) && long_args=(--long) + + # Model-free: build, hermetic C tests, kernel goldens, hermetic tokenizers. + (( skip_hermetic )) || _step run make test + (( skip_build )) || _step run make cpu + _step check_kernel_golden + if (( ! skip_tokenizer )); then + _step check_tokenizer_parity + _step check_wordpiece_parity + fi + + # CPU model checks (need a model directory). + if [[ -n "$model_dir" ]]; then + _step check_workspace_api --model-dir "$model_dir" + _step check_batch_parity --model-dir "$model_dir" --backend cpu \ + --batch-size "$batch_size" "${long_args[@]}" + if [[ -n "$context_model_dir" ]]; then + _step check_contextual_batch_parity --model-dir "$context_model_dir" \ + --backend cpu --runs "$context_runs" + fi + if [[ -n "$late_model_dir" ]]; then + _step check_late_interaction --model-dir "$late_model_dir" --backend cpu + fi + else + warn "checks: no --model-dir; skipping CPU model checks (workspace, batch/contextual/late parity)" + fi + + # MLX phase (macOS only): rebuild as the GPU backend, rerun the model checks. + if (( ! skip_mlx )); then + (( skip_build )) || _step run make gpu + if [[ -n "$model_dir" ]]; then + _step check_batch_parity --model-dir "$model_dir" --backend mlx \ + --batch-size "$batch_size" + _step check_mlx_model_limits --model-dir "$model_dir" + (( skip_mlx_quantized )) || _step check_mlx_quantized_parity \ + --model-dir "$model_dir" --batch-size "$batch_size" + if [[ -n "$context_model_dir" ]]; then + _step check_contextual_batch_parity --model-dir "$context_model_dir" \ + --backend mlx --runs "$context_runs" + fi + if [[ -n "$late_model_dir" ]]; then + _step check_late_interaction --model-dir "$late_model_dir" --backend mlx + fi + fi + fi + + (( skip_code_quality )) || _step check_code_quality + info "checks: all green" } check_kernel_golden() { @@ -233,6 +342,10 @@ usage() { Usage: ./all.bash COMMAND [ARGS...] + Aggregate: checks full local suite: build + hermetic tests + CPU/MLX + model checks + code quality. Pass --model-dir DIR to + add the model checks; `checks --help` for options. + ┏━━[ checks ]━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ ┃ ┃ ┃ check-code-quality Run clang builds, analysis ┃ diff --git a/devtools/verify_builds.sh b/devtools/verify_builds.sh deleted file mode 100755 index 25347b5..0000000 --- a/devtools/verify_builds.sh +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)" - -DEFAULT_MODEL_GLOB='models--perplexity-ai--pplx-embed-v1-0.6b/snapshots/*' -DEFAULT_CONTEXT_MODEL_GLOB='models--perplexity-ai--pplx-embed-context-v1-0.6B/snapshots/*' - -model_dir="" -context_model_dir="" -binary="./ffwd-cli" -batch_size=4 -skip_build=0 -skip_mlx=0 -platform_skip_mlx=0 -skip_contextual=0 -context_runs=0 -late_model_dir="" -long=0 -skip_code_quality=0 -skip_hermetic=0 -skip_tokenizer=0 -skip_mlx_quantized=0 - -die() { - echo "error: $*" >&2 - exit 2 -} - -usage() { - cat <<EOF -Usage: $(basename "$0") [options] - -Options: - --model-dir DIR 0.6B model snapshot; defaults to the HF cache - --context-model-dir DIR context 0.6B snapshot; defaults to the HF cache - --binary PATH binary to test, default: ./ffwd-cli - --batch-size N batch size, default: 4 - --skip-build skip make cpu / make gpu - --skip-hermetic skip the model-free make test suite - --skip-mlx stop after the CPU build and CPU checks - --skip-contextual skip contextual parity checks - --skip-tokenizer skip Hugging Face tokenizer parity - --skip-mlx-quantized skip the MLX Q8 parity check - --context-runs N timed contextual batches per backend after parity - --late-model-dir DIR optional late-interaction 0.6B snapshot to verify - --long also run the long CPU attention parity check - --skip-code-quality skip the final compiler/analyzer/tidy CI gate - -h, --help show this help -EOF -} - -find_cached_snapshot() { - local glob="$1" - local label="$2" - local hub="$HOME/.cache/huggingface/hub" - local last="" - local p - - while IFS= read -r p; do - if [[ -d "$p" && -f "$p/config.json" ]]; then - last="$p" - fi - done < <({ compgen -G "$hub/$glob" || true; } | sort) - - if [[ -z "$last" ]]; then - echo "no cached $label snapshot found; pass the model dir" >&2 - exit 1 - fi - - printf '%s\n' "$last" -} - -find_default_model() { - find_cached_snapshot "$DEFAULT_MODEL_GLOB" "pplx-embed-v1-0.6b" -} - -find_default_context_model() { - find_cached_snapshot "$DEFAULT_CONTEXT_MODEL_GLOB" "pplx-embed-context-v1-0.6b" -} - -run() { - printf '+' - printf ' %q' "$@" - printf '\n' - (cd "$ROOT" && "$@") -} - -run_parity() { - local binary_arg="$1" - local model_dir_arg="$2" - local backend="$3" - local batch_size_arg="$4" - local use_long="$5" - - local cmd=( - "$ROOT/all.bash" check-batch-parity - --binary "$binary_arg" - --model-dir "$model_dir_arg" - --backend "$backend" - --batch-size "$batch_size_arg" - ) - - if (( use_long )); then - cmd+=(--long) - fi - - run "${cmd[@]}" -} - -run_contextual() { - local model_dir_arg="$1" - local backend="$2" - local runs="$3" - - run \ - "$ROOT/all.bash" check-context-batch-parity \ - --model-dir "$model_dir_arg" \ - --backend "$backend" \ - --runs "$runs" -} - -run_late() { - local model_dir_arg="$1" - local backend="$2" - - run \ - "$ROOT/all.bash" check-late-interaction \ - --model-dir "$model_dir_arg" \ - --backend "$backend" -} - -while (($#)); do - case "$1" in - --model-dir) - (($# >= 2)) || die "--model-dir requires a value" - model_dir="$2" - shift 2 - ;; - --model-dir=*) - model_dir="${1#*=}" - shift - ;; - - --context-model-dir) - (($# >= 2)) || die "--context-model-dir requires a value" - context_model_dir="$2" - shift 2 - ;; - --context-model-dir=*) - context_model_dir="${1#*=}" - shift - ;; - - --binary) - (($# >= 2)) || die "--binary requires a value" - binary="$2" - shift 2 - ;; - --binary=*) - binary="${1#*=}" - shift - ;; - - --batch-size) - (($# >= 2)) || die "--batch-size requires a value" - batch_size="$2" - shift 2 - ;; - --batch-size=*) - batch_size="${1#*=}" - shift - ;; - - --skip-build) - skip_build=1 - shift - ;; - --skip-hermetic) - skip_hermetic=1 - shift - ;; - --skip-mlx) - skip_mlx=1 - shift - ;; - --skip-contextual) - skip_contextual=1 - shift - ;; - --skip-tokenizer) - skip_tokenizer=1 - shift - ;; - --skip-mlx-quantized) - skip_mlx_quantized=1 - shift - ;; - - --context-runs) - (($# >= 2)) || die "--context-runs requires a value" - context_runs="$2" - shift 2 - ;; - --context-runs=*) - context_runs="${1#*=}" - shift - ;; - - --late-model-dir) - (($# >= 2)) || die "--late-model-dir requires a value" - late_model_dir="$2" - shift 2 - ;; - --late-model-dir=*) - late_model_dir="${1#*=}" - shift - ;; - - --long) - long=1 - shift - ;; - --skip-code-quality) - skip_code_quality=1 - shift - ;; - - -h|--help) - usage - exit 0 - ;; - - *) - die "unknown argument: $1" - ;; - esac -done - -if [[ "$(uname -s)" != "Darwin" && $skip_mlx -eq 0 ]]; then - echo "info: MLX checks require macOS; skipping on $(uname -s)" - skip_mlx=1 - platform_skip_mlx=1 -fi - -if [[ -z "$model_dir" ]]; then - model_dir="$(find_default_model)" -fi - -if (( skip_contextual )); then - context_model_dir="" -elif [[ -z "$context_model_dir" ]]; then - context_model_dir="$(find_default_context_model)" -fi - -if (( ! skip_hermetic )); then - run make test -fi - -if (( ! skip_build )); then - run make cpu -fi - -run "$ROOT/all.bash" check-kernel-golden --blas - -if (( ! skip_tokenizer )); then - run "$ROOT/all.bash" check-tokenizer-parity --model-dir "$model_dir" -fi - -run "$ROOT/all.bash" check-workspace-api --model-dir "$model_dir" - -run_parity "$binary" "$model_dir" cpu "$batch_size" 0 - -if (( long )); then - run_parity "$binary" "$model_dir" cpu "$batch_size" 1 -fi - -if [[ -n "$context_model_dir" ]]; then - run_contextual "$context_model_dir" cpu "$context_runs" -fi - -if [[ -n "$late_model_dir" ]]; then - run_late "$late_model_dir" cpu -fi - -if (( skip_mlx )); then - if (( platform_skip_mlx && ! skip_code_quality )); then - run "$ROOT/all.bash" check-code-quality - fi - if (( skip_build )); then - echo "ok: CPU checks passed; builds and MLX checks skipped" - else - echo "ok: CPU build verified; MLX build skipped" - fi - exit 0 -fi - -if (( ! skip_build )); then - run make gpu -fi - -run_parity "$binary" "$model_dir" mlx "$batch_size" 0 - -run \ - "$ROOT/all.bash" check-mlx-model-limits \ - --binary "$binary" \ - --model-dir "$model_dir" - -if (( ! skip_mlx_quantized )); then - run \ - "$ROOT/all.bash" check-mlx-quantized-parity \ - --binary "$binary" \ - --model-dir "$model_dir" \ - --batch-size "$batch_size" -fi - -if [[ -n "$context_model_dir" ]]; then - run_contextual "$context_model_dir" mlx "$context_runs" -fi - -if [[ -n "$late_model_dir" ]]; then - run_late "$late_model_dir" mlx -fi - -if (( ! skip_code_quality )); then - run "$ROOT/all.bash" check-code-quality -fi - -echo "verify ok" |
