aboutsummaryrefslogtreecommitdiff
path: root/devtools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-21 08:26:31 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 08:26:31 +0200
commit445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf (patch)
tree09c7e853df4dd6fece5e3758709aad7bd6ea9503 /devtools
parenta5f67f2f828a7ac5c541ec54abf758597efcfbb6 (diff)
downloadffwd-445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf.tar.gz
ffwd-445e0ff83bcf13f04d0a5b6f89f89fe6ada8bebf.zip
build: rehome check scripts and modernize all.bash
Diffstat (limited to '')
-rwxr-xr-xdevtools/check_code_quality.sh181
-rwxr-xr-xdevtools/container-build-test.sh87
-rwxr-xr-xdevtools/verify_builds.sh330
3 files changed, 598 insertions, 0 deletions
diff --git a/devtools/check_code_quality.sh b/devtools/check_code_quality.sh
new file mode 100755
index 0000000..305ba50
--- /dev/null
+++ b/devtools/check_code_quality.sh
@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$ROOT"
+
+# Lint and analyze every C source under libffwd/, tests/, and bench/. Each
+# platform keeps its own GPU backend and skips the other's, which it does not
+# build (macOS builds MLX, Linux builds CUDA).
+SYS="$(uname -s)"
+SRC_SOURCES=()
+for f in libffwd/*.c; do
+ [[ "$SYS" == Darwin && "$f" == libffwd/backend_cuda.c ]] && continue
+ [[ "$SYS" != Darwin && ( "$f" == libffwd/mlx.c || "$f" == libffwd/backend_mlx.c ) ]] && continue
+ SRC_SOURCES+=("$f")
+done
+
+TEST_C_SOURCES=(tests/*.c)
+BENCH_C_SOURCES=(bench/*.c)
+
+# CUDA backend sources are C++17 and only built on Linux. Parse them with the
+# clang CUDA front end in host-only mode so no GPU or device toolchain is
+# required; macOS has no CUDA toolkit, so skip them there.
+CU_SOURCES=()
+if [[ "$SYS" != Darwin ]]; then
+ for f in libffwd/*.cu; do
+ [[ -e "$f" ]] && CU_SOURCES+=("$f")
+ done
+fi
+
+ANALYZE_SOURCES=("${SRC_SOURCES[@]}")
+TIDY_SOURCES=("${SRC_SOURCES[@]}" "${TEST_C_SOURCES[@]}" "${BENCH_C_SOURCES[@]}")
+
+ANALYZE_COMMON=(
+ clang --analyze -Xanalyzer -analyzer-output=text
+ -Wall -Wextra
+ -I. -Ilibffwd -Ideps/ae
+)
+
+PLATFORM_CFLAGS=()
+if [[ "$(uname -s)" == "Darwin" ]]; then
+ PLATFORM_CFLAGS=(
+ -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX
+ -I/opt/homebrew/opt/mlx-c/include
+ -I/opt/homebrew/include
+ -I/usr/local/include
+ )
+else
+ PLATFORM_CFLAGS=(
+ -DUSE_BLAS -DUSE_OPENBLAS
+ -I/usr/include/openblas
+ )
+fi
+
+# Flags for the CUDA backend front end. CUDA_HOME matches the Makefile default.
+# --cuda-host-only parses host code only so device codegen and libdevice are not
+# needed; the macros mirror the nvcc command in the Makefile.
+CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}"
+CUDA_CFLAGS=(
+ -x cuda --cuda-host-only "--cuda-path=$CUDA_HOME"
+ -std=c++17
+ -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU
+ -I"$CUDA_HOME/include"
+ -I/usr/include/openblas
+)
+
+print_cmd() {
+ printf '+'
+ printf ' %q' "$@"
+ printf '\n'
+}
+
+run() {
+ print_cmd "$@"
+ "$@"
+}
+
+run_analyze() {
+ local output
+ local status
+
+ print_cmd "$@"
+
+ set +e
+ output="$("$@" 2>&1)"
+ status=$?
+ set -e
+
+ if [[ -n "$output" ]]; then
+ printf '%s\n' "$output"
+ fi
+
+ if (( status != 0 )); then
+ return "$status"
+ fi
+
+ if grep -q 'warning:' <<<"$output"; then
+ echo "clang analyzer warnings found" >&2
+ return 1
+ fi
+}
+
+have() {
+ command -v "$1" >/dev/null 2>&1
+}
+
+find_clang_tidy() {
+ local brew_path="/opt/homebrew/opt/llvm@21/bin/clang-tidy"
+
+ if command -v clang-tidy >/dev/null 2>&1; then
+ command -v clang-tidy
+ return 0
+ fi
+
+ if [[ -x "$brew_path" ]]; then
+ printf '%s\n' "$brew_path"
+ return 0
+ fi
+
+ return 1
+}
+
+main() {
+ local clang_tidy
+
+ if ! have clang; then
+ echo "clang not found" >&2
+ return 1
+ fi
+
+ run git --no-pager diff --check
+ run make cpu CC=clang
+
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ "${ANALYZE_SOURCES[@]}"
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ -I. -Ilibffwd -Itests "${TEST_C_SOURCES[@]}"
+ run_analyze "${ANALYZE_COMMON[@]}" "${PLATFORM_CFLAGS[@]}" \
+ -I. -Ilibffwd -Ibench "${BENCH_C_SOURCES[@]}"
+
+ if (( ${#CU_SOURCES[@]} > 0 )); then
+ run_analyze "${ANALYZE_COMMON[@]}" "${CUDA_CFLAGS[@]}" \
+ "${CU_SOURCES[@]}"
+ fi
+
+ if clang_tidy="$(find_clang_tidy)"; then
+ run \
+ "$clang_tidy" \
+ --config-file=.clang-tidy \
+ "${TIDY_SOURCES[@]}" \
+ -- \
+ -std=c17 \
+ "${PLATFORM_CFLAGS[@]}" \
+ -I. \
+ -Ilibffwd \
+ -Itests \
+ -Ibench \
+ -Ideps/ae
+
+ if (( ${#CU_SOURCES[@]} > 0 )); then
+ run \
+ "$clang_tidy" \
+ --config-file=.clang-tidy \
+ "${CU_SOURCES[@]}" \
+ -- \
+ "${CUDA_CFLAGS[@]}" \
+ -I. \
+ -Ilibffwd \
+ -Ideps/ae
+ fi
+ else
+ echo "clang-tidy not found; skipping"
+ fi
+
+ if [[ "$(uname -s)" == "Darwin" ]]; then
+ run make gpu CC=clang
+ fi
+}
+
+main "$@"
diff --git a/devtools/container-build-test.sh b/devtools/container-build-test.sh
new file mode 100755
index 0000000..4dcc627
--- /dev/null
+++ b/devtools/container-build-test.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Build, test and bench the OpenBLAS backend in a throwaway Ubuntu container to
+# catch Linux portability breakage a macOS/Accelerate build hides. Needs Apple's
+# `container` CLI running. Run from anywhere in the repo: it finds the repo root
+# via git, bind-mounts it at /work, installs the Runpod toolchain, re-execs
+# itself inside with --in-container, and runs make cpu/test/bench/bench-server-
+# utils/debug, then `make clean` (skip with KEEP=1). Override the image with
+# IMAGE=ubuntu:22.04. Exits non-zero at the first failing step.
+set -euo pipefail
+
+IMAGE="${IMAGE:-ubuntu:24.04}"
+
+# --- inside the container: bootstrap toolchain, then build/test ---------------
+run_in_container() {
+ export DEBIAN_FRONTEND=noninteractive
+
+ echo "==================== apt bootstrap ===================="
+ apt-get update -qq
+ apt-get install -y --no-install-recommends \
+ build-essential ca-certificates libcjson-dev libopenblas-dev \
+ pkg-config git >/dev/null
+
+ echo "==================== system / toolchain ===================="
+ uname -a
+ cc --version | sed -n '1p'
+ echo "libcjson $(pkg-config --modversion libcjson 2>/dev/null || echo MISSING) :: $(pkg-config --cflags --libs libcjson 2>/dev/null)"
+ echo "openblas $(pkg-config --modversion openblas 2>/dev/null || echo MISSING) :: $(pkg-config --cflags --libs openblas 2>/dev/null)"
+
+ cd /work
+ # /work is a bind mount; git refuses to operate on it without this opt-in.
+ git config --global --add safe.directory /work 2>/dev/null || true
+ make clean >/dev/null 2>&1 || true
+
+ echo "==================== make cpu ===================="
+ make cpu
+ ls -la ffwd-cli ffwd-server
+ ./ffwd-cli --version || true
+
+ echo "==================== make test ===================="
+ make test
+
+ echo "==================== make bench ===================="
+ make bench
+
+ echo "==================== make bench-server-utils ===================="
+ make bench-server-utils
+
+ echo "==================== make debug (sanitizers) ===================="
+ make debug
+
+ if [ -n "${KEEP:-}" ]; then
+ echo "==================== KEEP set: leaving artifacts in tree ===================="
+ else
+ echo "==================== make clean ===================="
+ make clean
+ residual=$(ls -d build ffwd-cli ffwd-server 2>/dev/null | wc -l | tr -d ' ')
+ echo "residual build artifacts: $residual (want 0)"
+ fi
+
+ echo "==================== ALL STEPS COMPLETED ===================="
+}
+
+# --- host side: launch the container and re-exec this script inside it --------
+launch_on_host() {
+ command -v container >/dev/null 2>&1 || {
+ echo "error: Apple 'container' CLI not found on PATH" >&2; exit 1; }
+
+ local repo_root script_rel
+ repo_root="$(git rev-parse --show-toplevel)"
+ # Path of this script relative to the repo root, so it resolves under /work.
+ script_rel="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")"
+ script_rel="${script_rel#"$repo_root"/}"
+
+ echo "repo: $repo_root"
+ echo "image: $IMAGE"
+ exec container run --rm \
+ --env "KEEP=${KEEP:-}" \
+ --mount "source=$repo_root,target=/work" \
+ "$IMAGE" \
+ bash "/work/$script_rel" --in-container
+}
+
+if [ "${1:-}" = "--in-container" ]; then
+ run_in_container
+else
+ launch_on_host
+fi
diff --git a/devtools/verify_builds.sh b/devtools/verify_builds.sh
new file mode 100755
index 0000000..25347b5
--- /dev/null
+++ b/devtools/verify_builds.sh
@@ -0,0 +1,330 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)"
+
+DEFAULT_MODEL_GLOB='models--perplexity-ai--pplx-embed-v1-0.6b/snapshots/*'
+DEFAULT_CONTEXT_MODEL_GLOB='models--perplexity-ai--pplx-embed-context-v1-0.6B/snapshots/*'
+
+model_dir=""
+context_model_dir=""
+binary="./ffwd-cli"
+batch_size=4
+skip_build=0
+skip_mlx=0
+platform_skip_mlx=0
+skip_contextual=0
+context_runs=0
+late_model_dir=""
+long=0
+skip_code_quality=0
+skip_hermetic=0
+skip_tokenizer=0
+skip_mlx_quantized=0
+
+die() {
+ echo "error: $*" >&2
+ exit 2
+}
+
+usage() {
+ cat <<EOF
+Usage: $(basename "$0") [options]
+
+Options:
+ --model-dir DIR 0.6B model snapshot; defaults to the HF cache
+ --context-model-dir DIR context 0.6B snapshot; defaults to the HF cache
+ --binary PATH binary to test, default: ./ffwd-cli
+ --batch-size N batch size, default: 4
+ --skip-build skip make cpu / make gpu
+ --skip-hermetic skip the model-free make test suite
+ --skip-mlx stop after the CPU build and CPU checks
+ --skip-contextual skip contextual parity checks
+ --skip-tokenizer skip Hugging Face tokenizer parity
+ --skip-mlx-quantized skip the MLX Q8 parity check
+ --context-runs N timed contextual batches per backend after parity
+ --late-model-dir DIR optional late-interaction 0.6B snapshot to verify
+ --long also run the long CPU attention parity check
+ --skip-code-quality skip the final compiler/analyzer/tidy CI gate
+ -h, --help show this help
+EOF
+}
+
+find_cached_snapshot() {
+ local glob="$1"
+ local label="$2"
+ local hub="$HOME/.cache/huggingface/hub"
+ local last=""
+ local p
+
+ while IFS= read -r p; do
+ if [[ -d "$p" && -f "$p/config.json" ]]; then
+ last="$p"
+ fi
+ done < <({ compgen -G "$hub/$glob" || true; } | sort)
+
+ if [[ -z "$last" ]]; then
+ echo "no cached $label snapshot found; pass the model dir" >&2
+ exit 1
+ fi
+
+ printf '%s\n' "$last"
+}
+
+find_default_model() {
+ find_cached_snapshot "$DEFAULT_MODEL_GLOB" "pplx-embed-v1-0.6b"
+}
+
+find_default_context_model() {
+ find_cached_snapshot "$DEFAULT_CONTEXT_MODEL_GLOB" "pplx-embed-context-v1-0.6b"
+}
+
+run() {
+ printf '+'
+ printf ' %q' "$@"
+ printf '\n'
+ (cd "$ROOT" && "$@")
+}
+
+run_parity() {
+ local binary_arg="$1"
+ local model_dir_arg="$2"
+ local backend="$3"
+ local batch_size_arg="$4"
+ local use_long="$5"
+
+ local cmd=(
+ "$ROOT/all.bash" check-batch-parity
+ --binary "$binary_arg"
+ --model-dir "$model_dir_arg"
+ --backend "$backend"
+ --batch-size "$batch_size_arg"
+ )
+
+ if (( use_long )); then
+ cmd+=(--long)
+ fi
+
+ run "${cmd[@]}"
+}
+
+run_contextual() {
+ local model_dir_arg="$1"
+ local backend="$2"
+ local runs="$3"
+
+ run \
+ "$ROOT/all.bash" check-context-batch-parity \
+ --model-dir "$model_dir_arg" \
+ --backend "$backend" \
+ --runs "$runs"
+}
+
+run_late() {
+ local model_dir_arg="$1"
+ local backend="$2"
+
+ run \
+ "$ROOT/all.bash" check-late-interaction \
+ --model-dir "$model_dir_arg" \
+ --backend "$backend"
+}
+
+while (($#)); do
+ case "$1" in
+ --model-dir)
+ (($# >= 2)) || die "--model-dir requires a value"
+ model_dir="$2"
+ shift 2
+ ;;
+ --model-dir=*)
+ model_dir="${1#*=}"
+ shift
+ ;;
+
+ --context-model-dir)
+ (($# >= 2)) || die "--context-model-dir requires a value"
+ context_model_dir="$2"
+ shift 2
+ ;;
+ --context-model-dir=*)
+ context_model_dir="${1#*=}"
+ shift
+ ;;
+
+ --binary)
+ (($# >= 2)) || die "--binary requires a value"
+ binary="$2"
+ shift 2
+ ;;
+ --binary=*)
+ binary="${1#*=}"
+ shift
+ ;;
+
+ --batch-size)
+ (($# >= 2)) || die "--batch-size requires a value"
+ batch_size="$2"
+ shift 2
+ ;;
+ --batch-size=*)
+ batch_size="${1#*=}"
+ shift
+ ;;
+
+ --skip-build)
+ skip_build=1
+ shift
+ ;;
+ --skip-hermetic)
+ skip_hermetic=1
+ shift
+ ;;
+ --skip-mlx)
+ skip_mlx=1
+ shift
+ ;;
+ --skip-contextual)
+ skip_contextual=1
+ shift
+ ;;
+ --skip-tokenizer)
+ skip_tokenizer=1
+ shift
+ ;;
+ --skip-mlx-quantized)
+ skip_mlx_quantized=1
+ shift
+ ;;
+
+ --context-runs)
+ (($# >= 2)) || die "--context-runs requires a value"
+ context_runs="$2"
+ shift 2
+ ;;
+ --context-runs=*)
+ context_runs="${1#*=}"
+ shift
+ ;;
+
+ --late-model-dir)
+ (($# >= 2)) || die "--late-model-dir requires a value"
+ late_model_dir="$2"
+ shift 2
+ ;;
+ --late-model-dir=*)
+ late_model_dir="${1#*=}"
+ shift
+ ;;
+
+ --long)
+ long=1
+ shift
+ ;;
+ --skip-code-quality)
+ skip_code_quality=1
+ shift
+ ;;
+
+ -h|--help)
+ usage
+ exit 0
+ ;;
+
+ *)
+ die "unknown argument: $1"
+ ;;
+ esac
+done
+
+if [[ "$(uname -s)" != "Darwin" && $skip_mlx -eq 0 ]]; then
+ echo "info: MLX checks require macOS; skipping on $(uname -s)"
+ skip_mlx=1
+ platform_skip_mlx=1
+fi
+
+if [[ -z "$model_dir" ]]; then
+ model_dir="$(find_default_model)"
+fi
+
+if (( skip_contextual )); then
+ context_model_dir=""
+elif [[ -z "$context_model_dir" ]]; then
+ context_model_dir="$(find_default_context_model)"
+fi
+
+if (( ! skip_hermetic )); then
+ run make test
+fi
+
+if (( ! skip_build )); then
+ run make cpu
+fi
+
+run "$ROOT/all.bash" check-kernel-golden --blas
+
+if (( ! skip_tokenizer )); then
+ run "$ROOT/all.bash" check-tokenizer-parity --model-dir "$model_dir"
+fi
+
+run "$ROOT/all.bash" check-workspace-api --model-dir "$model_dir"
+
+run_parity "$binary" "$model_dir" cpu "$batch_size" 0
+
+if (( long )); then
+ run_parity "$binary" "$model_dir" cpu "$batch_size" 1
+fi
+
+if [[ -n "$context_model_dir" ]]; then
+ run_contextual "$context_model_dir" cpu "$context_runs"
+fi
+
+if [[ -n "$late_model_dir" ]]; then
+ run_late "$late_model_dir" cpu
+fi
+
+if (( skip_mlx )); then
+ if (( platform_skip_mlx && ! skip_code_quality )); then
+ run "$ROOT/all.bash" check-code-quality
+ fi
+ if (( skip_build )); then
+ echo "ok: CPU checks passed; builds and MLX checks skipped"
+ else
+ echo "ok: CPU build verified; MLX build skipped"
+ fi
+ exit 0
+fi
+
+if (( ! skip_build )); then
+ run make gpu
+fi
+
+run_parity "$binary" "$model_dir" mlx "$batch_size" 0
+
+run \
+ "$ROOT/all.bash" check-mlx-model-limits \
+ --binary "$binary" \
+ --model-dir "$model_dir"
+
+if (( ! skip_mlx_quantized )); then
+ run \
+ "$ROOT/all.bash" check-mlx-quantized-parity \
+ --binary "$binary" \
+ --model-dir "$model_dir" \
+ --batch-size "$batch_size"
+fi
+
+if [[ -n "$context_model_dir" ]]; then
+ run_contextual "$context_model_dir" mlx "$context_runs"
+fi
+
+if [[ -n "$late_model_dir" ]]; then
+ run_late "$late_model_dir" mlx
+fi
+
+if (( ! skip_code_quality )); then
+ run "$ROOT/all.bash" check-code-quality
+fi
+
+echo "verify ok"