aboutsummaryrefslogtreecommitdiff
path: root/tests/Makefile
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:36:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commit6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch)
tree1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/Makefile
parentdbdf714bb57578431924e5d069089bdfd9f4c604 (diff)
downloadffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz
ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/Makefile')
-rw-r--r--tests/Makefile160
1 files changed, 160 insertions, 0 deletions
diff --git a/tests/Makefile b/tests/Makefile
new file mode 100644
index 0000000..92540c3
--- /dev/null
+++ b/tests/Makefile
@@ -0,0 +1,160 @@
+ROOT := ..
+include ../build.mk
+
+CFLAGS += -std=c11
+
+BLAS_LIBS := $(HOST_BLAS_LDFLAGS)
+
+# Every test links the blas libffwd archive (built into the per-config build
+# tree by the libffwd-blas phony) and includes its headers; only the fixture
+# headers are local to this directory.
+LIBFFWD_DIR := ../libffwd
+LIBFFWD_A := $(ROOT)/build/$(MODE)/blas/libffwd.a
+
+# Tokenizer and safetensors tests pull self-contained objects (no thread/BLAS
+# link flags); kernel and model tests pull the math kernels and forward pass.
+TOK_TESTS := test_bpe test_wordpiece test_sentencepiece
+ST_TESTS := test_safetensors
+KERNEL_TESTS := test_kernels
+FFWD_TESTS := test_cls test_late test_workspace test_bf16_model test_qwen2 \
+ test_qwen3 test_xlm_roberta
+# test_kernels_generic recompiles the kernels with no USE_BLAS to exercise the
+# scalar/SIMD paths (no cblas); test_kernels covers the blas-built kernels.
+KERNEL_SRCS := $(addprefix $(LIBFFWD_DIR)/,act.c gemm.c norm.c rope.c attn.c threadpool.c generic.c avx.c neon.c)
+TESTS := $(TOK_TESTS) $(ST_TESTS) $(KERNEL_TESTS) test_kernels_generic $(FFWD_TESTS)
+
+GEN_FIXTURE_BIN := gen_fixture
+
+PARITY_TOK_DRIVERS := tokenizer_dump wordpiece_dump
+PARITY_FFWD_DRIVERS := bert_dump contextual_batch
+INTEGRATION_DRIVERS := late_check
+FFWD_DRIVERS := $(PARITY_FFWD_DRIVERS) $(INTEGRATION_DRIVERS)
+PARITY_DRIVERS := $(PARITY_TOK_DRIVERS) $(FFWD_DRIVERS)
+
+.PHONY: all test coverage clean libffwd-blas \
+ parity-tokenizer-driver parity-wordpiece-driver parity-bert-driver \
+ parity-context-driver parity-context-driver-mlx late-check-driver \
+ late-check-driver-mlx
+
+all: $(TESTS) $(GEN_FIXTURE_BIN)
+
+test: $(TESTS)
+ @rc=0; for t in $(TESTS); do ./$$t || rc=1; done; \
+ if [ $$rc -eq 0 ]; then echo "tests ok"; else echo "tests not ok"; fi; \
+ exit $$rc
+
+# The archive is backend-specific, so a phony (not a file rule) drives the
+# sub-make; tests list it as an order prerequisite and link $(LIBFFWD_A).
+libffwd-blas:
+ $(MAKE) -C $(LIBFFWD_DIR) BACKEND=blas
+
+parity-tokenizer-driver: tokenizer_dump
+parity-wordpiece-driver: wordpiece_dump
+parity-bert-driver: bert_dump
+parity-context-driver: contextual_batch
+parity-context-driver-mlx: contextual_batch_mlx
+late-check-driver: late_check
+late-check-driver-mlx: late_check_mlx
+
+$(foreach t,$(TOK_TESTS),\
+ $(eval $(call c_program_deps,$(t),$(t).c,libffwd-blas,$(LIBFFWD_A),-I$(LIBFFWD_DIR),-lm)))
+
+$(foreach t,$(ST_TESTS),\
+ $(eval $(call c_program_deps,$(t),$(t).c,libffwd-blas,$(LIBFFWD_A),-I$(LIBFFWD_DIR),)))
+
+$(foreach t,$(KERNEL_TESTS),\
+ $(eval $(call c_program_deps,$(t),$(t).c,libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)))
+
+# Generic (no USE_BLAS) kernel build: compile the kernel sources directly so the
+# scalar/SIMD paths are linked instead of cblas; no Accelerate/OpenBLAS needed.
+$(eval $(call c_program_deps,test_kernels_generic,test_kernels.c $(KERNEL_SRCS),,,-pthread -I$(LIBFFWD_DIR),-lm))
+
+$(foreach t,$(FFWD_TESTS),\
+ $(eval $(call c_program_deps,$(t),$(t).c,libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)))
+
+$(foreach t,$(PARITY_TOK_DRIVERS),\
+ $(eval $(call c_program_deps,$(t),$(t).c,libffwd-blas,$(LIBFFWD_A),-I$(LIBFFWD_DIR),-lm)))
+
+$(foreach t,$(FFWD_DRIVERS),\
+ $(eval $(call c_program_deps,$(t),$(t).c,libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)))
+
+# The MLX driver variants reuse the BLAS engine archive for the CPU reference
+# paths (e.g. ffwd_late_maxsim) and add mlx.c for the device API. The archive
+# calls cblas, so the driver links Accelerate/OpenBLAS itself ($(HOST_BLAS_LDFLAGS))
+# on top of the Metal/MLX libraries. macOS only, built on demand.
+MLX_DRIVER_CFLAGS := -pthread -DCHECK_MLX -I$(MLXC_PREFIX)/include -I$(LIBFFWD_DIR) $(HOST_BLAS_CFLAGS)
+MLX_DRIVER_LDLIBS := $(MLX_BACKEND_LDFLAGS) $(HOST_BLAS_LDFLAGS) -lm
+
+$(eval $(call c_program_deps,contextual_batch_mlx,contextual_batch.c $(LIBFFWD_DIR)/mlx.c,libffwd-blas,$(LIBFFWD_A),$(MLX_DRIVER_CFLAGS),$(MLX_DRIVER_LDLIBS)))
+
+$(eval $(call c_program_deps,late_check_mlx,late_check.c $(LIBFFWD_DIR)/mlx.c,libffwd-blas,$(LIBFFWD_A),$(MLX_DRIVER_CFLAGS),$(MLX_DRIVER_LDLIBS)))
+
+$(eval $(call c_program_deps,$(GEN_FIXTURE_BIN),gen_fixture.c,model_fixture.h tiny_model.h tok_fixture.h,,-I.,-lm))
+
+# Line coverage (needs clang plus llvm-cov / llvm-profdata; xcrun on macOS).
+# The compile, run, and report steps all run from the repo root so the recorded
+# source paths stay clean (libffwd/x.c, tests/x.c) with no leading '..'; that
+# is what keeps the generated HTML's css/js links resolving correctly. Each test
+# is compiled at -O0 with the engine sources built in and instrumented, run to
+# emit a .profraw, then merged into a text summary and an HTML report under
+# tests/coverage/html. Only library sources are reported.
+COV_ROOT := ..
+COV_DIR := coverage
+COV_OUT := tests/$(COV_DIR)
+COV_FLAGS := -fprofile-instr-generate -fcoverage-mapping -O0
+COV_CFLAGS := -Wall -Wextra -Ilibffwd -Itests -pthread $(HOST_BLAS_CFLAGS) $(COV_FLAGS)
+COV_LDLIBS := $(BLAS_LIBS) -lm
+# Report library sources only: drop the test harnesses (test_*), the fixture
+# headers, and system headers.
+COV_IGNORE := -ignore-filename-regex='test_|_fixture|tiny_model|/opt/|/usr/'
+# The blas engine, root-relative: every libffwd source except the GPU backends.
+ENGINE_COV_SRCS := $(filter-out libffwd/mlx.c libffwd/backend_mlx.c libffwd/backend_cuda.c,$(patsubst $(LIBFFWD_DIR)/%,libffwd/%,$(wildcard $(LIBFFWD_DIR)/*.c)))
+COV_BINS := $(addprefix $(COV_OUT)/,$(filter-out test_kernels_generic,$(TESTS)))
+
+ifeq ($(OS),Darwin)
+LLVM_PROFDATA := xcrun llvm-profdata
+LLVM_COV := xcrun llvm-cov
+else
+LLVM_PROFDATA := llvm-profdata
+LLVM_COV := llvm-cov
+endif
+
+# Compile (from the repo root) one instrumented test with the engine built in,
+# then run it. cov_min links a minimal source subset $(2) for a test that
+# supplies its own globals (the full engine would double-define them).
+define cov_one
+cd $(COV_ROOT) && $(CC) $(COV_CFLAGS) -o $(COV_OUT)/$(1) tests/$(1).c $(ENGINE_COV_SRCS) $(COV_LDLIBS) && \
+LLVM_PROFILE_FILE=$(COV_OUT)/$(1).profraw $(COV_OUT)/$(1)
+endef
+
+define cov_min
+cd $(COV_ROOT) && $(CC) $(COV_CFLAGS) -o $(COV_OUT)/$(1) tests/$(1).c $(2) $(COV_LDLIBS) && \
+LLVM_PROFILE_FILE=$(COV_OUT)/$(1).profraw $(COV_OUT)/$(1)
+endef
+
+coverage:
+ $(RM) -r $(COV_DIR)
+ @mkdir -p $(COV_DIR)
+ $(call cov_min,test_bpe,libffwd/bpe.c)
+ $(call cov_one,test_wordpiece)
+ $(call cov_one,test_sentencepiece)
+ $(call cov_one,test_safetensors)
+ $(call cov_one,test_kernels)
+ $(call cov_one,test_cls)
+ $(call cov_one,test_late)
+ $(call cov_one,test_workspace)
+ $(call cov_one,test_bf16_model)
+ $(call cov_one,test_qwen2)
+ $(call cov_one,test_qwen3)
+ $(call cov_one,test_xlm_roberta)
+ cd $(COV_ROOT) && $(LLVM_PROFDATA) merge -sparse $(COV_OUT)/*.profraw -o $(COV_OUT)/tests.profdata
+ cd $(COV_ROOT) && $(LLVM_COV) report -instr-profile=$(COV_OUT)/tests.profdata $(COV_IGNORE) \
+ $(addprefix -object ,$(COV_BINS))
+ cd $(COV_ROOT) && $(LLVM_COV) show -format=html -output-dir=$(COV_OUT)/html \
+ -instr-profile=$(COV_OUT)/tests.profdata -show-line-counts-or-regions \
+ $(COV_IGNORE) $(addprefix -object ,$(COV_BINS))
+ @echo "HTML report: $(COV_OUT)/html/index.html"
+
+clean:
+ $(RM) -r $(TESTS) $(PARITY_DRIVERS) contextual_batch_mlx late_check_mlx \
+ *.dSYM $(GEN_FIXTURE_BIN) $(COV_DIR)