aboutsummaryrefslogtreecommitdiff
path: root/bench/Makefile
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:45:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitbc8bebe78358e3a37bad0867b2d94955cac01d47 (patch)
treed5a49a34b95b45cec0917a2a07707159825fb912 /bench/Makefile
parent6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff)
downloadffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz
ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/Makefile')
-rw-r--r--bench/Makefile73
1 files changed, 73 insertions, 0 deletions
diff --git a/bench/Makefile b/bench/Makefile
new file mode 100644
index 0000000..9eeaa95
--- /dev/null
+++ b/bench/Makefile
@@ -0,0 +1,73 @@
+ROOT := ..
+include ../build.mk
+
+CFLAGS += -std=c11
+
+BLAS_LIBS := $(HOST_BLAS_LDFLAGS)
+
+# Benchmarks link the blas libffwd archive from the per-config build tree and
+# include its headers.
+LIBFFWD_DIR := ../libffwd
+LIBFFWD_A := $(ROOT)/build/$(MODE)/blas/libffwd.a
+SERVER_DIR := ../tools/server
+
+RESULTS := results
+# Commit, a -dirty marker when the tree has uncommitted changes, and a timestamp,
+# so repeated runs at the same commit do not overwrite each other.
+STAMP ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S)
+
+BENCHES := bench_kernels bench_tokenizer bench_server_utils bench_model \
+ bench_late_rerank
+
+.PHONY: all bench bench-model bench-tokenizer bench-server-utils clean \
+ libffwd-blas
+
+all: $(BENCHES)
+
+# The archive is built with a backend-specific flag, so a phony target (not a
+# file rule) drives the sub-make; the benchmarks list it as an order prerequisite
+# and link the resulting $(LIBFFWD_A).
+libffwd-blas:
+ $(MAKE) -C $(LIBFFWD_DIR) BACKEND=blas
+
+$(eval $(call c_program_deps,bench_kernels,bench_kernels.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm))
+
+$(eval $(call c_program_deps,bench_tokenizer,bench_tokenizer.c,libffwd-blas,$(LIBFFWD_A),-I$(LIBFFWD_DIR),-lm))
+
+# The server utilities (base64, sbuf, util) depend on no library, so the bench
+# compiles those sources directly here and keeps the build self-contained.
+SERVER_UTIL_SRCS := $(SERVER_DIR)/base64.c $(SERVER_DIR)/sbuf.c $(SERVER_DIR)/util.c
+$(eval $(call c_program_deps,bench_server_utils,bench_server_utils.c $(SERVER_UTIL_SRCS),bench.h,,-I$(SERVER_DIR),-lm))
+
+$(eval $(call c_program_deps,bench_model,bench_model.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm))
+
+$(eval $(call c_program_deps,bench_late_rerank,bench_late_rerank.c,libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm))
+
+# MLX rerank benchmark: the blas engine archive for the CPU reference paths plus
+# mlx.c for the device API. The archive calls cblas, so the bench links
+# Accelerate/OpenBLAS itself ($(HOST_BLAS_LDFLAGS)) on top of Metal/MLX. macOS only.
+MLX_DRIVER_CFLAGS := -pthread -DCHECK_MLX -I$(MLXC_PREFIX)/include -I$(LIBFFWD_DIR) $(HOST_BLAS_CFLAGS)
+MLX_DRIVER_LDLIBS := $(MLX_BACKEND_LDFLAGS) $(HOST_BLAS_LDFLAGS) -lm
+$(eval $(call c_program_deps,bench_late_rerank_mlx,bench_late_rerank.c $(LIBFFWD_DIR)/mlx.c,libffwd-blas,$(LIBFFWD_A),$(MLX_DRIVER_CFLAGS),$(MLX_DRIVER_LDLIBS)))
+
+bench: bench_kernels
+ @mkdir -p $(RESULTS)
+ ./bench_kernels --json $(RESULTS)/kernels-$(STAMP).json
+ @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json"
+
+bench-server-utils: bench_server_utils
+ @mkdir -p $(RESULTS)
+ ./bench_server_utils --json $(RESULTS)/server-utils-$(STAMP).json
+ @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json"
+
+bench-model: bench_model
+ @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; }
+ @mkdir -p $(RESULTS)
+ ./bench_model "$(MODEL_DIR)" --json $(RESULTS)/model-$(STAMP).json
+ @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json"
+
+bench-tokenizer: bench_tokenizer
+ @echo "run: ./bench_tokenizer <model_dir-or-vocab.json> <runs> TEXT..."
+
+clean:
+ $(RM) -r $(BENCHES) bench_late_rerank_mlx *.dSYM