diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:45:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | bc8bebe78358e3a37bad0867b2d94955cac01d47 (patch) | |
| tree | d5a49a34b95b45cec0917a2a07707159825fb912 /bench/Makefile | |
| parent | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff) | |
| download | ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip | |
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/Makefile')
| -rw-r--r-- | bench/Makefile | 73 |
1 files changed, 73 insertions, 0 deletions
diff --git a/bench/Makefile b/bench/Makefile new file mode 100644 index 0000000..9eeaa95 --- /dev/null +++ b/bench/Makefile @@ -0,0 +1,73 @@ +ROOT := .. +include ../build.mk + +CFLAGS += -std=c11 + +BLAS_LIBS := $(HOST_BLAS_LDFLAGS) + +# Benchmarks link the blas libffwd archive from the per-config build tree and +# include its headers. +LIBFFWD_DIR := ../libffwd +LIBFFWD_A := $(ROOT)/build/$(MODE)/blas/libffwd.a +SERVER_DIR := ../tools/server + +RESULTS := results +# Commit, a -dirty marker when the tree has uncommitted changes, and a timestamp, +# so repeated runs at the same commit do not overwrite each other. +STAMP ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S) + +BENCHES := bench_kernels bench_tokenizer bench_server_utils bench_model \ + bench_late_rerank + +.PHONY: all bench bench-model bench-tokenizer bench-server-utils clean \ + libffwd-blas + +all: $(BENCHES) + +# The archive is built with a backend-specific flag, so a phony target (not a +# file rule) drives the sub-make; the benchmarks list it as an order prerequisite +# and link the resulting $(LIBFFWD_A). +libffwd-blas: + $(MAKE) -C $(LIBFFWD_DIR) BACKEND=blas + +$(eval $(call c_program_deps,bench_kernels,bench_kernels.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)) + +$(eval $(call c_program_deps,bench_tokenizer,bench_tokenizer.c,libffwd-blas,$(LIBFFWD_A),-I$(LIBFFWD_DIR),-lm)) + +# The server utilities (base64, sbuf, util) depend on no library, so the bench +# compiles those sources directly here and keeps the build self-contained. +SERVER_UTIL_SRCS := $(SERVER_DIR)/base64.c $(SERVER_DIR)/sbuf.c $(SERVER_DIR)/util.c +$(eval $(call c_program_deps,bench_server_utils,bench_server_utils.c $(SERVER_UTIL_SRCS),bench.h,,-I$(SERVER_DIR),-lm)) + +$(eval $(call c_program_deps,bench_model,bench_model.c,bench.h libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)) + +$(eval $(call c_program_deps,bench_late_rerank,bench_late_rerank.c,libffwd-blas,$(LIBFFWD_A),-pthread -I$(LIBFFWD_DIR),$(BLAS_LIBS) -lm)) + +# MLX rerank benchmark: the blas engine archive for the CPU reference paths plus +# mlx.c for the device API. The archive calls cblas, so the bench links +# Accelerate/OpenBLAS itself ($(HOST_BLAS_LDFLAGS)) on top of Metal/MLX. macOS only. +MLX_DRIVER_CFLAGS := -pthread -DCHECK_MLX -I$(MLXC_PREFIX)/include -I$(LIBFFWD_DIR) $(HOST_BLAS_CFLAGS) +MLX_DRIVER_LDLIBS := $(MLX_BACKEND_LDFLAGS) $(HOST_BLAS_LDFLAGS) -lm +$(eval $(call c_program_deps,bench_late_rerank_mlx,bench_late_rerank.c $(LIBFFWD_DIR)/mlx.c,libffwd-blas,$(LIBFFWD_A),$(MLX_DRIVER_CFLAGS),$(MLX_DRIVER_LDLIBS))) + +bench: bench_kernels + @mkdir -p $(RESULTS) + ./bench_kernels --json $(RESULTS)/kernels-$(STAMP).json + @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json" + +bench-server-utils: bench_server_utils + @mkdir -p $(RESULTS) + ./bench_server_utils --json $(RESULTS)/server-utils-$(STAMP).json + @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json" + +bench-model: bench_model + @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; } + @mkdir -p $(RESULTS) + ./bench_model "$(MODEL_DIR)" --json $(RESULTS)/model-$(STAMP).json + @echo "compare: ./benchstat.py $(RESULTS)/OLD.json $(RESULTS)/NEW.json" + +bench-tokenizer: bench_tokenizer + @echo "run: ./bench_tokenizer <model_dir-or-vocab.json> <runs> TEXT..." + +clean: + $(RM) -r $(BENCHES) bench_late_rerank_mlx *.dSYM |
