aboutsummaryrefslogtreecommitdiff
path: root/Makefile
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 09:31:05 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit308bb1b6a058855bb1a2cc45db13f83810cf91b6 (patch)
tree5a08839e050f01421768717e7f4be2b35f5e2f71 /Makefile
parent490a72cf64f191becf4a6f01f06a06899f6a5e78 (diff)
downloadffwd-308bb1b6a058855bb1a2cc45db13f83810cf91b6.tar.gz
ffwd-308bb1b6a058855bb1a2cc45db13f83810cf91b6.zip
tests: hermetic BPE tokenizer test with fast/slow path equality
Diffstat (limited to 'Makefile')
-rw-r--r--Makefile4
1 files changed, 4 insertions, 0 deletions
diff --git a/Makefile b/Makefile
index 05a0088..76dc98c 100644
--- a/Makefile
+++ b/Makefile
@@ -164,6 +164,9 @@ test:
$(CC) -Wall -Wextra -O2 $(TEST_BLAS_CFLAGS) -I. -o tests/test_kernels_blas \
tests/test_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
./tests/test_kernels_blas
+ $(CC) -Wall -Wextra -O2 -I. -o tests/test_tokenizer \
+ tests/test_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS) -lm -lpthread
+ ./tests/test_tokenizer
$(CC) -Wall -Wextra -O2 -I. -o tests/test_safetensors \
tests/test_safetensors.c qwen_safetensors.c
./tests/test_safetensors
@@ -252,4 +255,5 @@ clean:
$(TARGET) $(SERVER_TARGET) $(LIB) libpplxembed.dylib libpplxembed.so \
tests/test_kernels_generic tests/test_kernels_blas \
tests/test_safetensors tests/test_bf16_model tests/test_server \
+ tests/test_tokenizer \
bench/bench_tokenizer