aboutsummaryrefslogtreecommitdiff
path: root/Makefile
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 14:46:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 14:46:04 +0200
commit8e8836796e7dd22820c820c0916e83ecdbf38368 (patch)
tree4ed9213bd6335d64ec050e27749a6f98047db7a0 /Makefile
parent458563c163a885e336ef3365922af4d17e578f56 (diff)
downloadffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.tar.gz
ffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.zip
tokenizer: add WordPiece backend for BERT-family models
Diffstat (limited to 'Makefile')
-rw-r--r--Makefile3
1 files changed, 3 insertions, 0 deletions
diff --git a/Makefile b/Makefile
index 76b9f57..3dfb8dd 100644
--- a/Makefile
+++ b/Makefile
@@ -164,6 +164,7 @@ endif
TEST_CC_FLAGS = -Wall -Wextra -O2 -I.
TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS)
TEST_TOKENIZER_SRCS = tests/test_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS)
+TEST_WORDPIECE_SRCS = tests/test_wordpiece.c wordpiece_tokenizer.c
TEST_SAFETENSORS_SRCS = tests/test_safetensors.c qwen_safetensors.c
TEST_BF16_SRCS = tests/test_bf16_model.c embed.c embed_config.c qwen_safetensors.c \
$(KERNEL_SRCS)
@@ -194,6 +195,8 @@ test:
$(CC) $(TEST_CC_FLAGS) -o tests/test_tokenizer \
$(TEST_TOKENIZER_SRCS) -lm -lpthread
./tests/test_tokenizer
+ $(CC) $(TEST_CC_FLAGS) -o tests/test_wordpiece $(TEST_WORDPIECE_SRCS)
+ ./tests/test_wordpiece
$(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS)
./tests/test_safetensors
$(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \