diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 14:46:04 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 14:46:04 +0200 |
| commit | 8e8836796e7dd22820c820c0916e83ecdbf38368 (patch) | |
| tree | 4ed9213bd6335d64ec050e27749a6f98047db7a0 /Makefile | |
| parent | 458563c163a885e336ef3365922af4d17e578f56 (diff) | |
| download | ffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.tar.gz ffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.zip | |
tokenizer: add WordPiece backend for BERT-family models
Diffstat (limited to 'Makefile')
| -rw-r--r-- | Makefile | 3 |
1 files changed, 3 insertions, 0 deletions
@@ -164,6 +164,7 @@ endif TEST_CC_FLAGS = -Wall -Wextra -O2 -I. TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS) TEST_TOKENIZER_SRCS = tests/test_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS) +TEST_WORDPIECE_SRCS = tests/test_wordpiece.c wordpiece_tokenizer.c TEST_SAFETENSORS_SRCS = tests/test_safetensors.c qwen_safetensors.c TEST_BF16_SRCS = tests/test_bf16_model.c embed.c embed_config.c qwen_safetensors.c \ $(KERNEL_SRCS) @@ -194,6 +195,8 @@ test: $(CC) $(TEST_CC_FLAGS) -o tests/test_tokenizer \ $(TEST_TOKENIZER_SRCS) -lm -lpthread ./tests/test_tokenizer + $(CC) $(TEST_CC_FLAGS) -o tests/test_wordpiece $(TEST_WORDPIECE_SRCS) + ./tests/test_wordpiece $(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS) ./tests/test_safetensors $(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \ |
