aboutsummaryrefslogtreecommitdiff
path: root/Makefile
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:18:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 08:18:12 +0200
commit35400f3a6b3b38d86eba4ab99a2984f584b4c961 (patch)
tree1c49a65ddd6a9cef5d804976798303f13fe4c339 /Makefile
parentee07842fa0951a9a171ebd5e1011f25e2f8bf5fc (diff)
downloadffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.tar.gz
ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.zip
tokenizer: full-Unicode punctuation and control classification
Diffstat (limited to 'Makefile')
-rw-r--r--Makefile2
1 files changed, 1 insertions, 1 deletions
diff --git a/Makefile b/Makefile
index 506fdb4..9de5f83 100644
--- a/Makefile
+++ b/Makefile
@@ -400,7 +400,7 @@ qwen_kernels_avx.o: qwen_kernels_avx.c qwen_kernels_impl.h
qwen_tokenizer.o: qwen_tokenizer.c qwen_tokenizer.h qwen_kernels.h
$(CC) $(CFLAGS) -c -o $@ $<
-wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h
+wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h wordpiece_unicode.h
$(CC) $(CFLAGS) -c -o $@ $<
qwen_safetensors.o: qwen_safetensors.c qwen_safetensors.h