diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:18:12 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 08:18:12 +0200 |
| commit | 35400f3a6b3b38d86eba4ab99a2984f584b4c961 (patch) | |
| tree | 1c49a65ddd6a9cef5d804976798303f13fe4c339 /Makefile | |
| parent | ee07842fa0951a9a171ebd5e1011f25e2f8bf5fc (diff) | |
| download | ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.tar.gz ffwd-35400f3a6b3b38d86eba4ab99a2984f584b4c961.zip | |
tokenizer: full-Unicode punctuation and control classification
Diffstat (limited to 'Makefile')
| -rw-r--r-- | Makefile | 2 |
1 files changed, 1 insertions, 1 deletions
@@ -400,7 +400,7 @@ qwen_kernels_avx.o: qwen_kernels_avx.c qwen_kernels_impl.h qwen_tokenizer.o: qwen_tokenizer.c qwen_tokenizer.h qwen_kernels.h $(CC) $(CFLAGS) -c -o $@ $< -wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h +wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h wordpiece_unicode.h $(CC) $(CFLAGS) -c -o $@ $< qwen_safetensors.o: qwen_safetensors.c qwen_safetensors.h |
