From 849c571a8e79d92c1fd3dabecf0fb3621aa4f59b Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 16 Jun 2026 08:56:02 +0200 Subject: cli: tokenize BERT models with WordPiece and [CLS]/[SEP] --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 9de5f83..bab019a 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ TEST_SERVER_SRCS = tests/test_server.c embed.c embed_config.c \ deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c # The CLI check builds the real CLI (CPU backend) plus a driver that runs it. TEST_CLI_BIN_SRCS = embed_cli.c embed.c embed_config.c \ - qwen_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) + qwen_tokenizer.c wordpiece_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) test: $(CC) $(TEST_CC_FLAGS) -o tests/test_kernels_generic \ @@ -381,7 +381,7 @@ embed_server.o: embed_server.c embed_server.h embed_build.h embed.h embed_mlx.h embed_mlx.o: embed_mlx.c embed_mlx.h embed_config.h embed.h qwen_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< -embed_cli.o: embed_cli.c embed_build.h embed.h qwen_kernels.h qwen_tokenizer.h +embed_cli.o: embed_cli.c embed_build.h embed.h qwen_kernels.h qwen_tokenizer.h wordpiece_tokenizer.h $(CC) $(CFLAGS) $(VERSION_CFLAGS) -Ideps/ae -c -o $@ $< -- cgit v1.3.1