aboutsummaryrefslogtreecommitdiff
path: root/main.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-03 19:26:37 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitfab2f29360320849df6d97d1d01843158402c4d5 (patch)
treed8bebd611cabf083f162a4e53b43d39453a8cfe5 /main.c
parente751321fc00e07aeeb9337f5f21b7191b051c08a (diff)
downloadffwd-fab2f29360320849df6d97d1d01843158402c4d5.tar.gz
ffwd-fab2f29360320849df6d97d1d01843158402c4d5.zip
tokenizer: add reusable encode workspace
Diffstat (limited to 'main.c')
-rw-r--r--main.c19
1 files changed, 18 insertions, 1 deletions
diff --git a/main.c b/main.c
index 2256409..a5728d2 100644
--- a/main.c
+++ b/main.c
@@ -93,6 +93,7 @@ typedef struct {
pplx_mlx_ctx_t *mlx_ctx;
#endif
qwen_tokenizer_t *tok;
+ qwen_tokenizer_workspace_t *tok_ws;
int dim;
} engine_t;
@@ -149,7 +150,8 @@ static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port
static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
{
memset(out, 0, sizeof(*out));
- out->ids = qwen_tokenizer_encode(e->tok, text, &out->n_tokens);
+ out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws,
+ text, &out->n_tokens);
if (!out->ids || out->n_tokens == 0) {
fprintf(stderr, "tokenization failed: %s\n", text);
free(out->ids);
@@ -835,6 +837,20 @@ int main(int argc, char *argv[])
fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
e.dim, now_ms() - t0, use_mlx ? " (MLX)" : "");
+ e.tok_ws = qwen_tokenizer_workspace_new();
+ if (!e.tok_ws) {
+ fprintf(stderr, "failed to allocate tokenizer workspace\n");
+ if (e.workspace) pplx_workspace_free(e.workspace);
+ if (e.model) pplx_model_free(e.model);
+ if (e.dist) pplx_dist_client_free(e.dist);
+#ifdef USE_MLX
+ if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
+#endif
+ qwen_tokenizer_free(tok);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+
/* Run */
int rc;
if (stdin_mode)
@@ -849,6 +865,7 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
#endif
+ qwen_tokenizer_workspace_free(e.tok_ws);
qwen_tokenizer_free(tok);
free_model_specs(&model_specs);
return rc;