diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-03 19:26:37 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | fab2f29360320849df6d97d1d01843158402c4d5 (patch) | |
| tree | d8bebd611cabf083f162a4e53b43d39453a8cfe5 /pplx_server.c | |
| parent | e751321fc00e07aeeb9337f5f21b7191b051c08a (diff) | |
| download | ffwd-fab2f29360320849df6d97d1d01843158402c4d5.tar.gz ffwd-fab2f29360320849df6d97d1d01843158402c4d5.zip | |
tokenizer: add reusable encode workspace
Diffstat (limited to 'pplx_server.c')
| -rw-r--r-- | pplx_server.c | 11 |
1 files changed, 10 insertions, 1 deletions
diff --git a/pplx_server.c b/pplx_server.c index 135a4d4..5bba373 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -210,6 +210,7 @@ typedef struct { const model_info *info; char *path; qwen_tokenizer_t *tok; + qwen_tokenizer_workspace_t *tok_ws; int context_separator_id; pplx_model_t *cpu_model; pplx_workspace_t *cpu_ws; @@ -1065,7 +1066,8 @@ static void contextual_request_free(contextual_request *r) { static int tokenize_one(loaded_model *m, const char *text, token_buf *out) { memset(out, 0, sizeof(*out)); - out->ids = qwen_tokenizer_encode(m->tok, text, &out->n_tokens); + out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, + text, &out->n_tokens); if (!out->ids || out->n_tokens <= 0) { free(out->ids); memset(out, 0, sizeof(*out)); @@ -1967,6 +1969,12 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { server_log("pplx-serve: failed to load tokenizer: %s", vocab_path); return -1; } + m->tok_ws = qwen_tokenizer_workspace_new(); + if (!m->tok_ws) { + server_log("pplx-serve: failed to allocate tokenizer workspace: %s", + path); + return -1; + } m->context_separator_id = PPLX_CONTEXT_SEPARATOR_TOKEN_ID; #ifdef USE_MLX if (s->use_mlx) { @@ -1998,6 +2006,7 @@ static void free_models(http_server *s) { for (int i = 0; i < 4; i++) { loaded_model *m = &s->models[i]; free(m->path); + if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws); if (m->tok) qwen_tokenizer_free(m->tok); if (m->cpu_ws) pplx_workspace_free(m->cpu_ws); if (m->cpu_model) pplx_model_free(m->cpu_model); |
