diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-03 19:26:37 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | fab2f29360320849df6d97d1d01843158402c4d5 (patch) | |
| tree | d8bebd611cabf083f162a4e53b43d39453a8cfe5 | |
| parent | e751321fc00e07aeeb9337f5f21b7191b051c08a (diff) | |
| download | ffwd-fab2f29360320849df6d97d1d01843158402c4d5.tar.gz ffwd-fab2f29360320849df6d97d1d01843158402c4d5.zip | |
tokenizer: add reusable encode workspace
| -rw-r--r-- | main.c | 19 | ||||
| -rw-r--r-- | pplx_server.c | 11 | ||||
| -rw-r--r-- | qwen_asr_tokenizer.c | 427 | ||||
| -rw-r--r-- | qwen_asr_tokenizer.h | 23 |
4 files changed, 478 insertions, 2 deletions
@@ -93,6 +93,7 @@ typedef struct { pplx_mlx_ctx_t *mlx_ctx; #endif qwen_tokenizer_t *tok; + qwen_tokenizer_workspace_t *tok_ws; int dim; } engine_t; @@ -149,7 +150,8 @@ static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) { memset(out, 0, sizeof(*out)); - out->ids = qwen_tokenizer_encode(e->tok, text, &out->n_tokens); + out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, + text, &out->n_tokens); if (!out->ids || out->n_tokens == 0) { fprintf(stderr, "tokenization failed: %s\n", text); free(out->ids); @@ -835,6 +837,20 @@ int main(int argc, char *argv[]) fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0, use_mlx ? " (MLX)" : ""); + e.tok_ws = qwen_tokenizer_workspace_new(); + if (!e.tok_ws) { + fprintf(stderr, "failed to allocate tokenizer workspace\n"); + if (e.workspace) pplx_workspace_free(e.workspace); + if (e.model) pplx_model_free(e.model); + if (e.dist) pplx_dist_client_free(e.dist); +#ifdef USE_MLX + if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); +#endif + qwen_tokenizer_free(tok); + free_model_specs(&model_specs); + return 1; + } + /* Run */ int rc; if (stdin_mode) @@ -849,6 +865,7 @@ int main(int argc, char *argv[]) #ifdef USE_MLX if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); #endif + qwen_tokenizer_workspace_free(e.tok_ws); qwen_tokenizer_free(tok); free_model_specs(&model_specs); return rc; diff --git a/pplx_server.c b/pplx_server.c index 135a4d4..5bba373 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -210,6 +210,7 @@ typedef struct { const model_info *info; char *path; qwen_tokenizer_t *tok; + qwen_tokenizer_workspace_t *tok_ws; int context_separator_id; pplx_model_t *cpu_model; pplx_workspace_t *cpu_ws; @@ -1065,7 +1066,8 @@ static void contextual_request_free(contextual_request *r) { static int tokenize_one(loaded_model *m, const char *text, token_buf *out) { memset(out, 0, sizeof(*out)); - out->ids = qwen_tokenizer_encode(m->tok, text, &out->n_tokens); + out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, + text, &out->n_tokens); if (!out->ids || out->n_tokens <= 0) { free(out->ids); memset(out, 0, sizeof(*out)); @@ -1967,6 +1969,12 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { server_log("pplx-serve: failed to load tokenizer: %s", vocab_path); return -1; } + m->tok_ws = qwen_tokenizer_workspace_new(); + if (!m->tok_ws) { + server_log("pplx-serve: failed to allocate tokenizer workspace: %s", + path); + return -1; + } m->context_separator_id = PPLX_CONTEXT_SEPARATOR_TOKEN_ID; #ifdef USE_MLX if (s->use_mlx) { @@ -1998,6 +2006,7 @@ static void free_models(http_server *s) { for (int i = 0; i < 4; i++) { loaded_model *m = &s->models[i]; free(m->path); + if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws); if (m->tok) qwen_tokenizer_free(m->tok); if (m->cpu_ws) pplx_workspace_free(m->cpu_ws); if (m->cpu_model) pplx_model_free(m->cpu_model); diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c index 8c1b84a..8c86f22 100644 --- a/qwen_asr_tokenizer.c +++ b/qwen_asr_tokenizer.c @@ -825,6 +825,365 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int return 0; } +typedef struct { + size_t off; +} bpe_sym_ref_t; + +struct qwen_tokenizer_workspace { + char *normalized; + size_t normalized_cap; + char *mapped; + size_t mapped_cap; + char *arena; + size_t arena_cap; + size_t arena_len; + bpe_sym_ref_t *syms; + int syms_cap; + int *ids; + int ids_cap; +}; + +static int reserve_bytes(char **buf, size_t *cap, size_t need) { + if (*cap >= need) return 0; + size_t new_cap = *cap ? *cap : 256; + while (new_cap < need) { + if (new_cap > SIZE_MAX / 2) return -1; + new_cap *= 2; + } + char *p = (char *)realloc(*buf, new_cap); + if (!p) return -1; + *buf = p; + *cap = new_cap; + return 0; +} + +static int reserve_ints(int **buf, int *cap, int need) { + if (*cap >= need) return 0; + int new_cap = *cap ? *cap : 64; + while (new_cap < need) { + if (new_cap > INT_MAX / 2) return -1; + new_cap *= 2; + } + int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int)); + if (!p) return -1; + *buf = p; + *cap = new_cap; + return 0; +} + +static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) { + if (ws->syms_cap >= need) return 0; + int new_cap = ws->syms_cap ? ws->syms_cap : 64; + while (new_cap < need) { + if (new_cap > INT_MAX / 2) return -1; + new_cap *= 2; + } + bpe_sym_ref_t *p = + (bpe_sym_ref_t *)realloc(ws->syms, + (size_t)new_cap * sizeof(*ws->syms)); + if (!p) return -1; + ws->syms = p; + ws->syms_cap = new_cap; + return 0; +} + +static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, + size_t len, size_t *off) { + if (ws->arena_len > SIZE_MAX - len - 1) return -1; + size_t need = ws->arena_len + len + 1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) return -1; + *off = ws->arena_len; + memcpy(ws->arena + ws->arena_len, s, len); + ws->arena_len += len; + ws->arena[ws->arena_len++] = '\0'; + return 0; +} + +static const char *arena_str(const qwen_tokenizer_workspace_t *ws, + bpe_sym_ref_t sym) { + return ws->arena + sym.off; +} + +static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, + const char *text, const char **out) { + const unsigned char *p = (const unsigned char *)text; + const unsigned char *end = p + strlen(text); + size_t in_len = (size_t)(end - p); + if (in_len > (SIZE_MAX - 1) / 2) return -1; + if (reserve_bytes(&ws->normalized, &ws->normalized_cap, + in_len * 2 + 1) != 0) + return -1; + + size_t w = 0; + while (p < end) { + const unsigned char *cur = p; + int cp = 0, len = 0; + if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + p += len; + + if (p < end) { + int mark = 0, mark_len = 0; + utf8_decode_cp_len(p, end, &mark, &mark_len); + int composed = compose_latin_mark(cp, mark); + if (composed) { + char tmp[4]; + int n = utf8_encode_full_cp(composed, tmp); + memcpy(ws->normalized + w, tmp, (size_t)n); + w += (size_t)n; + p += mark_len; + continue; + } + } + + memcpy(ws->normalized + w, cur, (size_t)len); + w += (size_t)len; + } + + ws->normalized[w] = '\0'; + *out = ws->normalized; + return 0; +} + +static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, + const char *text, size_t len, + const char **out) { + init_gpt2_mapping(); + if (len > (SIZE_MAX - 1) / 3) return -1; + if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0) + return -1; + + size_t w = 0; + for (size_t i = 0; i < len; i++) { + int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; + char tmp[4]; + int n = utf8_encode_cp(cp, tmp); + memcpy(ws->mapped + w, tmp, (size_t)n); + w += (size_t)n; + } + ws->mapped[w] = '\0'; + *out = ws->mapped; + return 0; +} + +static int output_append_id(int *out_ids, int out_cap, int *n_ids, + int *overflow, int id) { + if (*n_ids < out_cap && out_ids) + out_ids[*n_ids] = id; + else + *overflow = 1; + (*n_ids)++; + return 0; +} + +static int encode_bpe_word_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *mapped, + int *out_ids, int out_cap, + int *n_ids, int *overflow) { + if (!mapped || !*mapped) return 0; + + int n_syms = 0; + for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + int l = utf8_char_len(*p); + p += l; + n_syms++; + } + if (n_syms <= 0) return 0; + if (reserve_syms(ws, n_syms) != 0) return -1; + + ws->arena_len = 0; + int i = 0; + for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + int l = utf8_char_len(*p); + size_t off = 0; + if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0) + return -1; + ws->syms[i++].off = off; + p += l; + } + + while (n_syms > 1) { + int best_rank = INT_MAX; + int best_i = -1; + for (int j = 0; j < n_syms - 1; j++) { + int r = merge_rank(tok, arena_str(ws, ws->syms[j]), + arena_str(ws, ws->syms[j + 1])); + if (r < best_rank) { + best_rank = r; + best_i = j; + } + } + if (best_i < 0 || best_rank == INT_MAX) break; + + const char *a = arena_str(ws, ws->syms[best_i]); + const char *b = arena_str(ws, ws->syms[best_i + 1]); + size_t la = strlen(a), lb = strlen(b); + if (la > SIZE_MAX - lb) return -1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, + ws->arena_len + la + lb + 1) != 0) + return -1; + + a = arena_str(ws, ws->syms[best_i]); + b = arena_str(ws, ws->syms[best_i + 1]); + size_t off = ws->arena_len; + memcpy(ws->arena + ws->arena_len, a, la); + ws->arena_len += la; + memcpy(ws->arena + ws->arena_len, b, lb); + ws->arena_len += lb; + ws->arena[ws->arena_len++] = '\0'; + ws->syms[best_i].off = off; + for (int j = best_i + 1; j < n_syms - 1; j++) + ws->syms[j] = ws->syms[j + 1]; + n_syms--; + } + + for (int j = 0; j < n_syms; j++) { + int id = map_get((const str_int_entry_t *)tok->vocab_map, + tok->vocab_map_cap, arena_str(ws, ws->syms[j])); + if (id < 0) return -1; + output_append_id(out_ids, out_cap, n_ids, overflow, id); + } + return 0; +} + +static int append_encoded_piece_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *piece, size_t piece_len, + int *out_ids, int out_cap, + int *n_ids, int *overflow) { + if (piece_len == 0) return 0; + + const char *mapped = NULL; + if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0) + return -1; + return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, + overflow); +} + +static int encode_pretokenized_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, int out_cap, + int *out_n) { + const unsigned char *start = (const unsigned char *)text; + const unsigned char *p = start; + const unsigned char *end = start + strlen(text); + int n_ids = 0; + int overflow = 0; + + while (p < end) { + const unsigned char *piece = p; + int cp = 0, len = 0; + if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + + int clen = match_contraction(p, end); + if (clen > 0) { + p += clen; + } else { + const unsigned char *next = p + len; + int next_cp = 0, next_len = 0; + int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); + + if (!is_newline_cp(cp) && !is_word_cp(cp) && + has_next && is_letter_cp(next_cp)) { + p = next + next_len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_letter_cp(c)) break; + p += l; + } + } else if (is_letter_cp(cp)) { + p += len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_letter_cp(c)) break; + p += l; + } + } else if (is_number_cp(cp)) { + p += len; + } else if (is_space_cp(cp)) { + const unsigned char *q = p; + int saw_newline = 0; + while (q < end) { + int c = 0, l = 0; + utf8_decode_cp_len(q, end, &c, &l); + if (!is_space_cp(c)) break; + if (is_newline_cp(c)) saw_newline = 1; + q += l; + if (saw_newline) { + while (q < end) { + int c2 = 0, l2 = 0; + utf8_decode_cp_len(q, end, &c2, &l2); + if (!is_newline_cp(c2)) break; + q += l2; + } + break; + } + } + + if (saw_newline) { + p = q; + } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && + !is_word_cp(next_cp)) { + p = next + next_len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (is_space_cp(c) || is_word_cp(c)) break; + p += l; + } + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_newline_cp(c)) break; + p += l; + } + } else { + const unsigned char *last = p; + const unsigned char *q2 = p; + int count = 0; + while (q2 < end) { + int c = 0, l = 0; + utf8_decode_cp_len(q2, end, &c, &l); + if (!is_space_cp(c)) break; + last = q2; + q2 += l; + count++; + } + if (q2 < end && count > 1) + p = last; + else + p = q2; + } + } else { + p += len; + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (is_space_cp(c) || is_word_cp(c)) break; + p += l; + } + while (p < end) { + int c = 0, l = 0; + utf8_decode_cp_len(p, end, &c, &l); + if (!is_newline_cp(c)) break; + p += l; + } + } + } + + if (append_encoded_piece_into(tok, ws, (const char *)piece, + (size_t)(p - piece), out_ids, out_cap, + &n_ids, &overflow) != 0) + return -1; + } + + *out_n = n_ids; + return overflow ? -2 : 0; +} + static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) { const char *slash = strrchr(vocab_path, '/'); if (!slash) { @@ -1043,6 +1402,74 @@ int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *o return ids; } +qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) { + return (qwen_tokenizer_workspace_t *)calloc(1, + sizeof(qwen_tokenizer_workspace_t)); +} + +void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { + if (!ws) return; + free(ws->normalized); + free(ws->mapped); + free(ws->arena); + free(ws->syms); + free(ws->ids); + free(ws); +} + +int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, int out_cap, + int *out_n_tokens) { + if (out_n_tokens) *out_n_tokens = 0; + if (!tok || !ws || !text || out_cap < 0) return -1; + if (text[0] == '\0') return 0; + + const char *normalized = NULL; + if (normalize_nfc_latin_into(ws, text, &normalized) != 0) + return -1; + + int n_ids = 0; + int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, + &n_ids); + if (out_n_tokens) *out_n_tokens = n_ids; + return rc; +} + +int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_n_tokens) { + if (out_n_tokens) *out_n_tokens = 0; + if (!tok || !ws || !text || text[0] == '\0') return NULL; + + size_t len = strlen(text); + size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8; + if (cap_sz > (size_t)INT_MAX) return NULL; + int cap = (int)cap_sz; + + if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0) + return NULL; + + int n_ids = 0; + int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, + &n_ids); + if (rc == -2) { + if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) + return NULL; + rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, + &n_ids); + } + if (rc != 0 || n_ids <= 0) return NULL; + + int *ids = (int *)malloc((size_t)n_ids * sizeof(int)); + if (!ids) return NULL; + memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int)); + if (out_n_tokens) *out_n_tokens = n_ids; + return ids; +} + void qwen_tokenizer_free(qwen_tokenizer_t *tok) { if (!tok) return; diff --git a/qwen_asr_tokenizer.h b/qwen_asr_tokenizer.h index 0cea192..742d0d1 100644 --- a/qwen_asr_tokenizer.h +++ b/qwen_asr_tokenizer.h @@ -21,6 +21,8 @@ typedef struct { int merge_map_cap; } qwen_tokenizer_t; +typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; + /* Load tokenizer from vocab.json in model directory */ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); @@ -32,6 +34,27 @@ const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); * Returns NULL on error (and sets *out_n_tokens to 0). */ int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); +/* Reusable scratch for allocation-light repeated encoding. */ +qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); +void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); + +/* Encode into caller-provided out_ids[0..out_cap-1]. + * Returns 0 on success, -1 on error, -2 if out_cap is too small. + * In all cases *out_n_tokens is set to the number of tokens needed/emitted + * when out_n_tokens is non-NULL. */ +int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, int out_cap, + int *out_n_tokens); + +/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and + * performs only the final returned int[] allocation. */ +int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_n_tokens); + /* Free tokenizer */ void qwen_tokenizer_free(qwen_tokenizer_t *tok); |
