aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-03 19:26:37 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitfab2f29360320849df6d97d1d01843158402c4d5 (patch)
treed8bebd611cabf083f162a4e53b43d39453a8cfe5
parente751321fc00e07aeeb9337f5f21b7191b051c08a (diff)
downloadffwd-fab2f29360320849df6d97d1d01843158402c4d5.tar.gz
ffwd-fab2f29360320849df6d97d1d01843158402c4d5.zip
tokenizer: add reusable encode workspace
-rw-r--r--main.c19
-rw-r--r--pplx_server.c11
-rw-r--r--qwen_asr_tokenizer.c427
-rw-r--r--qwen_asr_tokenizer.h23
4 files changed, 478 insertions, 2 deletions
diff --git a/main.c b/main.c
index 2256409..a5728d2 100644
--- a/main.c
+++ b/main.c
@@ -93,6 +93,7 @@ typedef struct {
pplx_mlx_ctx_t *mlx_ctx;
#endif
qwen_tokenizer_t *tok;
+ qwen_tokenizer_workspace_t *tok_ws;
int dim;
} engine_t;
@@ -149,7 +150,8 @@ static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port
static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
{
memset(out, 0, sizeof(*out));
- out->ids = qwen_tokenizer_encode(e->tok, text, &out->n_tokens);
+ out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws,
+ text, &out->n_tokens);
if (!out->ids || out->n_tokens == 0) {
fprintf(stderr, "tokenization failed: %s\n", text);
free(out->ids);
@@ -835,6 +837,20 @@ int main(int argc, char *argv[])
fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
e.dim, now_ms() - t0, use_mlx ? " (MLX)" : "");
+ e.tok_ws = qwen_tokenizer_workspace_new();
+ if (!e.tok_ws) {
+ fprintf(stderr, "failed to allocate tokenizer workspace\n");
+ if (e.workspace) pplx_workspace_free(e.workspace);
+ if (e.model) pplx_model_free(e.model);
+ if (e.dist) pplx_dist_client_free(e.dist);
+#ifdef USE_MLX
+ if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
+#endif
+ qwen_tokenizer_free(tok);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+
/* Run */
int rc;
if (stdin_mode)
@@ -849,6 +865,7 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
#endif
+ qwen_tokenizer_workspace_free(e.tok_ws);
qwen_tokenizer_free(tok);
free_model_specs(&model_specs);
return rc;
diff --git a/pplx_server.c b/pplx_server.c
index 135a4d4..5bba373 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -210,6 +210,7 @@ typedef struct {
const model_info *info;
char *path;
qwen_tokenizer_t *tok;
+ qwen_tokenizer_workspace_t *tok_ws;
int context_separator_id;
pplx_model_t *cpu_model;
pplx_workspace_t *cpu_ws;
@@ -1065,7 +1066,8 @@ static void contextual_request_free(contextual_request *r) {
static int tokenize_one(loaded_model *m, const char *text, token_buf *out) {
memset(out, 0, sizeof(*out));
- out->ids = qwen_tokenizer_encode(m->tok, text, &out->n_tokens);
+ out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws,
+ text, &out->n_tokens);
if (!out->ids || out->n_tokens <= 0) {
free(out->ids);
memset(out, 0, sizeof(*out));
@@ -1967,6 +1969,12 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
server_log("pplx-serve: failed to load tokenizer: %s", vocab_path);
return -1;
}
+ m->tok_ws = qwen_tokenizer_workspace_new();
+ if (!m->tok_ws) {
+ server_log("pplx-serve: failed to allocate tokenizer workspace: %s",
+ path);
+ return -1;
+ }
m->context_separator_id = PPLX_CONTEXT_SEPARATOR_TOKEN_ID;
#ifdef USE_MLX
if (s->use_mlx) {
@@ -1998,6 +2006,7 @@ static void free_models(http_server *s) {
for (int i = 0; i < 4; i++) {
loaded_model *m = &s->models[i];
free(m->path);
+ if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws);
if (m->tok) qwen_tokenizer_free(m->tok);
if (m->cpu_ws) pplx_workspace_free(m->cpu_ws);
if (m->cpu_model) pplx_model_free(m->cpu_model);
diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c
index 8c1b84a..8c86f22 100644
--- a/qwen_asr_tokenizer.c
+++ b/qwen_asr_tokenizer.c
@@ -825,6 +825,365 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int
return 0;
}
+typedef struct {
+ size_t off;
+} bpe_sym_ref_t;
+
+struct qwen_tokenizer_workspace {
+ char *normalized;
+ size_t normalized_cap;
+ char *mapped;
+ size_t mapped_cap;
+ char *arena;
+ size_t arena_cap;
+ size_t arena_len;
+ bpe_sym_ref_t *syms;
+ int syms_cap;
+ int *ids;
+ int ids_cap;
+};
+
+static int reserve_bytes(char **buf, size_t *cap, size_t need) {
+ if (*cap >= need) return 0;
+ size_t new_cap = *cap ? *cap : 256;
+ while (new_cap < need) {
+ if (new_cap > SIZE_MAX / 2) return -1;
+ new_cap *= 2;
+ }
+ char *p = (char *)realloc(*buf, new_cap);
+ if (!p) return -1;
+ *buf = p;
+ *cap = new_cap;
+ return 0;
+}
+
+static int reserve_ints(int **buf, int *cap, int need) {
+ if (*cap >= need) return 0;
+ int new_cap = *cap ? *cap : 64;
+ while (new_cap < need) {
+ if (new_cap > INT_MAX / 2) return -1;
+ new_cap *= 2;
+ }
+ int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int));
+ if (!p) return -1;
+ *buf = p;
+ *cap = new_cap;
+ return 0;
+}
+
+static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) {
+ if (ws->syms_cap >= need) return 0;
+ int new_cap = ws->syms_cap ? ws->syms_cap : 64;
+ while (new_cap < need) {
+ if (new_cap > INT_MAX / 2) return -1;
+ new_cap *= 2;
+ }
+ bpe_sym_ref_t *p =
+ (bpe_sym_ref_t *)realloc(ws->syms,
+ (size_t)new_cap * sizeof(*ws->syms));
+ if (!p) return -1;
+ ws->syms = p;
+ ws->syms_cap = new_cap;
+ return 0;
+}
+
+static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s,
+ size_t len, size_t *off) {
+ if (ws->arena_len > SIZE_MAX - len - 1) return -1;
+ size_t need = ws->arena_len + len + 1;
+ if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) return -1;
+ *off = ws->arena_len;
+ memcpy(ws->arena + ws->arena_len, s, len);
+ ws->arena_len += len;
+ ws->arena[ws->arena_len++] = '\0';
+ return 0;
+}
+
+static const char *arena_str(const qwen_tokenizer_workspace_t *ws,
+ bpe_sym_ref_t sym) {
+ return ws->arena + sym.off;
+}
+
+static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws,
+ const char *text, const char **out) {
+ const unsigned char *p = (const unsigned char *)text;
+ const unsigned char *end = p + strlen(text);
+ size_t in_len = (size_t)(end - p);
+ if (in_len > (SIZE_MAX - 1) / 2) return -1;
+ if (reserve_bytes(&ws->normalized, &ws->normalized_cap,
+ in_len * 2 + 1) != 0)
+ return -1;
+
+ size_t w = 0;
+ while (p < end) {
+ const unsigned char *cur = p;
+ int cp = 0, len = 0;
+ if (!utf8_decode_cp_len(p, end, &cp, &len)) break;
+ p += len;
+
+ if (p < end) {
+ int mark = 0, mark_len = 0;
+ utf8_decode_cp_len(p, end, &mark, &mark_len);
+ int composed = compose_latin_mark(cp, mark);
+ if (composed) {
+ char tmp[4];
+ int n = utf8_encode_full_cp(composed, tmp);
+ memcpy(ws->normalized + w, tmp, (size_t)n);
+ w += (size_t)n;
+ p += mark_len;
+ continue;
+ }
+ }
+
+ memcpy(ws->normalized + w, cur, (size_t)len);
+ w += (size_t)len;
+ }
+
+ ws->normalized[w] = '\0';
+ *out = ws->normalized;
+ return 0;
+}
+
+static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws,
+ const char *text, size_t len,
+ const char **out) {
+ init_gpt2_mapping();
+ if (len > (SIZE_MAX - 1) / 3) return -1;
+ if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0)
+ return -1;
+
+ size_t w = 0;
+ for (size_t i = 0; i < len; i++) {
+ int cp = gpt2_byte_to_unicode[(unsigned char)text[i]];
+ char tmp[4];
+ int n = utf8_encode_cp(cp, tmp);
+ memcpy(ws->mapped + w, tmp, (size_t)n);
+ w += (size_t)n;
+ }
+ ws->mapped[w] = '\0';
+ *out = ws->mapped;
+ return 0;
+}
+
+static int output_append_id(int *out_ids, int out_cap, int *n_ids,
+ int *overflow, int id) {
+ if (*n_ids < out_cap && out_ids)
+ out_ids[*n_ids] = id;
+ else
+ *overflow = 1;
+ (*n_ids)++;
+ return 0;
+}
+
+static int encode_bpe_word_into(const qwen_tokenizer_t *tok,
+ qwen_tokenizer_workspace_t *ws,
+ const char *mapped,
+ int *out_ids, int out_cap,
+ int *n_ids, int *overflow) {
+ if (!mapped || !*mapped) return 0;
+
+ int n_syms = 0;
+ for (const unsigned char *p = (const unsigned char *)mapped; *p; ) {
+ int l = utf8_char_len(*p);
+ p += l;
+ n_syms++;
+ }
+ if (n_syms <= 0) return 0;
+ if (reserve_syms(ws, n_syms) != 0) return -1;
+
+ ws->arena_len = 0;
+ int i = 0;
+ for (const unsigned char *p = (const unsigned char *)mapped; *p; ) {
+ int l = utf8_char_len(*p);
+ size_t off = 0;
+ if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0)
+ return -1;
+ ws->syms[i++].off = off;
+ p += l;
+ }
+
+ while (n_syms > 1) {
+ int best_rank = INT_MAX;
+ int best_i = -1;
+ for (int j = 0; j < n_syms - 1; j++) {
+ int r = merge_rank(tok, arena_str(ws, ws->syms[j]),
+ arena_str(ws, ws->syms[j + 1]));
+ if (r < best_rank) {
+ best_rank = r;
+ best_i = j;
+ }
+ }
+ if (best_i < 0 || best_rank == INT_MAX) break;
+
+ const char *a = arena_str(ws, ws->syms[best_i]);
+ const char *b = arena_str(ws, ws->syms[best_i + 1]);
+ size_t la = strlen(a), lb = strlen(b);
+ if (la > SIZE_MAX - lb) return -1;
+ if (reserve_bytes(&ws->arena, &ws->arena_cap,
+ ws->arena_len + la + lb + 1) != 0)
+ return -1;
+
+ a = arena_str(ws, ws->syms[best_i]);
+ b = arena_str(ws, ws->syms[best_i + 1]);
+ size_t off = ws->arena_len;
+ memcpy(ws->arena + ws->arena_len, a, la);
+ ws->arena_len += la;
+ memcpy(ws->arena + ws->arena_len, b, lb);
+ ws->arena_len += lb;
+ ws->arena[ws->arena_len++] = '\0';
+ ws->syms[best_i].off = off;
+ for (int j = best_i + 1; j < n_syms - 1; j++)
+ ws->syms[j] = ws->syms[j + 1];
+ n_syms--;
+ }
+
+ for (int j = 0; j < n_syms; j++) {
+ int id = map_get((const str_int_entry_t *)tok->vocab_map,
+ tok->vocab_map_cap, arena_str(ws, ws->syms[j]));
+ if (id < 0) return -1;
+ output_append_id(out_ids, out_cap, n_ids, overflow, id);
+ }
+ return 0;
+}
+
+static int append_encoded_piece_into(const qwen_tokenizer_t *tok,
+ qwen_tokenizer_workspace_t *ws,
+ const char *piece, size_t piece_len,
+ int *out_ids, int out_cap,
+ int *n_ids, int *overflow) {
+ if (piece_len == 0) return 0;
+
+ const char *mapped = NULL;
+ if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0)
+ return -1;
+ return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids,
+ overflow);
+}
+
+static int encode_pretokenized_into(const qwen_tokenizer_t *tok,
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_ids, int out_cap,
+ int *out_n) {
+ const unsigned char *start = (const unsigned char *)text;
+ const unsigned char *p = start;
+ const unsigned char *end = start + strlen(text);
+ int n_ids = 0;
+ int overflow = 0;
+
+ while (p < end) {
+ const unsigned char *piece = p;
+ int cp = 0, len = 0;
+ if (!utf8_decode_cp_len(p, end, &cp, &len)) break;
+
+ int clen = match_contraction(p, end);
+ if (clen > 0) {
+ p += clen;
+ } else {
+ const unsigned char *next = p + len;
+ int next_cp = 0, next_len = 0;
+ int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len);
+
+ if (!is_newline_cp(cp) && !is_word_cp(cp) &&
+ has_next && is_letter_cp(next_cp)) {
+ p = next + next_len;
+ while (p < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(p, end, &c, &l);
+ if (!is_letter_cp(c)) break;
+ p += l;
+ }
+ } else if (is_letter_cp(cp)) {
+ p += len;
+ while (p < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(p, end, &c, &l);
+ if (!is_letter_cp(c)) break;
+ p += l;
+ }
+ } else if (is_number_cp(cp)) {
+ p += len;
+ } else if (is_space_cp(cp)) {
+ const unsigned char *q = p;
+ int saw_newline = 0;
+ while (q < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(q, end, &c, &l);
+ if (!is_space_cp(c)) break;
+ if (is_newline_cp(c)) saw_newline = 1;
+ q += l;
+ if (saw_newline) {
+ while (q < end) {
+ int c2 = 0, l2 = 0;
+ utf8_decode_cp_len(q, end, &c2, &l2);
+ if (!is_newline_cp(c2)) break;
+ q += l2;
+ }
+ break;
+ }
+ }
+
+ if (saw_newline) {
+ p = q;
+ } else if (cp == ' ' && has_next && !is_space_cp(next_cp) &&
+ !is_word_cp(next_cp)) {
+ p = next + next_len;
+ while (p < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(p, end, &c, &l);
+ if (is_space_cp(c) || is_word_cp(c)) break;
+ p += l;
+ }
+ while (p < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(p, end, &c, &l);
+ if (!is_newline_cp(c)) break;
+ p += l;
+ }
+ } else {
+ const unsigned char *last = p;
+ const unsigned char *q2 = p;
+ int count = 0;
+ while (q2 < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(q2, end, &c, &l);
+ if (!is_space_cp(c)) break;
+ last = q2;
+ q2 += l;
+ count++;
+ }
+ if (q2 < end && count > 1)
+ p = last;
+ else
+ p = q2;
+ }
+ } else {
+ p += len;
+ while (p < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(p, end, &c, &l);
+ if (is_space_cp(c) || is_word_cp(c)) break;
+ p += l;
+ }
+ while (p < end) {
+ int c = 0, l = 0;
+ utf8_decode_cp_len(p, end, &c, &l);
+ if (!is_newline_cp(c)) break;
+ p += l;
+ }
+ }
+ }
+
+ if (append_encoded_piece_into(tok, ws, (const char *)piece,
+ (size_t)(p - piece), out_ids, out_cap,
+ &n_ids, &overflow) != 0)
+ return -1;
+ }
+
+ *out_n = n_ids;
+ return overflow ? -2 : 0;
+}
+
static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) {
const char *slash = strrchr(vocab_path, '/');
if (!slash) {
@@ -1043,6 +1402,74 @@ int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *o
return ids;
}
+qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) {
+ return (qwen_tokenizer_workspace_t *)calloc(1,
+ sizeof(qwen_tokenizer_workspace_t));
+}
+
+void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) {
+ if (!ws) return;
+ free(ws->normalized);
+ free(ws->mapped);
+ free(ws->arena);
+ free(ws->syms);
+ free(ws->ids);
+ free(ws);
+}
+
+int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_ids, int out_cap,
+ int *out_n_tokens) {
+ if (out_n_tokens) *out_n_tokens = 0;
+ if (!tok || !ws || !text || out_cap < 0) return -1;
+ if (text[0] == '\0') return 0;
+
+ const char *normalized = NULL;
+ if (normalize_nfc_latin_into(ws, text, &normalized) != 0)
+ return -1;
+
+ int n_ids = 0;
+ int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap,
+ &n_ids);
+ if (out_n_tokens) *out_n_tokens = n_ids;
+ return rc;
+}
+
+int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens) {
+ if (out_n_tokens) *out_n_tokens = 0;
+ if (!tok || !ws || !text || text[0] == '\0') return NULL;
+
+ size_t len = strlen(text);
+ size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8;
+ if (cap_sz > (size_t)INT_MAX) return NULL;
+ int cap = (int)cap_sz;
+
+ if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0)
+ return NULL;
+
+ int n_ids = 0;
+ int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap,
+ &n_ids);
+ if (rc == -2) {
+ if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0)
+ return NULL;
+ rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap,
+ &n_ids);
+ }
+ if (rc != 0 || n_ids <= 0) return NULL;
+
+ int *ids = (int *)malloc((size_t)n_ids * sizeof(int));
+ if (!ids) return NULL;
+ memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int));
+ if (out_n_tokens) *out_n_tokens = n_ids;
+ return ids;
+}
+
void qwen_tokenizer_free(qwen_tokenizer_t *tok) {
if (!tok) return;
diff --git a/qwen_asr_tokenizer.h b/qwen_asr_tokenizer.h
index 0cea192..742d0d1 100644
--- a/qwen_asr_tokenizer.h
+++ b/qwen_asr_tokenizer.h
@@ -21,6 +21,8 @@ typedef struct {
int merge_map_cap;
} qwen_tokenizer_t;
+typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t;
+
/* Load tokenizer from vocab.json in model directory */
qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path);
@@ -32,6 +34,27 @@ const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id);
* Returns NULL on error (and sets *out_n_tokens to 0). */
int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
+/* Reusable scratch for allocation-light repeated encoding. */
+qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void);
+void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws);
+
+/* Encode into caller-provided out_ids[0..out_cap-1].
+ * Returns 0 on success, -1 on error, -2 if out_cap is too small.
+ * In all cases *out_n_tokens is set to the number of tokens needed/emitted
+ * when out_n_tokens is non-NULL. */
+int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_ids, int out_cap,
+ int *out_n_tokens);
+
+/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and
+ * performs only the final returned int[] allocation. */
+int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
/* Free tokenizer */
void qwen_tokenizer_free(qwen_tokenizer_t *tok);