From 4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 10 Jun 2026 13:04:50 +0200 Subject: server: resolve contextual separator from vocab, test contextual path --- qwen_tokenizer.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'qwen_tokenizer.c') diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c index 67dcaf7..ee95326 100644 --- a/qwen_tokenizer.c +++ b/qwen_tokenizer.c @@ -1644,6 +1644,12 @@ const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) { return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; } +int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token) { + if (!tok || !token || !tok->vocab_map) return -1; + return map_get((const str_int_entry_t *)tok->vocab_map, + tok->vocab_map_cap, token); +} + int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !text || text[0] == '\0') return NULL; -- cgit v1.3.1