aboutsummaryrefslogtreecommitdiff
path: root/qwen_tokenizer.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 13:04:50 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3 (patch)
tree2e441978cc9badf091976aa53bc64297432e862a /qwen_tokenizer.c
parent3947e97fbf2a0b79c500ce925bcc9fe15f89899e (diff)
downloadffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.tar.gz
ffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.zip
server: resolve contextual separator from vocab, test contextual path
Diffstat (limited to 'qwen_tokenizer.c')
-rw-r--r--qwen_tokenizer.c6
1 files changed, 6 insertions, 0 deletions
diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c
index 67dcaf7..ee95326 100644
--- a/qwen_tokenizer.c
+++ b/qwen_tokenizer.c
@@ -1644,6 +1644,12 @@ const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) {
return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : "";
}
+int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token) {
+ if (!tok || !token || !tok->vocab_map) return -1;
+ return map_get((const str_int_entry_t *)tok->vocab_map,
+ tok->vocab_map_cap, token);
+}
+
int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) {
if (out_n_tokens) *out_n_tokens = 0;
if (!tok || !text || text[0] == '\0') return NULL;