aboutsummaryrefslogtreecommitdiff
path: root/qwen_tokenizer.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 13:04:50 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3 (patch)
tree2e441978cc9badf091976aa53bc64297432e862a /qwen_tokenizer.h
parent3947e97fbf2a0b79c500ce925bcc9fe15f89899e (diff)
downloadffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.tar.gz
ffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.zip
server: resolve contextual separator from vocab, test contextual path
Diffstat (limited to 'qwen_tokenizer.h')
-rw-r--r--qwen_tokenizer.h4
1 files changed, 4 insertions, 0 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h
index e4b1195..c698ea3 100644
--- a/qwen_tokenizer.h
+++ b/qwen_tokenizer.h
@@ -37,6 +37,10 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path);
/* Decode a single token ID to text. Returns pointer to internal string. */
const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id);
+/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>").
+ * Returns -1 when the token is not in the vocab. */
+int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token);
+
/* Encode UTF-8 text into token IDs using BPE.
* Returns malloc'd array of token IDs and sets *out_n_tokens.
* Returns NULL on error (and sets *out_n_tokens to 0). */