From 4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 10 Jun 2026 13:04:50 +0200 Subject: server: resolve contextual separator from vocab, test contextual path --- qwen_tokenizer.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'qwen_tokenizer.h') diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h index e4b1195..c698ea3 100644 --- a/qwen_tokenizer.h +++ b/qwen_tokenizer.h @@ -37,6 +37,10 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); /* Decode a single token ID to text. Returns pointer to internal string. */ const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); +/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>"). + * Returns -1 when the token is not in the vocab. */ +int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token); + /* Encode UTF-8 text into token IDs using BPE. * Returns malloc'd array of token IDs and sets *out_n_tokens. * Returns NULL on error (and sets *out_n_tokens to 0). */ -- cgit v1.3.1