From 490a72cf64f191becf4a6f01f06a06899f6a5e78 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 10 Jun 2026 09:23:02 +0200 Subject: tokenizer: integer-pair BPE merging with linked-list symbols --- qwen_tokenizer.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'qwen_tokenizer.h') diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h index 5d80237..e4b1195 100644 --- a/qwen_tokenizer.h +++ b/qwen_tokenizer.h @@ -19,6 +19,14 @@ typedef struct { int vocab_map_cap; void *merge_map; int merge_map_cap; + /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id). + * NULL when the vocab could not resolve every merge; encoding then falls + * back to the string-based merge loop. */ + void *int_merges; + int int_merges_cap; + /* Byte-level unicode codepoint -> single-character token id (512 slots), + * or NULL when any byte token is missing from the vocab. */ + int *cp_to_id; } qwen_tokenizer_t; typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; -- cgit v1.3.1