aboutsummaryrefslogtreecommitdiff
path: root/qwen_tokenizer.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 09:23:02 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit490a72cf64f191becf4a6f01f06a06899f6a5e78 (patch)
treed56b1d4e2c92450eb5a0ebe6fea766878cdb1fd4 /qwen_tokenizer.h
parent3999ed4fba0a0cf7c4b9ae941d387682839640e6 (diff)
downloadffwd-490a72cf64f191becf4a6f01f06a06899f6a5e78.tar.gz
ffwd-490a72cf64f191becf4a6f01f06a06899f6a5e78.zip
tokenizer: integer-pair BPE merging with linked-list symbols
Diffstat (limited to 'qwen_tokenizer.h')
-rw-r--r--qwen_tokenizer.h8
1 files changed, 8 insertions, 0 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h
index 5d80237..e4b1195 100644
--- a/qwen_tokenizer.h
+++ b/qwen_tokenizer.h
@@ -19,6 +19,14 @@ typedef struct {
int vocab_map_cap;
void *merge_map;
int merge_map_cap;
+ /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id).
+ * NULL when the vocab could not resolve every merge; encoding then falls
+ * back to the string-based merge loop. */
+ void *int_merges;
+ int int_merges_cap;
+ /* Byte-level unicode codepoint -> single-character token id (512 slots),
+ * or NULL when any byte token is missing from the vocab. */
+ int *cp_to_id;
} qwen_tokenizer_t;
typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t;