diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 09:23:02 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 490a72cf64f191becf4a6f01f06a06899f6a5e78 (patch) | |
| tree | d56b1d4e2c92450eb5a0ebe6fea766878cdb1fd4 /qwen_tokenizer.h | |
| parent | 3999ed4fba0a0cf7c4b9ae941d387682839640e6 (diff) | |
| download | ffwd-490a72cf64f191becf4a6f01f06a06899f6a5e78.tar.gz ffwd-490a72cf64f191becf4a6f01f06a06899f6a5e78.zip | |
tokenizer: integer-pair BPE merging with linked-list symbols
Diffstat (limited to 'qwen_tokenizer.h')
| -rw-r--r-- | qwen_tokenizer.h | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h index 5d80237..e4b1195 100644 --- a/qwen_tokenizer.h +++ b/qwen_tokenizer.h @@ -19,6 +19,14 @@ typedef struct { int vocab_map_cap; void *merge_map; int merge_map_cap; + /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id). + * NULL when the vocab could not resolve every merge; encoding then falls + * back to the string-based merge loop. */ + void *int_merges; + int int_merges_cap; + /* Byte-level unicode codepoint -> single-character token id (512 slots), + * or NULL when any byte token is missing from the vocab. */ + int *cp_to_id; } qwen_tokenizer_t; typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; |
