diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /qwen_tokenizer.h | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'qwen_tokenizer.h')
| -rw-r--r-- | qwen_tokenizer.h | 86 |
1 files changed, 0 insertions, 86 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h deleted file mode 100644 index f7a3941..0000000 --- a/qwen_tokenizer.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * qwen_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) - * - * Supports: - * - decode token ID -> UTF-8 text - * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) - */ - -#ifndef QWEN_TOKENIZER_H -#define QWEN_TOKENIZER_H - -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying EMBED_API are exported - * from the shared library; everything else stays internal. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - -typedef struct { - char **id_to_text; /* [vocab_size] decoded text strings */ - char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ - int vocab_size; - - /* Internal hash maps (opaque to callers) */ - void *vocab_map; - int vocab_map_cap; - void *merge_map; - int merge_map_cap; - /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id). - * NULL when the vocab could not resolve every merge; encoding then falls - * back to the string-based merge loop. */ - void *int_merges; - int int_merges_cap; - /* Byte-level unicode codepoint -> single-character token id (512 slots), - * or NULL when any byte token is missing from the vocab. */ - int *cp_to_id; -} qwen_tokenizer_t; - -typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; - -/* Load tokenizer from vocab.json in model directory */ -EMBED_API qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); - -/* Decode a single token ID to text. Returns pointer to internal string. */ -EMBED_API const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); - -/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>"). - * Returns -1 when the token is not in the vocab. */ -EMBED_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token); - -/* Encode UTF-8 text into token IDs using BPE. - * Returns malloc'd array of token IDs and sets *out_n_tokens. - * Returns NULL on error (and sets *out_n_tokens to 0). */ -EMBED_API int * -qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); - -/* Reusable scratch for allocation-light repeated encoding. */ -EMBED_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); -EMBED_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); - -/* Encode into caller-provided out_ids[0..out_cap-1]. - * Returns 0 on success, -1 on error, -2 if out_cap is too small. - * In all cases *out_n_tokens is set to the number of tokens needed/emitted - * when out_n_tokens is non-NULL. */ -EMBED_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens); - -/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and - * performs only the final returned int[] allocation. */ -EMBED_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens); - -/* Free tokenizer */ -EMBED_API void qwen_tokenizer_free(qwen_tokenizer_t *tok); - -#endif /* QWEN_TOKENIZER_H */ |
