diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /src/tokenizer_bpe.h | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'src/tokenizer_bpe.h')
| -rw-r--r-- | src/tokenizer_bpe.h | 86 |
1 files changed, 86 insertions, 0 deletions
diff --git a/src/tokenizer_bpe.h b/src/tokenizer_bpe.h new file mode 100644 index 0000000..8d13d47 --- /dev/null +++ b/src/tokenizer_bpe.h @@ -0,0 +1,86 @@ +/* + * embed_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * + * Supports: + * - decode token ID -> UTF-8 text + * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) + */ + +#ifndef EMBED_TOKENIZER_H +#define EMBED_TOKENIZER_H + +/* Public-API export annotation. The libraries are built with + * -fvisibility=hidden, so only declarations carrying EMBED_API are exported + * from the shared library; everything else stays internal. */ +#ifndef EMBED_API +#if defined(__GNUC__) +#define EMBED_API __attribute__((visibility("default"))) +#else +#define EMBED_API +#endif +#endif + +typedef struct { + char **id_to_text; /* [vocab_size] decoded text strings */ + char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ + int vocab_size; + + /* Internal hash maps (opaque to callers) */ + void *vocab_map; + int vocab_map_cap; + void *merge_map; + int merge_map_cap; + /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id). + * NULL when the vocab could not resolve every merge; encoding then falls + * back to the string-based merge loop. */ + void *int_merges; + int int_merges_cap; + /* Byte-level unicode codepoint -> single-character token id (512 slots), + * or NULL when any byte token is missing from the vocab. */ + int *cp_to_id; +} embed_tokenizer_t; + +typedef struct embed_tokenizer_workspace embed_tokenizer_workspace_t; + +/* Load tokenizer from vocab.json in model directory */ +EMBED_API embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path); + +/* Decode a single token ID to text. Returns pointer to internal string. */ +EMBED_API const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id); + +/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>"). + * Returns -1 when the token is not in the vocab. */ +EMBED_API int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token); + +/* Encode UTF-8 text into token IDs using BPE. + * Returns malloc'd array of token IDs and sets *out_n_tokens. + * Returns NULL on error (and sets *out_n_tokens to 0). */ +EMBED_API int * +embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens); + +/* Reusable scratch for allocation-light repeated encoding. */ +EMBED_API embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void); +EMBED_API void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws); + +/* Encode into caller-provided out_ids[0..out_cap-1]. + * Returns 0 on success, -1 on error, -2 if out_cap is too small. + * In all cases *out_n_tokens is set to the number of tokens needed/emitted + * when out_n_tokens is non-NULL. */ +EMBED_API int embed_tokenizer_encode_into(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); + +/* Same tokenization as embed_tokenizer_encode(), but reuses ws internally and + * performs only the final returned int[] allocation. */ +EMBED_API int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, + const char *text, + int *out_n_tokens); + +/* Free tokenizer */ +EMBED_API void embed_tokenizer_free(embed_tokenizer_t *tok); + +#endif /* EMBED_TOKENIZER_H */ |
