diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
| commit | c4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch) | |
| tree | 81ce5675ce886aad0309a41f4d6a696826e0a061 /qwen_tokenizer.h | |
| parent | b377ba8656bfb807ed8dda6c9f26942b50ce626d (diff) | |
| download | ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip | |
chore: run clang-format
Diffstat (limited to 'qwen_tokenizer.h')
| -rw-r--r-- | qwen_tokenizer.h | 22 |
1 files changed, 12 insertions, 10 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h index d7670b7..f7a3941 100644 --- a/qwen_tokenizer.h +++ b/qwen_tokenizer.h @@ -21,8 +21,8 @@ #endif typedef struct { - char **id_to_text; /* [vocab_size] decoded text strings */ - char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ + char **id_to_text; /* [vocab_size] decoded text strings */ + char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ int vocab_size; /* Internal hash maps (opaque to callers) */ @@ -55,7 +55,8 @@ EMBED_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *t /* Encode UTF-8 text into token IDs using BPE. * Returns malloc'd array of token IDs and sets *out_n_tokens. * Returns NULL on error (and sets *out_n_tokens to 0). */ -EMBED_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); +EMBED_API int * +qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); /* Reusable scratch for allocation-light repeated encoding. */ EMBED_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); @@ -66,17 +67,18 @@ EMBED_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); * In all cases *out_n_tokens is set to the number of tokens needed/emitted * when out_n_tokens is non-NULL. */ EMBED_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, int out_cap, - int *out_n_tokens); + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); /* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and * performs only the final returned int[] allocation. */ EMBED_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens); + qwen_tokenizer_workspace_t *ws, + const char *text, + int *out_n_tokens); /* Free tokenizer */ EMBED_API void qwen_tokenizer_free(qwen_tokenizer_t *tok); |
