aboutsummaryrefslogtreecommitdiff
path: root/qwen_tokenizer.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
commitc4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch)
tree81ce5675ce886aad0309a41f4d6a696826e0a061 /qwen_tokenizer.h
parentb377ba8656bfb807ed8dda6c9f26942b50ce626d (diff)
downloadffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz
ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip
chore: run clang-format
Diffstat (limited to 'qwen_tokenizer.h')
-rw-r--r--qwen_tokenizer.h22
1 files changed, 12 insertions, 10 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h
index d7670b7..f7a3941 100644
--- a/qwen_tokenizer.h
+++ b/qwen_tokenizer.h
@@ -21,8 +21,8 @@
#endif
typedef struct {
- char **id_to_text; /* [vocab_size] decoded text strings */
- char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
+ char **id_to_text; /* [vocab_size] decoded text strings */
+ char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
int vocab_size;
/* Internal hash maps (opaque to callers) */
@@ -55,7 +55,8 @@ EMBED_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *t
/* Encode UTF-8 text into token IDs using BPE.
* Returns malloc'd array of token IDs and sets *out_n_tokens.
* Returns NULL on error (and sets *out_n_tokens to 0). */
-EMBED_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
+EMBED_API int *
+qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
/* Reusable scratch for allocation-light repeated encoding. */
EMBED_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void);
@@ -66,17 +67,18 @@ EMBED_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws);
* In all cases *out_n_tokens is set to the number of tokens needed/emitted
* when out_n_tokens is non-NULL. */
EMBED_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids, int out_cap,
- int *out_n_tokens);
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and
* performs only the final returned int[] allocation. */
EMBED_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
/* Free tokenizer */
EMBED_API void qwen_tokenizer_free(qwen_tokenizer_t *tok);