aboutsummaryrefslogtreecommitdiff
path: root/qwen_tokenizer.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-11 23:40:01 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:10 +0200
commit978f4f7969190265e127e8a3d704bb173fbd171c (patch)
tree42f3a89e1ecb481ef0e5196a775ea37edb4b98f7 /qwen_tokenizer.h
parentd9ea2cea7dfa1eb25526a3e9b729beda5ccb006f (diff)
downloadffwd-978f4f7969190265e127e8a3d704bb173fbd171c.tar.gz
ffwd-978f4f7969190265e127e8a3d704bb173fbd171c.zip
chore: rename project
Diffstat (limited to 'qwen_tokenizer.h')
-rw-r--r--qwen_tokenizer.h26
1 files changed, 13 insertions, 13 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h
index 4c3834c..d7670b7 100644
--- a/qwen_tokenizer.h
+++ b/qwen_tokenizer.h
@@ -10,13 +10,13 @@
#define QWEN_TOKENIZER_H
/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying PPLX_API are exported
+ * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
* from the shared library; everything else stays internal. */
-#ifndef PPLX_API
+#ifndef EMBED_API
#if defined(__GNUC__)
-#define PPLX_API __attribute__((visibility("default")))
+#define EMBED_API __attribute__((visibility("default")))
#else
-#define PPLX_API
+#define EMBED_API
#endif
#endif
@@ -43,29 +43,29 @@ typedef struct {
typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t;
/* Load tokenizer from vocab.json in model directory */
-PPLX_API qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path);
+EMBED_API qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path);
/* Decode a single token ID to text. Returns pointer to internal string. */
-PPLX_API const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id);
+EMBED_API const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id);
/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>").
* Returns -1 when the token is not in the vocab. */
-PPLX_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token);
+EMBED_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token);
/* Encode UTF-8 text into token IDs using BPE.
* Returns malloc'd array of token IDs and sets *out_n_tokens.
* Returns NULL on error (and sets *out_n_tokens to 0). */
-PPLX_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
+EMBED_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
/* Reusable scratch for allocation-light repeated encoding. */
-PPLX_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void);
-PPLX_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws);
+EMBED_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void);
+EMBED_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws);
/* Encode into caller-provided out_ids[0..out_cap-1].
* Returns 0 on success, -1 on error, -2 if out_cap is too small.
* In all cases *out_n_tokens is set to the number of tokens needed/emitted
* when out_n_tokens is non-NULL. */
-PPLX_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
+EMBED_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *text,
int *out_ids, int out_cap,
@@ -73,12 +73,12 @@ PPLX_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and
* performs only the final returned int[] allocation. */
-PPLX_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
+EMBED_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *text,
int *out_n_tokens);
/* Free tokenizer */
-PPLX_API void qwen_tokenizer_free(qwen_tokenizer_t *tok);
+EMBED_API void qwen_tokenizer_free(qwen_tokenizer_t *tok);
#endif /* QWEN_TOKENIZER_H */