aboutsummaryrefslogtreecommitdiff
path: root/qwen_tokenizer.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 15:49:59 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit7fe02715aac68b958250b43f9d48d641af0a045c (patch)
tree7bc866f87999991ef0a3ffa0dd36ce91444e5172 /qwen_tokenizer.h
parente8259865f41ceb50dd28b267186d11c62a465135 (diff)
downloadffwd-7fe02715aac68b958250b43f9d48d641af0a045c.tar.gz
ffwd-7fe02715aac68b958250b43f9d48d641af0a045c.zip
build: export only the public API from the shared library
Diffstat (limited to 'qwen_tokenizer.h')
-rw-r--r--qwen_tokenizer.h29
1 files changed, 20 insertions, 9 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h
index c698ea3..4c3834c 100644
--- a/qwen_tokenizer.h
+++ b/qwen_tokenizer.h
@@ -9,6 +9,17 @@
#ifndef QWEN_TOKENIZER_H
#define QWEN_TOKENIZER_H
+/* Public-API export annotation. The libraries are built with
+ * -fvisibility=hidden, so only declarations carrying PPLX_API are exported
+ * from the shared library; everything else stays internal. */
+#ifndef PPLX_API
+#if defined(__GNUC__)
+#define PPLX_API __attribute__((visibility("default")))
+#else
+#define PPLX_API
+#endif
+#endif
+
typedef struct {
char **id_to_text; /* [vocab_size] decoded text strings */
char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
@@ -32,29 +43,29 @@ typedef struct {
typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t;
/* Load tokenizer from vocab.json in model directory */
-qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path);
+PPLX_API qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path);
/* Decode a single token ID to text. Returns pointer to internal string. */
-const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id);
+PPLX_API const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id);
/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>").
* Returns -1 when the token is not in the vocab. */
-int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token);
+PPLX_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token);
/* Encode UTF-8 text into token IDs using BPE.
* Returns malloc'd array of token IDs and sets *out_n_tokens.
* Returns NULL on error (and sets *out_n_tokens to 0). */
-int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
+PPLX_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens);
/* Reusable scratch for allocation-light repeated encoding. */
-qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void);
-void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws);
+PPLX_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void);
+PPLX_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws);
/* Encode into caller-provided out_ids[0..out_cap-1].
* Returns 0 on success, -1 on error, -2 if out_cap is too small.
* In all cases *out_n_tokens is set to the number of tokens needed/emitted
* when out_n_tokens is non-NULL. */
-int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
+PPLX_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *text,
int *out_ids, int out_cap,
@@ -62,12 +73,12 @@ int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok,
/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and
* performs only the final returned int[] allocation. */
-int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
+PPLX_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
const char *text,
int *out_n_tokens);
/* Free tokenizer */
-void qwen_tokenizer_free(qwen_tokenizer_t *tok);
+PPLX_API void qwen_tokenizer_free(qwen_tokenizer_t *tok);
#endif /* QWEN_TOKENIZER_H */