diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 15:49:59 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 7fe02715aac68b958250b43f9d48d641af0a045c (patch) | |
| tree | 7bc866f87999991ef0a3ffa0dd36ce91444e5172 /qwen_tokenizer.h | |
| parent | e8259865f41ceb50dd28b267186d11c62a465135 (diff) | |
| download | ffwd-7fe02715aac68b958250b43f9d48d641af0a045c.tar.gz ffwd-7fe02715aac68b958250b43f9d48d641af0a045c.zip | |
build: export only the public API from the shared library
Diffstat (limited to 'qwen_tokenizer.h')
| -rw-r--r-- | qwen_tokenizer.h | 29 |
1 files changed, 20 insertions, 9 deletions
diff --git a/qwen_tokenizer.h b/qwen_tokenizer.h index c698ea3..4c3834c 100644 --- a/qwen_tokenizer.h +++ b/qwen_tokenizer.h @@ -9,6 +9,17 @@ #ifndef QWEN_TOKENIZER_H #define QWEN_TOKENIZER_H +/* Public-API export annotation. The libraries are built with + * -fvisibility=hidden, so only declarations carrying PPLX_API are exported + * from the shared library; everything else stays internal. */ +#ifndef PPLX_API +#if defined(__GNUC__) +#define PPLX_API __attribute__((visibility("default"))) +#else +#define PPLX_API +#endif +#endif + typedef struct { char **id_to_text; /* [vocab_size] decoded text strings */ char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ @@ -32,29 +43,29 @@ typedef struct { typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; /* Load tokenizer from vocab.json in model directory */ -qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); +PPLX_API qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); /* Decode a single token ID to text. Returns pointer to internal string. */ -const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); +PPLX_API const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); /* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>"). * Returns -1 when the token is not in the vocab. */ -int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token); +PPLX_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token); /* Encode UTF-8 text into token IDs using BPE. * Returns malloc'd array of token IDs and sets *out_n_tokens. * Returns NULL on error (and sets *out_n_tokens to 0). */ -int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); +PPLX_API int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); /* Reusable scratch for allocation-light repeated encoding. */ -qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); -void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); +PPLX_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); +PPLX_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); /* Encode into caller-provided out_ids[0..out_cap-1]. * Returns 0 on success, -1 on error, -2 if out_cap is too small. * In all cases *out_n_tokens is set to the number of tokens needed/emitted * when out_n_tokens is non-NULL. */ -int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, +PPLX_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, int *out_ids, int out_cap, @@ -62,12 +73,12 @@ int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, /* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and * performs only the final returned int[] allocation. */ -int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, +PPLX_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, int *out_n_tokens); /* Free tokenizer */ -void qwen_tokenizer_free(qwen_tokenizer_t *tok); +PPLX_API void qwen_tokenizer_free(qwen_tokenizer_t *tok); #endif /* QWEN_TOKENIZER_H */ |
