/*
* tokenizer_sentencepiece.h - SentencePiece tokenizer for XLM-R style models.
*
* Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`,
* or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R
* and multilingual E5. It encodes text only; frontends add the model's special
* token layout, e.g. ... for XLM-R.
*/
#ifndef SENTENCEPIECE_TOKENIZER_H
#define SENTENCEPIECE_TOKENIZER_H
#include
#include
#ifndef EMBED_API
#if defined(__GNUC__)
#define EMBED_API __attribute__((visibility("default")))
#else
#define EMBED_API
#endif
#endif
typedef enum {
SENTENCEPIECE_IDENTITY = 0,
SENTENCEPIECE_XLM_ROBERTA = 1,
} sentencepiece_id_map_t;
typedef struct {
char **id_to_piece; /* [piece_count] raw SentencePiece ids */
uint16_t *piece_lens; /* byte length for id_to_piece entries */
float *scores; /* SentencePiece score per raw id */
uint8_t *types; /* SentencePiece piece type per raw id */
int piece_count; /* raw SentencePiece vocabulary size */
int vocab_size; /* model embedding vocab size after id mapping */
int unk_id; /* model-space unknown-token id */
int bos_id; /* model-space beginning-token id, or -1 */
int eos_id; /* model-space end-token id, or -1 */
int pad_id; /* model-space pad-token id, or -1 */
int mask_id; /* model-space mask-token id, or -1 */
int max_piece_len; /* max normal/user/unused piece byte length */
double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
int add_dummy_prefix;
int remove_extra_whitespaces;
int escape_whitespaces;
sentencepiece_id_map_t id_map;
void *piece_map; /* piece bytes -> raw SentencePiece id */
int piece_map_cap;
uint32_t *xcda;
size_t xcda_count;
char *prefix_replacements;
size_t prefix_replacements_len;
} sentencepiece_tokenizer_t;
typedef struct sentencepiece_workspace sentencepiece_workspace_t;
/* Load the first supported SentencePiece model found in model_dir. Returns NULL
* on missing model file, unsupported model type, malformed protobuf, or OOM. */
EMBED_API sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir);
/* Decode one model-space id to an internal token string, or NULL when invalid. */
EMBED_API const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id);
/* Resolve a named token (e.g. "", "", "", "") to a
* model-space id, or -1 when absent. */
EMBED_API int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok,
const char *token);
/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns
* NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an
* error. */
EMBED_API int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
const char *text,
int *out_n_tokens);
EMBED_API sentencepiece_workspace_t *sentencepiece_workspace_new(void);
EMBED_API void sentencepiece_workspace_free(sentencepiece_workspace_t *ws);
/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if
* out_cap is too small. *out_n_tokens receives the emitted/needed token count. */
EMBED_API int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
sentencepiece_workspace_t *ws,
const char *text,
int *out_ids,
int out_cap,
int *out_n_tokens);
/* Same tokenization as sentencepiece_tokenizer_encode(), reusing ws internally
* and performing only the final returned int[] allocation. */
EMBED_API int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
sentencepiece_workspace_t *ws,
const char *text,
int *out_n_tokens);
EMBED_API void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok);
#endif /* SENTENCEPIECE_TOKENIZER_H */