/*
* XLM-R SentencePiece Unigram tokenizer.
* Encodes text using the checkpoint's precompiled normalization table.
* The caller adds the leading and trailing tokens.
*/
#ifndef TOK_SPM_H
#define TOK_SPM_H
#include
#include
enum {
TOK_XLMR_BOS_ID = 0,
TOK_XLMR_EOS_ID = 2,
TOK_XLMR_UNK_ID = 3,
};
typedef struct {
char **id_to_piece; /* [piece_count] raw SentencePiece ids */
uint16_t *piece_lens; /* byte length for id_to_piece entries */
float *scores; /* SentencePiece score per raw id */
int piece_count; /* raw SentencePiece vocabulary size */
int max_piece_len; /* longest normal piece in bytes */
double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
void *piece_map; /* piece bytes -> raw SentencePiece id */
int piece_map_cap;
uint32_t *xcda;
size_t xcda_count;
char *prefix_replacements;
size_t prefix_replacements_len;
} tok_spm_t;
typedef struct tok_spm_workspace tok_spm_workspace_t;
/* Load model_dir/sentencepiece.bpe.model with XLM-R vocabulary and normalization.
* Return NULL on missing or unsupported model, malformed protobuf, or OOM. */
tok_spm_t *tok_spm_load(const char *model_dir);
tok_spm_workspace_t *tok_spm_workspace_new(void);
void tok_spm_workspace_free(tok_spm_workspace_t *ws);
/* Return malloc-owned model-space token IDs, reusing workspace scratch.
* Empty text returns NULL with a zero count. */
int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
tok_spm_workspace_t *ws,
const char *text,
int *out_n_tokens);
void tok_spm_free(tok_spm_t *tok);
#endif /* TOK_SPM_H */