/* * XLM-R SentencePiece Unigram tokenizer. * Encodes text using the checkpoint's precompiled normalization table. * The caller adds the leading and trailing tokens. */ #ifndef TOK_SPM_H #define TOK_SPM_H #include #include enum { TOK_XLMR_BOS_ID = 0, TOK_XLMR_EOS_ID = 2, TOK_XLMR_UNK_ID = 3, }; typedef struct { char **id_to_piece; /* [piece_count] raw SentencePiece ids */ uint16_t *piece_lens; /* byte length for id_to_piece entries */ float *scores; /* SentencePiece score per raw id */ int piece_count; /* raw SentencePiece vocabulary size */ int max_piece_len; /* longest normal piece in bytes */ double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ void *piece_map; /* piece bytes -> raw SentencePiece id */ int piece_map_cap; uint32_t *xcda; size_t xcda_count; char *prefix_replacements; size_t prefix_replacements_len; } tok_spm_t; typedef struct tok_spm_workspace tok_spm_workspace_t; /* Load model_dir/sentencepiece.bpe.model with XLM-R vocabulary and normalization. * Return NULL on missing or unsupported model, malformed protobuf, or OOM. */ tok_spm_t *tok_spm_load(const char *model_dir); tok_spm_workspace_t *tok_spm_workspace_new(void); void tok_spm_workspace_free(tok_spm_workspace_t *ws); /* Return malloc-owned model-space token IDs, reusing workspace scratch. * Empty text returns NULL with a zero count. */ int *tok_spm_encode_with_workspace(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text, int *out_n_tokens); void tok_spm_free(tok_spm_t *tok); #endif /* TOK_SPM_H */