#include "encoder.h" #include "sentencepiece.h" #include #include #include #include struct langdet_tok { tok_spm_t *spm; tok_spm_workspace_t *workspace; }; static void set_err(char *err, size_t errlen, const char *msg) { if (err && errlen) snprintf(err, errlen, "%s", msg); } langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) { if (!model_dir) { set_err(err, errlen, "model_dir is NULL"); return NULL; } langdet_tok_t *t = calloc(1, sizeof(*t)); if (!t) { set_err(err, errlen, "out of memory"); return NULL; } t->spm = tok_spm_load(model_dir); if (!t->spm) { set_err(err, errlen, "failed to load SentencePiece tokenizer"); goto fail; } t->workspace = tok_spm_workspace_new(); if (!t->workspace) { set_err(err, errlen, "out of memory"); goto fail; } return t; fail: langdet_tok_free(t); return NULL; } void langdet_tok_free(langdet_tok_t *t) { if (!t) return; tok_spm_workspace_free(t->workspace); tok_spm_free(t->spm); free(t); } int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { if (n_out) *n_out = 0; if (!t || !text) return NULL; int n = 0; int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { free(core); return NULL; } /* Empty text still produces the two special tokens. */ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); if (!ids) { free(core); return NULL; } memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); ids[0] = TOK_XLMR_BOS_ID; ids[n + 1] = TOK_XLMR_EOS_ID; if (n_out) *n_out = n + 2; return ids; }