From 37cbaef9ceb828c667da8f398027ba5415ee7475 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 29 Sep 2026 13:11:00 +0200 Subject: Rename CPU library API and sources to langdet --- liblangdet/tokenize.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 liblangdet/tokenize.c (limited to 'liblangdet/tokenize.c') diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c new file mode 100644 index 0000000..5e26a18 --- /dev/null +++ b/liblangdet/tokenize.c @@ -0,0 +1,85 @@ +#include "langdet.h" +#include "sentencepiece.h" + +#include +#include +#include +#include + +struct langdet_tok { + tok_spm_t *spm; + tok_spm_workspace_t *workspace; + int cls_id; + int sep_id; +}; + +static void set_err(char *err, size_t errlen, const char *msg) { + if (err && errlen) + snprintf(err, errlen, "%s", msg); +} + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) { + if (!model_dir) { + set_err(err, errlen, "model_dir is NULL"); + return NULL; + } + langdet_tok_t *t = calloc(1, sizeof(*t)); + if (!t) { + set_err(err, errlen, "out of memory"); + return NULL; + } + t->spm = tok_spm_load(model_dir); + if (!t->spm) { + set_err(err, errlen, "failed to load SentencePiece tokenizer"); + goto fail; + } + t->workspace = tok_spm_workspace_new(); + if (!t->workspace) { + set_err(err, errlen, "out of memory"); + goto fail; + } + t->cls_id = tok_spm_token_id(t->spm, ""); + t->sep_id = tok_spm_token_id(t->spm, ""); + if (t->cls_id < 0 || t->sep_id < 0) { + set_err(err, errlen, "SentencePiece vocab missing /"); + goto fail; + } + return t; + +fail: + langdet_tok_free(t); + return NULL; +} + +void langdet_tok_free(langdet_tok_t *t) { + if (!t) + return; + tok_spm_workspace_free(t->workspace); + tok_spm_free(t->spm); + free(t); +} + +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { + if (n_out) + *n_out = 0; + if (!t || !text) + return NULL; + int n = 0; + int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); + if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { + free(core); + return NULL; + } + /* Empty text still produces the two special tokens. */ + int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); + if (!ids) { + free(core); + return NULL; + } + memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); + ids[0] = t->cls_id; + ids[n + 1] = t->sep_id; + if (n_out) + *n_out = n + 2; + return ids; +} -- cgit v1.3.1