diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
| commit | 37cbaef9ceb828c667da8f398027ba5415ee7475 (patch) | |
| tree | 6103e92a9030766e78a35df00f7a1155d0d0ecf5 /liblangdet/tokenize.c | |
| parent | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff) | |
| download | langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip | |
Rename CPU library API and sources to langdet
Diffstat (limited to 'liblangdet/tokenize.c')
| -rw-r--r-- | liblangdet/tokenize.c | 85 |
1 files changed, 85 insertions, 0 deletions
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c new file mode 100644 index 0000000..5e26a18 --- /dev/null +++ b/liblangdet/tokenize.c @@ -0,0 +1,85 @@ +#include "langdet.h" +#include "sentencepiece.h" + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +struct langdet_tok { + tok_spm_t *spm; + tok_spm_workspace_t *workspace; + int cls_id; + int sep_id; +}; + +static void set_err(char *err, size_t errlen, const char *msg) { + if (err && errlen) + snprintf(err, errlen, "%s", msg); +} + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) { + if (!model_dir) { + set_err(err, errlen, "model_dir is NULL"); + return NULL; + } + langdet_tok_t *t = calloc(1, sizeof(*t)); + if (!t) { + set_err(err, errlen, "out of memory"); + return NULL; + } + t->spm = tok_spm_load(model_dir); + if (!t->spm) { + set_err(err, errlen, "failed to load SentencePiece tokenizer"); + goto fail; + } + t->workspace = tok_spm_workspace_new(); + if (!t->workspace) { + set_err(err, errlen, "out of memory"); + goto fail; + } + t->cls_id = tok_spm_token_id(t->spm, "<s>"); + t->sep_id = tok_spm_token_id(t->spm, "</s>"); + if (t->cls_id < 0 || t->sep_id < 0) { + set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); + goto fail; + } + return t; + +fail: + langdet_tok_free(t); + return NULL; +} + +void langdet_tok_free(langdet_tok_t *t) { + if (!t) + return; + tok_spm_workspace_free(t->workspace); + tok_spm_free(t->spm); + free(t); +} + +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { + if (n_out) + *n_out = 0; + if (!t || !text) + return NULL; + int n = 0; + int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); + if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { + free(core); + return NULL; + } + /* Empty text still produces the two special tokens. */ + int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); + if (!ids) { + free(core); + return NULL; + } + memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); + ids[0] = t->cls_id; + ids[n + 1] = t->sep_id; + if (n_out) + *n_out = n + 2; + return ids; +} |
