aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/tokenize.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
commit37cbaef9ceb828c667da8f398027ba5415ee7475 (patch)
tree6103e92a9030766e78a35df00f7a1155d0d0ecf5 /liblangdet/tokenize.c
parentcf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff)
downloadlangdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz
langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip
Rename CPU library API and sources to langdet
Diffstat (limited to 'liblangdet/tokenize.c')
-rw-r--r--liblangdet/tokenize.c85
1 files changed, 85 insertions, 0 deletions
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
new file mode 100644
index 0000000..5e26a18
--- /dev/null
+++ b/liblangdet/tokenize.c
@@ -0,0 +1,85 @@
+#include "langdet.h"
+#include "sentencepiece.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct langdet_tok {
+ tok_spm_t *spm;
+ tok_spm_workspace_t *workspace;
+ int cls_id;
+ int sep_id;
+};
+
+static void set_err(char *err, size_t errlen, const char *msg) {
+ if (err && errlen)
+ snprintf(err, errlen, "%s", msg);
+}
+
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) {
+ if (!model_dir) {
+ set_err(err, errlen, "model_dir is NULL");
+ return NULL;
+ }
+ langdet_tok_t *t = calloc(1, sizeof(*t));
+ if (!t) {
+ set_err(err, errlen, "out of memory");
+ return NULL;
+ }
+ t->spm = tok_spm_load(model_dir);
+ if (!t->spm) {
+ set_err(err, errlen, "failed to load SentencePiece tokenizer");
+ goto fail;
+ }
+ t->workspace = tok_spm_workspace_new();
+ if (!t->workspace) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ t->cls_id = tok_spm_token_id(t->spm, "<s>");
+ t->sep_id = tok_spm_token_id(t->spm, "</s>");
+ if (t->cls_id < 0 || t->sep_id < 0) {
+ set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
+ goto fail;
+ }
+ return t;
+
+fail:
+ langdet_tok_free(t);
+ return NULL;
+}
+
+void langdet_tok_free(langdet_tok_t *t) {
+ if (!t)
+ return;
+ tok_spm_workspace_free(t->workspace);
+ tok_spm_free(t->spm);
+ free(t);
+}
+
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
+ if (n_out)
+ *n_out = 0;
+ if (!t || !text)
+ return NULL;
+ int n = 0;
+ int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n);
+ if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) {
+ free(core);
+ return NULL;
+ }
+ /* Empty text still produces the two special tokens. */
+ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
+ return NULL;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = t->cls_id;
+ ids[n + 1] = t->sep_id;
+ if (n_out)
+ *n_out = n + 2;
+ return ids;
+}