aboutsummaryrefslogtreecommitdiff
path: root/libffwd/tokenize.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
commit37cbaef9ceb828c667da8f398027ba5415ee7475 (patch)
tree6103e92a9030766e78a35df00f7a1155d0d0ecf5 /libffwd/tokenize.c
parentcf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff)
downloadlangdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz
langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip
Rename CPU library API and sources to langdet
Diffstat (limited to 'libffwd/tokenize.c')
-rw-r--r--libffwd/tokenize.c85
1 files changed, 0 insertions, 85 deletions
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c
deleted file mode 100644
index 8e1000f..0000000
--- a/libffwd/tokenize.c
+++ /dev/null
@@ -1,85 +0,0 @@
-#include "ffwd.h"
-#include "sentencepiece.h"
-
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-struct ffwd_tok {
- tok_spm_t *spm;
- tok_spm_workspace_t *workspace;
- int cls_id;
- int sep_id;
-};
-
-static void set_err(char *err, size_t errlen, const char *msg) {
- if (err && errlen)
- snprintf(err, errlen, "%s", msg);
-}
-
-ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) {
- if (!model_dir) {
- set_err(err, errlen, "model_dir is NULL");
- return NULL;
- }
- ffwd_tok_t *t = calloc(1, sizeof(*t));
- if (!t) {
- set_err(err, errlen, "out of memory");
- return NULL;
- }
- t->spm = tok_spm_load(model_dir);
- if (!t->spm) {
- set_err(err, errlen, "failed to load SentencePiece tokenizer");
- goto fail;
- }
- t->workspace = tok_spm_workspace_new();
- if (!t->workspace) {
- set_err(err, errlen, "out of memory");
- goto fail;
- }
- t->cls_id = tok_spm_token_id(t->spm, "<s>");
- t->sep_id = tok_spm_token_id(t->spm, "</s>");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
- goto fail;
- }
- return t;
-
-fail:
- ffwd_tok_free(t);
- return NULL;
-}
-
-void ffwd_tok_free(ffwd_tok_t *t) {
- if (!t)
- return;
- tok_spm_workspace_free(t->workspace);
- tok_spm_free(t->spm);
- free(t);
-}
-
-int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) {
- if (n_out)
- *n_out = 0;
- if (!t || !text)
- return NULL;
- int n = 0;
- int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n);
- if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) {
- free(core);
- return NULL;
- }
- /* Empty text still produces the two special tokens. */
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- return NULL;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = t->cls_id;
- ids[n + 1] = t->sep_id;
- if (n_out)
- *n_out = n + 2;
- return ids;
-}