From 37cbaef9ceb828c667da8f398027ba5415ee7475 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 29 Sep 2026 13:11:00 +0200 Subject: Rename CPU library API and sources to langdet --- libffwd/tokenize.c | 85 ------------------------------------------------------ 1 file changed, 85 deletions(-) delete mode 100644 libffwd/tokenize.c (limited to 'libffwd/tokenize.c') diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c deleted file mode 100644 index 8e1000f..0000000 --- a/libffwd/tokenize.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "ffwd.h" -#include "sentencepiece.h" - -#include -#include -#include -#include - -struct ffwd_tok { - tok_spm_t *spm; - tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; -}; - -static void set_err(char *err, size_t errlen, const char *msg) { - if (err && errlen) - snprintf(err, errlen, "%s", msg); -} - -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) { - if (!model_dir) { - set_err(err, errlen, "model_dir is NULL"); - return NULL; - } - ffwd_tok_t *t = calloc(1, sizeof(*t)); - if (!t) { - set_err(err, errlen, "out of memory"); - return NULL; - } - t->spm = tok_spm_load(model_dir); - if (!t->spm) { - set_err(err, errlen, "failed to load SentencePiece tokenizer"); - goto fail; - } - t->workspace = tok_spm_workspace_new(); - if (!t->workspace) { - set_err(err, errlen, "out of memory"); - goto fail; - } - t->cls_id = tok_spm_token_id(t->spm, ""); - t->sep_id = tok_spm_token_id(t->spm, ""); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing /"); - goto fail; - } - return t; - -fail: - ffwd_tok_free(t); - return NULL; -} - -void ffwd_tok_free(ffwd_tok_t *t) { - if (!t) - return; - tok_spm_workspace_free(t->workspace); - tok_spm_free(t->spm); - free(t); -} - -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) { - if (n_out) - *n_out = 0; - if (!t || !text) - return NULL; - int n = 0; - int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); - if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { - free(core); - return NULL; - } - /* Empty text still produces the two special tokens. */ - int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - return NULL; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; - if (n_out) - *n_out = n + 2; - return ids; -} -- cgit v1.3.1