diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
| commit | 37cbaef9ceb828c667da8f398027ba5415ee7475 (patch) | |
| tree | 6103e92a9030766e78a35df00f7a1155d0d0ecf5 /libffwd/tokenize.c | |
| parent | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff) | |
| download | langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip | |
Rename CPU library API and sources to langdet
Diffstat (limited to 'libffwd/tokenize.c')
| -rw-r--r-- | libffwd/tokenize.c | 85 |
1 files changed, 0 insertions, 85 deletions
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c deleted file mode 100644 index 8e1000f..0000000 --- a/libffwd/tokenize.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "ffwd.h" -#include "sentencepiece.h" - -#include <limits.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -struct ffwd_tok { - tok_spm_t *spm; - tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; -}; - -static void set_err(char *err, size_t errlen, const char *msg) { - if (err && errlen) - snprintf(err, errlen, "%s", msg); -} - -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) { - if (!model_dir) { - set_err(err, errlen, "model_dir is NULL"); - return NULL; - } - ffwd_tok_t *t = calloc(1, sizeof(*t)); - if (!t) { - set_err(err, errlen, "out of memory"); - return NULL; - } - t->spm = tok_spm_load(model_dir); - if (!t->spm) { - set_err(err, errlen, "failed to load SentencePiece tokenizer"); - goto fail; - } - t->workspace = tok_spm_workspace_new(); - if (!t->workspace) { - set_err(err, errlen, "out of memory"); - goto fail; - } - t->cls_id = tok_spm_token_id(t->spm, "<s>"); - t->sep_id = tok_spm_token_id(t->spm, "</s>"); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); - goto fail; - } - return t; - -fail: - ffwd_tok_free(t); - return NULL; -} - -void ffwd_tok_free(ffwd_tok_t *t) { - if (!t) - return; - tok_spm_workspace_free(t->workspace); - tok_spm_free(t->spm); - free(t); -} - -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) { - if (n_out) - *n_out = 0; - if (!t || !text) - return NULL; - int n = 0; - int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); - if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { - free(core); - return NULL; - } - /* Empty text still produces the two special tokens. */ - int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - return NULL; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; - if (n_out) - *n_out = n + 2; - return ids; -} |
