diff options
Diffstat (limited to 'libffwd/tokenize.c')
| -rw-r--r-- | libffwd/tokenize.c | 85 |
1 files changed, 0 insertions, 85 deletions
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c deleted file mode 100644 index 8e1000f..0000000 --- a/libffwd/tokenize.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "ffwd.h" -#include "sentencepiece.h" - -#include <limits.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -struct ffwd_tok { - tok_spm_t *spm; - tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; -}; - -static void set_err(char *err, size_t errlen, const char *msg) { - if (err && errlen) - snprintf(err, errlen, "%s", msg); -} - -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) { - if (!model_dir) { - set_err(err, errlen, "model_dir is NULL"); - return NULL; - } - ffwd_tok_t *t = calloc(1, sizeof(*t)); - if (!t) { - set_err(err, errlen, "out of memory"); - return NULL; - } - t->spm = tok_spm_load(model_dir); - if (!t->spm) { - set_err(err, errlen, "failed to load SentencePiece tokenizer"); - goto fail; - } - t->workspace = tok_spm_workspace_new(); - if (!t->workspace) { - set_err(err, errlen, "out of memory"); - goto fail; - } - t->cls_id = tok_spm_token_id(t->spm, "<s>"); - t->sep_id = tok_spm_token_id(t->spm, "</s>"); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); - goto fail; - } - return t; - -fail: - ffwd_tok_free(t); - return NULL; -} - -void ffwd_tok_free(ffwd_tok_t *t) { - if (!t) - return; - tok_spm_workspace_free(t->workspace); - tok_spm_free(t->spm); - free(t); -} - -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) { - if (n_out) - *n_out = 0; - if (!t || !text) - return NULL; - int n = 0; - int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n); - if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) { - free(core); - return NULL; - } - /* Empty text still produces the two special tokens. */ - int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - return NULL; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; - if (n_out) - *n_out = n + 2; - return ids; -} |
