aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/tokenize.c
blob: 8b4344cc98f315b740cddc9b5aa310f43cac9c2a (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
#include "encoder.h"
#include "sentencepiece.h"

#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

struct langdet_tok {
    tok_spm_t *spm;
    tok_spm_workspace_t *workspace;
};

static void set_err(char *err, size_t errlen, const char *msg) {
    if (err && errlen)
        snprintf(err, errlen, "%s", msg);
}

langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) {
    if (!model_dir) {
        set_err(err, errlen, "model_dir is NULL");
        return NULL;
    }
    langdet_tok_t *t = calloc(1, sizeof(*t));
    if (!t) {
        set_err(err, errlen, "out of memory");
        return NULL;
    }
    t->spm = tok_spm_load(model_dir);
    if (!t->spm) {
        set_err(err, errlen, "failed to load SentencePiece tokenizer");
        goto fail;
    }
    t->workspace = tok_spm_workspace_new();
    if (!t->workspace) {
        set_err(err, errlen, "out of memory");
        goto fail;
    }
    return t;

fail:
    langdet_tok_free(t);
    return NULL;
}

void langdet_tok_free(langdet_tok_t *t) {
    if (!t)
        return;
    tok_spm_workspace_free(t->workspace);
    tok_spm_free(t->spm);
    free(t);
}

int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
    if (n_out)
        *n_out = 0;
    if (!t || !text)
        return NULL;
    int n = 0;
    int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n);
    if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) {
        free(core);
        return NULL;
    }
    /* Empty text still produces the two special tokens. */
    int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
    if (!ids) {
        free(core);
        return NULL;
    }
    memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
    ids[0] = TOK_XLMR_BOS_ID;
    ids[n + 1] = TOK_XLMR_EOS_ID;
    if (n_out)
        *n_out = n + 2;
    return ids;
}