1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
#include "encoder.h"
#include "sentencepiece.h"
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
struct langdet_tok {
tok_spm_t *spm;
tok_spm_workspace_t *workspace;
};
static void set_err(char *err, size_t errlen, const char *msg) {
if (err && errlen)
snprintf(err, errlen, "%s", msg);
}
langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) {
if (!model_dir) {
set_err(err, errlen, "model_dir is NULL");
return NULL;
}
langdet_tok_t *t = calloc(1, sizeof(*t));
if (!t) {
set_err(err, errlen, "out of memory");
return NULL;
}
t->spm = tok_spm_load(model_dir);
if (!t->spm) {
set_err(err, errlen, "failed to load SentencePiece tokenizer");
goto fail;
}
t->workspace = tok_spm_workspace_new();
if (!t->workspace) {
set_err(err, errlen, "out of memory");
goto fail;
}
return t;
fail:
langdet_tok_free(t);
return NULL;
}
void langdet_tok_free(langdet_tok_t *t) {
if (!t)
return;
tok_spm_workspace_free(t->workspace);
tok_spm_free(t->spm);
free(t);
}
int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
if (n_out)
*n_out = 0;
if (!t || !text)
return NULL;
int n = 0;
int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n);
if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) {
free(core);
return NULL;
}
/* Empty text still produces the two special tokens. */
int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
if (!ids) {
free(core);
return NULL;
}
memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
ids[0] = TOK_XLMR_BOS_ID;
ids[n + 1] = TOK_XLMR_EOS_ID;
if (n_out)
*n_out = n + 2;
return ids;
}
|