aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/tokenize.c
diff options
context:
space:
mode:
Diffstat (limited to 'liblangdet/tokenize.c')
-rw-r--r--liblangdet/tokenize.c12
1 files changed, 2 insertions, 10 deletions
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
index 96a5f42..8b4344c 100644
--- a/liblangdet/tokenize.c
+++ b/liblangdet/tokenize.c
@@ -9,8 +9,6 @@
struct langdet_tok {
tok_spm_t *spm;
tok_spm_workspace_t *workspace;
- int cls_id;
- int sep_id;
};
static void set_err(char *err, size_t errlen, const char *msg) {
@@ -38,12 +36,6 @@ langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen)
set_err(err, errlen, "out of memory");
goto fail;
}
- t->cls_id = tok_spm_token_id(t->spm, "<s>");
- t->sep_id = tok_spm_token_id(t->spm, "</s>");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
- goto fail;
- }
return t;
fail:
@@ -77,8 +69,8 @@ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
return NULL;
}
memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = t->cls_id;
- ids[n + 1] = t->sep_id;
+ ids[0] = TOK_XLMR_BOS_ID;
+ ids[n + 1] = TOK_XLMR_EOS_ID;
if (n_out)
*n_out = n + 2;
return ids;