diff options
Diffstat (limited to 'liblangdet/tokenize.c')
| -rw-r--r-- | liblangdet/tokenize.c | 12 |
1 files changed, 2 insertions, 10 deletions
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c index 96a5f42..8b4344c 100644 --- a/liblangdet/tokenize.c +++ b/liblangdet/tokenize.c @@ -9,8 +9,6 @@ struct langdet_tok { tok_spm_t *spm; tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; }; static void set_err(char *err, size_t errlen, const char *msg) { @@ -38,12 +36,6 @@ langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) set_err(err, errlen, "out of memory"); goto fail; } - t->cls_id = tok_spm_token_id(t->spm, "<s>"); - t->sep_id = tok_spm_token_id(t->spm, "</s>"); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); - goto fail; - } return t; fail: @@ -77,8 +69,8 @@ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { return NULL; } memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; + ids[0] = TOK_XLMR_BOS_ID; + ids[n + 1] = TOK_XLMR_EOS_ID; if (n_out) *n_out = n + 2; return ids; |
