diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
| commit | 91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch) | |
| tree | b16f686279a1f1b0691a6c0d65a544084fcc1c1e /liblangdet/tokenize.c | |
| parent | 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff) | |
| download | langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip | |
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'liblangdet/tokenize.c')
| -rw-r--r-- | liblangdet/tokenize.c | 12 |
1 files changed, 2 insertions, 10 deletions
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c index 96a5f42..8b4344c 100644 --- a/liblangdet/tokenize.c +++ b/liblangdet/tokenize.c @@ -9,8 +9,6 @@ struct langdet_tok { tok_spm_t *spm; tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; }; static void set_err(char *err, size_t errlen, const char *msg) { @@ -38,12 +36,6 @@ langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) set_err(err, errlen, "out of memory"); goto fail; } - t->cls_id = tok_spm_token_id(t->spm, "<s>"); - t->sep_id = tok_spm_token_id(t->spm, "</s>"); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); - goto fail; - } return t; fail: @@ -77,8 +69,8 @@ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { return NULL; } memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; + ids[0] = TOK_XLMR_BOS_ID; + ids[n + 1] = TOK_XLMR_EOS_ID; if (n_out) *n_out = n + 2; return ids; |
