aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/tokenize.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
commit91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch)
treeb16f686279a1f1b0691a6c0d65a544084fcc1c1e /liblangdet/tokenize.c
parent6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff)
downloadlangdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz
langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'liblangdet/tokenize.c')
-rw-r--r--liblangdet/tokenize.c12
1 files changed, 2 insertions, 10 deletions
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
index 96a5f42..8b4344c 100644
--- a/liblangdet/tokenize.c
+++ b/liblangdet/tokenize.c
@@ -9,8 +9,6 @@
struct langdet_tok {
tok_spm_t *spm;
tok_spm_workspace_t *workspace;
- int cls_id;
- int sep_id;
};
static void set_err(char *err, size_t errlen, const char *msg) {
@@ -38,12 +36,6 @@ langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen)
set_err(err, errlen, "out of memory");
goto fail;
}
- t->cls_id = tok_spm_token_id(t->spm, "<s>");
- t->sep_id = tok_spm_token_id(t->spm, "</s>");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
- goto fail;
- }
return t;
fail:
@@ -77,8 +69,8 @@ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
return NULL;
}
memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = t->cls_id;
- ids[n + 1] = t->sep_id;
+ ids[0] = TOK_XLMR_BOS_ID;
+ ids[n + 1] = TOK_XLMR_EOS_ID;
if (n_out)
*n_out = n + 2;
return ids;