aboutsummaryrefslogtreecommitdiff
path: root/embed_cli.c
diff options
context:
space:
mode:
Diffstat (limited to 'embed_cli.c')
-rw-r--r--embed_cli.c9
1 files changed, 8 insertions, 1 deletions
diff --git a/embed_cli.c b/embed_cli.c
index 06b40de..74071f0 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -647,7 +647,14 @@ int main(int argc, char *argv[])
e.dim = config->hidden_size;
}
e.append_terminal_token = config->append_terminal_token;
- e.terminal_token_id = config->terminal_token_id;
+ /* The appended terminal token is the tokenizer's <|endoftext|>, resolved
+ * like the contextual separator: a vocab that defines it (test models)
+ * wins; otherwise the released-model family constant applies. */
+ if (e.append_terminal_token) {
+ int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>");
+ e.terminal_token_id =
+ eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ }
if (verbose >= 1)
fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
e.dim, now_ms() - t0,