aboutsummaryrefslogtreecommitdiff
path: root/embed_cli.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-14 16:06:18 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 16:41:53 +0200
commitcb1f049e710cf8704ae29811b3058142c7f64840 (patch)
tree85df308a574fbc3408cd88cd82471896337c8669 /embed_cli.c
parentbb41b40bf881a1c59ee0d6153bcb0bb2f7a0044b (diff)
downloadffwd-cb1f049e710cf8704ae29811b3058142c7f64840.tar.gz
ffwd-cb1f049e710cf8704ae29811b3058142c7f64840.zip
all: add Qwen3-Embedding-4B + fix terminal token
Diffstat (limited to 'embed_cli.c')
-rw-r--r--embed_cli.c9
1 files changed, 8 insertions, 1 deletions
diff --git a/embed_cli.c b/embed_cli.c
index 06b40de..74071f0 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -647,7 +647,14 @@ int main(int argc, char *argv[])
e.dim = config->hidden_size;
}
e.append_terminal_token = config->append_terminal_token;
- e.terminal_token_id = config->terminal_token_id;
+ /* The appended terminal token is the tokenizer's <|endoftext|>, resolved
+ * like the contextual separator: a vocab that defines it (test models)
+ * wins; otherwise the released-model family constant applies. */
+ if (e.append_terminal_token) {
+ int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>");
+ e.terminal_token_id =
+ eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ }
if (verbose >= 1)
fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
e.dim, now_ms() - t0,