diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 16:06:18 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 16:41:53 +0200 |
| commit | cb1f049e710cf8704ae29811b3058142c7f64840 (patch) | |
| tree | 85df308a574fbc3408cd88cd82471896337c8669 /embed_cli.c | |
| parent | bb41b40bf881a1c59ee0d6153bcb0bb2f7a0044b (diff) | |
| download | ffwd-cb1f049e710cf8704ae29811b3058142c7f64840.tar.gz ffwd-cb1f049e710cf8704ae29811b3058142c7f64840.zip | |
all: add Qwen3-Embedding-4B + fix terminal token
Diffstat (limited to 'embed_cli.c')
| -rw-r--r-- | embed_cli.c | 9 |
1 files changed, 8 insertions, 1 deletions
diff --git a/embed_cli.c b/embed_cli.c index 06b40de..74071f0 100644 --- a/embed_cli.c +++ b/embed_cli.c @@ -647,7 +647,14 @@ int main(int argc, char *argv[]) e.dim = config->hidden_size; } e.append_terminal_token = config->append_terminal_token; - e.terminal_token_id = config->terminal_token_id; + /* The appended terminal token is the tokenizer's <|endoftext|>, resolved + * like the contextual separator: a vocab that defines it (test models) + * wins; otherwise the released-model family constant applies. */ + if (e.append_terminal_token) { + int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>"); + e.terminal_token_id = + eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; + } if (verbose >= 1) fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0, |
