diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 16:06:18 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 16:41:53 +0200 |
| commit | cb1f049e710cf8704ae29811b3058142c7f64840 (patch) | |
| tree | 85df308a574fbc3408cd88cd82471896337c8669 | |
| parent | bb41b40bf881a1c59ee0d6153bcb0bb2f7a0044b (diff) | |
| download | ffwd-cb1f049e710cf8704ae29811b3058142c7f64840.tar.gz ffwd-cb1f049e710cf8704ae29811b3058142c7f64840.zip | |
all: add Qwen3-Embedding-4B + fix terminal token
| -rw-r--r-- | README.md | 6 | ||||
| -rw-r--r-- | embed.c | 7 | ||||
| -rw-r--r-- | embed.h | 7 | ||||
| -rw-r--r-- | embed_cli.c | 9 | ||||
| -rw-r--r-- | embed_mlx.c | 8 | ||||
| -rw-r--r-- | embed_server.c | 14 | ||||
| -rw-r--r-- | tests/test_qwen3.c | 3 |
7 files changed, 29 insertions, 25 deletions
@@ -81,8 +81,10 @@ Late-interaction reranking endpoint: - `POST /v1/rerank` — late-interaction MaxSim reranking -For `Qwen3-Embedding-0.6B`, `/v1/embeddings` accepts `dimensions` from 32 -to 1024. Truncated Matryoshka embeddings are re-normalized before encoding. +For Qwen3-Embedding models, `/v1/embeddings` accepts `dimensions` from 32 up to +the model's output size (1024 for `Qwen3-Embedding-0.6B`, 2560 for +`Qwen3-Embedding-4B`). Truncated Matryoshka embeddings are re-normalized before +encoding. Example: @@ -239,14 +239,12 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) cfg->pooling_mode = EMBED_POOL_MEAN; cfg->normalize_embeddings = 0; cfg->append_terminal_token = 0; - cfg->terminal_token_id = -1; if (json_string_equals(buf, "model_type", "qwen3")) { cfg->attention_mode = EMBED_ATTENTION_CAUSAL; cfg->pooling_mode = EMBED_POOL_LAST_TOKEN; cfg->normalize_embeddings = 1; cfg->append_terminal_token = 1; - cfg->terminal_token_id = json_get_int(buf, "eos_token_id", -1); } cfg->q_dim = 0; @@ -267,10 +265,7 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f || - (cfg->append_terminal_token && - (cfg->terminal_token_id < 0 || - cfg->terminal_token_id >= cfg->vocab_size))) { + !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { fprintf(stderr, "embed_model_load: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, @@ -57,8 +57,13 @@ typedef struct { embed_attention_mode_t attention_mode; embed_pooling_mode_t pooling_mode; int normalize_embeddings; + /* Qwen3-Embedding pools the last token, which is the tokenizer's + * <|endoftext|> suffix added by its post-processor - not the model's chat + * eos_token_id (<|im_end|>). Text frontends append it and resolve the id + * from the tokenizer exactly like the contextual separator + * (EMBED_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The + * model forward consumes token ids exactly and never appends. */ int append_terminal_token; - int terminal_token_id; } embed_config_t; /* ======================================================================== diff --git a/embed_cli.c b/embed_cli.c index 06b40de..74071f0 100644 --- a/embed_cli.c +++ b/embed_cli.c @@ -647,7 +647,14 @@ int main(int argc, char *argv[]) e.dim = config->hidden_size; } e.append_terminal_token = config->append_terminal_token; - e.terminal_token_id = config->terminal_token_id; + /* The appended terminal token is the tokenizer's <|endoftext|>, resolved + * like the contextual separator: a vocab that defines it (test models) + * wins; otherwise the released-model family constant applies. */ + if (e.append_terminal_token) { + int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>"); + e.terminal_token_id = + eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; + } if (verbose >= 1) fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0, diff --git a/embed_mlx.c b/embed_mlx.c index cbdfa33..cb3b737 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -408,14 +408,11 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) cfg->pooling_mode = EMBED_POOL_MEAN; cfg->normalize_embeddings = 0; cfg->append_terminal_token = 0; - cfg->terminal_token_id = -1; if (json_string_equals_m(buf, "model_type", "qwen3")) { - const char *eos = json_find_m(buf, "eos_token_id"); cfg->attention_mode = EMBED_ATTENTION_CAUSAL; cfg->pooling_mode = EMBED_POOL_LAST_TOKEN; cfg->normalize_embeddings = 1; cfg->append_terminal_token = 1; - cfg->terminal_token_id = eos ? atoi(eos) : -1; } cfg->q_dim = 0; @@ -434,10 +431,7 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f || - (cfg->append_terminal_token && - (cfg->terminal_token_id < 0 || - cfg->terminal_token_id >= cfg->vocab_size))) { + !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { fprintf(stderr, "mlx: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, diff --git a/embed_server.c b/embed_server.c index 1d64e1d..f22a99c 100644 --- a/embed_server.c +++ b/embed_server.c @@ -206,6 +206,7 @@ typedef enum { MODEL_STD_06, MODEL_STD_4, MODEL_QWEN3_06, + MODEL_QWEN3_4, MODEL_CTX_06, MODEL_CTX_4, MODEL_LATE_06, @@ -238,6 +239,8 @@ static const model_info k_models[] = { EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0}, {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, 0.0, EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1}, + {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, 0.0, + EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1}, {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, 0.008, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0}, {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, 0.050, @@ -2576,13 +2579,8 @@ static int configure_loaded_model(loaded_model *m, m->info->id); return -1; } - if (config->append_terminal_token && - (config->terminal_token_id < 0 || - config->terminal_token_id >= config->vocab_size)) - return -1; - m->append_terminal_token = config->append_terminal_token; - m->terminal_token_id = config->terminal_token_id; + /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */ m->renormalize_truncated = config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN; @@ -2769,6 +2767,10 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>"); m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; + /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|> + * suffix - the same token as the separator, not the model's chat + * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */ + m->terminal_token_id = m->context_separator_id; if (m->info->kind == MODEL_KIND_LATE) { int id = qwen_tokenizer_token_id(m->tok, "[MASK]"); m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID; diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c index 8205ca1..4d63a35 100644 --- a/tests/test_qwen3.c +++ b/tests/test_qwen3.c @@ -40,8 +40,7 @@ int main(void) } if (cfg->attention_mode != EMBED_ATTENTION_CAUSAL || cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || - !cfg->normalize_embeddings || !cfg->append_terminal_token || - cfg->terminal_token_id != 15) { + !cfg->normalize_embeddings || !cfg->append_terminal_token) { fprintf(stderr, "Qwen3 config semantics were not detected\n"); return 1; } |
