aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-14 16:06:18 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 16:41:53 +0200
commitcb1f049e710cf8704ae29811b3058142c7f64840 (patch)
tree85df308a574fbc3408cd88cd82471896337c8669
parentbb41b40bf881a1c59ee0d6153bcb0bb2f7a0044b (diff)
downloadffwd-cb1f049e710cf8704ae29811b3058142c7f64840.tar.gz
ffwd-cb1f049e710cf8704ae29811b3058142c7f64840.zip
all: add Qwen3-Embedding-4B + fix terminal token
-rw-r--r--README.md6
-rw-r--r--embed.c7
-rw-r--r--embed.h7
-rw-r--r--embed_cli.c9
-rw-r--r--embed_mlx.c8
-rw-r--r--embed_server.c14
-rw-r--r--tests/test_qwen3.c3
7 files changed, 29 insertions, 25 deletions
diff --git a/README.md b/README.md
index 59ea6b9..f056d40 100644
--- a/README.md
+++ b/README.md
@@ -81,8 +81,10 @@ Late-interaction reranking endpoint:
- `POST /v1/rerank` — late-interaction MaxSim reranking
-For `Qwen3-Embedding-0.6B`, `/v1/embeddings` accepts `dimensions` from 32
-to 1024. Truncated Matryoshka embeddings are re-normalized before encoding.
+For Qwen3-Embedding models, `/v1/embeddings` accepts `dimensions` from 32 up to
+the model's output size (1024 for `Qwen3-Embedding-0.6B`, 2560 for
+`Qwen3-Embedding-4B`). Truncated Matryoshka embeddings are re-normalized before
+encoding.
Example:
diff --git a/embed.c b/embed.c
index 9feb99c..bd72846 100644
--- a/embed.c
+++ b/embed.c
@@ -239,14 +239,12 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
cfg->pooling_mode = EMBED_POOL_MEAN;
cfg->normalize_embeddings = 0;
cfg->append_terminal_token = 0;
- cfg->terminal_token_id = -1;
if (json_string_equals(buf, "model_type", "qwen3")) {
cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
cfg->pooling_mode = EMBED_POOL_LAST_TOKEN;
cfg->normalize_embeddings = 1;
cfg->append_terminal_token = 1;
- cfg->terminal_token_id = json_get_int(buf, "eos_token_id", -1);
}
cfg->q_dim = 0;
@@ -267,10 +265,7 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
(cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
!isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f ||
- (cfg->append_terminal_token &&
- (cfg->terminal_token_id < 0 ||
- cfg->terminal_token_id >= cfg->vocab_size))) {
+ !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
fprintf(stderr, "embed_model_load: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
diff --git a/embed.h b/embed.h
index 764e5c6..f7048b4 100644
--- a/embed.h
+++ b/embed.h
@@ -57,8 +57,13 @@ typedef struct {
embed_attention_mode_t attention_mode;
embed_pooling_mode_t pooling_mode;
int normalize_embeddings;
+ /* Qwen3-Embedding pools the last token, which is the tokenizer's
+ * <|endoftext|> suffix added by its post-processor - not the model's chat
+ * eos_token_id (<|im_end|>). Text frontends append it and resolve the id
+ * from the tokenizer exactly like the contextual separator
+ * (EMBED_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The
+ * model forward consumes token ids exactly and never appends. */
int append_terminal_token;
- int terminal_token_id;
} embed_config_t;
/* ========================================================================
diff --git a/embed_cli.c b/embed_cli.c
index 06b40de..74071f0 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -647,7 +647,14 @@ int main(int argc, char *argv[])
e.dim = config->hidden_size;
}
e.append_terminal_token = config->append_terminal_token;
- e.terminal_token_id = config->terminal_token_id;
+ /* The appended terminal token is the tokenizer's <|endoftext|>, resolved
+ * like the contextual separator: a vocab that defines it (test models)
+ * wins; otherwise the released-model family constant applies. */
+ if (e.append_terminal_token) {
+ int eot = qwen_tokenizer_token_id(tok, "<|endoftext|>");
+ e.terminal_token_id =
+ eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ }
if (verbose >= 1)
fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
e.dim, now_ms() - t0,
diff --git a/embed_mlx.c b/embed_mlx.c
index cbdfa33..cb3b737 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -408,14 +408,11 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
cfg->pooling_mode = EMBED_POOL_MEAN;
cfg->normalize_embeddings = 0;
cfg->append_terminal_token = 0;
- cfg->terminal_token_id = -1;
if (json_string_equals_m(buf, "model_type", "qwen3")) {
- const char *eos = json_find_m(buf, "eos_token_id");
cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
cfg->pooling_mode = EMBED_POOL_LAST_TOKEN;
cfg->normalize_embeddings = 1;
cfg->append_terminal_token = 1;
- cfg->terminal_token_id = eos ? atoi(eos) : -1;
}
cfg->q_dim = 0;
@@ -434,10 +431,7 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
(cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
!isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f ||
- (cfg->append_terminal_token &&
- (cfg->terminal_token_id < 0 ||
- cfg->terminal_token_id >= cfg->vocab_size))) {
+ !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
fprintf(stderr, "mlx: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
diff --git a/embed_server.c b/embed_server.c
index 1d64e1d..f22a99c 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -206,6 +206,7 @@ typedef enum {
MODEL_STD_06,
MODEL_STD_4,
MODEL_QWEN3_06,
+ MODEL_QWEN3_4,
MODEL_CTX_06,
MODEL_CTX_4,
MODEL_LATE_06,
@@ -238,6 +239,8 @@ static const model_info k_models[] = {
EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0},
{"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, 0.0,
EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1},
+ {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, 0.0,
+ EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1},
{"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, 0.008,
EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0},
{"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, 0.050,
@@ -2576,13 +2579,8 @@ static int configure_loaded_model(loaded_model *m,
m->info->id);
return -1;
}
- if (config->append_terminal_token &&
- (config->terminal_token_id < 0 ||
- config->terminal_token_id >= config->vocab_size))
- return -1;
-
m->append_terminal_token = config->append_terminal_token;
- m->terminal_token_id = config->terminal_token_id;
+ /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
m->renormalize_truncated =
config->normalize_embeddings &&
config->pooling_mode == EMBED_POOL_LAST_TOKEN;
@@ -2769,6 +2767,10 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>");
m->context_separator_id = sep_id >= 0
? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
+ /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|>
+ * suffix - the same token as the separator, not the model's chat
+ * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */
+ m->terminal_token_id = m->context_separator_id;
if (m->info->kind == MODEL_KIND_LATE) {
int id = qwen_tokenizer_token_id(m->tok, "[MASK]");
m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID;
diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c
index 8205ca1..4d63a35 100644
--- a/tests/test_qwen3.c
+++ b/tests/test_qwen3.c
@@ -40,8 +40,7 @@ int main(void)
}
if (cfg->attention_mode != EMBED_ATTENTION_CAUSAL ||
cfg->pooling_mode != EMBED_POOL_LAST_TOKEN ||
- !cfg->normalize_embeddings || !cfg->append_terminal_token ||
- cfg->terminal_token_id != 15) {
+ !cfg->normalize_embeddings || !cfg->append_terminal_token) {
fprintf(stderr, "Qwen3 config semantics were not detected\n");
return 1;
}