aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-14 15:19:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 15:19:12 +0200
commitf463018cd700690336898d109226d489deca036d (patch)
tree4e1d83f55a5e231af519aa4d647141db5de23af7
parentd98ee1568c0bd88304a12e59d12104066e01e8d0 (diff)
downloadffwd-f463018cd700690336898d109226d489deca036d.tar.gz
ffwd-f463018cd700690336898d109226d489deca036d.zip
all: add support for Qwen3-Embedding + causal GQA attention
-rw-r--r--Makefile11
-rw-r--r--README.md24
-rw-r--r--embed.c111
-rw-r--r--embed.h48
-rw-r--r--embed_cli.c31
-rw-r--r--embed_cuda.cu101
-rw-r--r--embed_internal.h6
-rw-r--r--embed_mlx.c151
-rw-r--r--embed_mlx.h2
-rw-r--r--embed_server.c136
-rw-r--r--qwen_kernels.c152
-rw-r--r--qwen_kernels.h14
-rw-r--r--tests/test_kernels.c49
-rw-r--r--tests/test_qwen3.c92
-rw-r--r--tests/test_server.c65
-rw-r--r--tests/tiny_model.h40
16 files changed, 901 insertions, 132 deletions
diff --git a/Makefile b/Makefile
index f7a3927..f1285aa 100644
--- a/Makefile
+++ b/Makefile
@@ -162,6 +162,8 @@ TEST_TOKENIZER_SRCS = tests/test_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS)
TEST_SAFETENSORS_SRCS = tests/test_safetensors.c qwen_safetensors.c
TEST_BF16_SRCS = tests/test_bf16_model.c embed.c qwen_safetensors.c \
$(KERNEL_SRCS)
+TEST_QWEN3_SRCS = tests/test_qwen3.c embed.c qwen_safetensors.c \
+ $(KERNEL_SRCS)
TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c qwen_tokenizer.c \
qwen_safetensors.c $(KERNEL_SRCS)
TEST_LATE_SRCS = tests/test_late.c embed.c qwen_tokenizer.c \
@@ -188,6 +190,9 @@ test:
$(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \
$(TEST_BF16_SRCS) -lm -lpthread
./tests/test_bf16_model
+ $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \
+ $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ ./tests/test_qwen3
$(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \
$(TEST_WORKSPACE_SRCS) -lm -lpthread
./tests/test_workspace
@@ -226,7 +231,7 @@ COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \
-fcoverage-compilation-dir=.
COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \
tests/test_tokenizer tests/test_safetensors \
- tests/test_bf16_model tests/test_workspace tests/test_late \
+ tests/test_bf16_model tests/test_qwen3 tests/test_workspace tests/test_late \
tests/cli_under_test tests/test_server
# Report on project sources only - not the harnesses, vendored deps, or
# system headers (cJSON lands under /opt or /usr otherwise).
@@ -256,6 +261,9 @@ coverage:
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_bf16_model \
$(TEST_BF16_SRCS) -lm -lpthread
LLVM_PROFILE_FILE=$(COV_DIR)/bf16_model.profraw ./tests/test_bf16_model
+ $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \
+ $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ LLVM_PROFILE_FILE=$(COV_DIR)/qwen3.profraw ./tests/test_qwen3
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \
$(TEST_WORKSPACE_SRCS) -lm -lpthread
LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace
@@ -381,6 +389,7 @@ clean:
$(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \
tests/test_kernels_generic tests/test_kernels_blas \
tests/test_safetensors tests/test_bf16_model tests/test_server \
+ tests/test_qwen3 \
tests/test_tokenizer tests/test_workspace tests/test_cli tests/test_late \
tests/cli_under_test bench/bench_tokenizer bench/bench_kernels bench/bench_model
rm -rf $(COV_DIR)
diff --git a/README.md b/README.md
index 977576e..f6b01b0 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,11 @@
# embed.c
-`embed.c` implements inference for
-[pplx-embed](https://huggingface.co/collections/perplexity-ai/pplx-embed)
-dense/contextual and late-interaction text embedding models.
+`embed.c` implements inference for:
+
+- [pplx-embed](https://huggingface.co/collections/perplexity-ai/pplx-embed)
+ dense, contextual, and late-interaction models
+- [Qwen3-Embedding](https://huggingface.co/collections/Qwen/qwen3-embedding)
+ causal, last-token pooled models
Supported platforms are CUDA/cuBLAS on Linux/NVIDIA, Apple Silicon MLX, and
CPU/BLAS.
@@ -34,6 +37,17 @@ matrix:
"Berlin is the capital of Germany."
```
+Qwen3 retrieval queries benefit from an explicit task instruction. Documents are
+embedded directly:
+
+```bash
+./embed -d ./Qwen3-Embedding-0.6B \
+ $'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: What is the capital of China?'
+```
+
+The Qwen3 tokenizer terminal token is appended automatically by the CLI and
+server. Qwen3 embeddings are L2-normalized; pplx embeddings remain unnormalized.
+
Pipe in lines and use `--stream` to get one JSON embedding per line:
```bash
@@ -54,6 +68,7 @@ Serves the HTTP API. Takes one or more `--model ID=DIR` pairs:
```bash
./embed-server \
--model pplx-embed-v1-0.6b=./model \
+ --model Qwen3-Embedding-0.6B=./qwen3-model \
--port 8000
```
@@ -66,6 +81,9 @@ Late-interaction reranking endpoint:
- `POST /v1/rerank` — late-interaction MaxSim reranking
+For `Qwen3-Embedding-0.6B`, `/v1/embeddings` accepts `dimensions` from 32
+to 1024. Truncated Matryoshka embeddings are re-normalized before encoding.
+
Example:
```bash
diff --git a/embed.c b/embed.c
index 562b314..9feb99c 100644
--- a/embed.c
+++ b/embed.c
@@ -185,6 +185,16 @@ static double json_get_double(const char *json, const char *key, double fallback
return atof(v);
}
+static int json_string_equals(const char *json, const char *key,
+ const char *expected)
+{
+ const char *v = json_find_key(json, key);
+ if (!v || *v != '"') return 0;
+ v++;
+ size_t len = strlen(expected);
+ return strncmp(v, expected, len) == 0 && v[len] == '"';
+}
+
static int parse_config(embed_config_t *cfg, const char *model_dir)
{
char path[1024];
@@ -225,6 +235,19 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
+ cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
+ cfg->pooling_mode = EMBED_POOL_MEAN;
+ cfg->normalize_embeddings = 0;
+ cfg->append_terminal_token = 0;
+ cfg->terminal_token_id = -1;
+
+ if (json_string_equals(buf, "model_type", "qwen3")) {
+ cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
+ cfg->pooling_mode = EMBED_POOL_LAST_TOKEN;
+ cfg->normalize_embeddings = 1;
+ cfg->append_terminal_token = 1;
+ cfg->terminal_token_id = json_get_int(buf, "eos_token_id", -1);
+ }
cfg->q_dim = 0;
cfg->kv_dim = 0;
@@ -244,7 +267,10 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
(cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
!isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
+ !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f ||
+ (cfg->append_terminal_token &&
+ (cfg->terminal_token_id < 0 ||
+ cfg->terminal_token_id >= cfg->vocab_size))) {
fprintf(stderr, "embed_model_load: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
@@ -259,10 +285,14 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
if (embed_verbose >= 1)
fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, "
- "inter=%d, head_dim=%d\n",
+ "inter=%d, head_dim=%d, attention=%s, pooling=%s\n",
cfg->hidden_size, cfg->n_layers,
cfg->n_heads, cfg->n_kv_heads,
- cfg->intermediate_size, cfg->head_dim);
+ cfg->intermediate_size, cfg->head_dim,
+ cfg->attention_mode == EMBED_ATTENTION_CAUSAL
+ ? "causal" : "bidirectional",
+ cfg->pooling_mode == EMBED_POOL_LAST_TOKEN
+ ? "last-token" : "mean");
return 0;
}
@@ -618,6 +648,10 @@ static embed_model_t *model_load_range_ex(const char *model_dir,
}
model->layer_start = layer_start;
model->layer_end = layer_end;
+ model->attention =
+ cfg->attention_mode == EMBED_ATTENTION_CAUSAL
+ ? qwen_causal_gqa_attention_packed_with_scratch
+ : qwen_bidirectional_gqa_attention_packed_with_scratch;
/* Open safetensors (handles single file or multi-shard) */
multi_safetensors_t *ms = multi_safetensors_open(model_dir);
@@ -1187,7 +1221,7 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
rope_cos, rope_sin, len, n_kv_heads, head_dim);
}
- qwen_bidirectional_gqa_attention_packed_with_scratch(
+ model->attention(
attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads,
n_kv_heads, head_dim, scale, ws->attn_scores,
ws->attn_scores_bytes);
@@ -1223,16 +1257,18 @@ static int forward_packed_inplace(const embed_model_t *model,
apply_final_norm);
}
-static void pool_embeddings(const embed_model_t *model,
- const embed_workspace_t *ws,
- const int *offsets, int batch,
- float *out_embeddings)
+static int pool_embeddings(const embed_model_t *model,
+ const embed_workspace_t *ws,
+ const int *offsets, int batch,
+ float *out_embeddings)
{
const embed_config_t *cfg = &model->config;
int hidden = cfg->hidden_size;
float eps = cfg->rms_norm_eps;
const float *norm_weight = model->weights.norm;
const float *x = ws->x;
+ if (hidden <= 0 || !norm_weight || !x)
+ return -1;
for (int b = 0; b < batch; b++) {
int start = offsets[b];
@@ -1241,23 +1277,36 @@ static void pool_embeddings(const embed_model_t *model,
float *emb = out_embeddings + (size_t)b * hidden;
memset(emb, 0, (size_t)hidden * sizeof(float));
- for (int i = start; i < end; i++) {
- const float *row = x + (size_t)i * hidden;
-
+ if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) {
+ const float *row = x + (size_t)(end - 1) * hidden;
float sum_sq = 0.0f;
for (int d = 0; d < hidden; d++)
sum_sq += row[d] * row[d];
-
float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps);
for (int d = 0; d < hidden; d++)
- emb[d] += row[d] * rms_inv * norm_weight[d];
- }
+ emb[d] = row[d] * rms_inv * norm_weight[d];
+ } else {
+ for (int i = start; i < end; i++) {
+ const float *row = x + (size_t)i * hidden;
- float inv_len = 1.0f / (float)len;
- for (int d = 0; d < hidden; d++)
- emb[d] *= inv_len;
+ float sum_sq = 0.0f;
+ for (int d = 0; d < hidden; d++)
+ sum_sq += row[d] * row[d];
+ float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps);
+ for (int d = 0; d < hidden; d++)
+ emb[d] += row[d] * rms_inv * norm_weight[d];
+ }
+ float inv_len = 1.0f / (float)len;
+ for (int d = 0; d < hidden; d++)
+ emb[d] *= inv_len;
+ }
+
+ if (cfg->normalize_embeddings &&
+ embed_l2_normalize(emb, hidden) != 0)
+ return -1;
}
+ return 0;
}
/* ========================================================================
@@ -1431,7 +1480,7 @@ int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws,
int rc = forward_packed_inplace(model, ws, inputs, batch, offsets,
total_seq, max_seq, 0);
if (rc == 0)
- pool_embeddings(model, ws, offsets, batch, out_embeddings);
+ rc = pool_embeddings(model, ws, offsets, batch, out_embeddings);
return rc;
}
@@ -1451,16 +1500,25 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states,
return -1;
float *emb = out_embeddings + (size_t)b * cfg->hidden_size;
- memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float));
- for (int t = 0; t < len; t++) {
- const float *row = states + (size_t)(offset + t) * cfg->hidden_size;
+ if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) {
+ const float *row =
+ states + (size_t)(offset + len - 1) * cfg->hidden_size;
+ memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float));
+ } else {
+ memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float));
+ for (int t = 0; t < len; t++) {
+ const float *row =
+ states + (size_t)(offset + t) * cfg->hidden_size;
+ for (int d = 0; d < cfg->hidden_size; d++)
+ emb[d] += row[d];
+ }
+ float inv_len = 1.0f / (float)len;
for (int d = 0; d < cfg->hidden_size; d++)
- emb[d] += row[d];
+ emb[d] *= inv_len;
}
-
- float inv_len = 1.0f / (float)len;
- for (int d = 0; d < cfg->hidden_size; d++)
- emb[d] *= inv_len;
+ if (cfg->normalize_embeddings &&
+ embed_l2_normalize(emb, cfg->hidden_size) != 0)
+ return -1;
offset += len;
}
return 0;
@@ -1480,6 +1538,7 @@ float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws,
if (!model || !ws || !token_ids || n_tokens <= 0) return NULL;
int hidden = model->config.hidden_size;
+ if (hidden <= 0) return NULL;
float *emb = malloc_floats((size_t)hidden);
if (!emb) return NULL;
diff --git a/embed.h b/embed.h
index 965d043..764e5c6 100644
--- a/embed.h
+++ b/embed.h
@@ -20,7 +20,7 @@
#include <stdint.h>
/* ========================================================================
- * Shared constants (identical across all models)
+ * Shared constants and family defaults
* ======================================================================== */
#define EMBED_VOCAB_SIZE 151936
@@ -32,18 +32,33 @@
* Model Configuration (populated from config.json at load time)
* ======================================================================== */
+typedef enum {
+ EMBED_ATTENTION_BIDIRECTIONAL = 0,
+ EMBED_ATTENTION_CAUSAL = 1,
+} embed_attention_mode_t;
+
+typedef enum {
+ EMBED_POOL_MEAN = 0,
+ EMBED_POOL_LAST_TOKEN = 1,
+} embed_pooling_mode_t;
+
typedef struct {
- int hidden_size; /* 0.6B: 1024 4B: 2560 */
- int n_layers; /* 0.6B: 28 4B: 36 */
- int n_heads; /* 0.6B: 16 4B: 32 */
- int n_kv_heads; /* 0.6B: 8 4B: 8 */
- int head_dim; /* always 128 */
+ int hidden_size;
+ int n_layers;
+ int n_heads;
+ int n_kv_heads;
+ int head_dim;
int q_dim; /* n_heads * head_dim */
int kv_dim; /* n_kv_heads * head_dim */
- int intermediate_size; /* 0.6B: 3072 4B: 9728 */
- int vocab_size; /* 151936 */
- float rms_norm_eps; /* 1e-6 */
- float rope_theta; /* 1e6 */
+ int intermediate_size;
+ int vocab_size;
+ float rms_norm_eps;
+ float rope_theta;
+ embed_attention_mode_t attention_mode;
+ embed_pooling_mode_t pooling_mode;
+ int normalize_embeddings;
+ int append_terminal_token;
+ int terminal_token_id;
} embed_config_t;
/* ========================================================================
@@ -110,13 +125,14 @@ EMBED_API size_t embed_workspace_nbytes(const embed_workspace_t *ws);
* Compute embedding for a token sequence.
*
* 1. Token embedding lookup
- * 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU)
+ * 2. N transformer layers (GQA + RoPE + SwiGLU)
* 3. Final RMSNorm
- * 4. Mean pooling over all positions
+ * 4. Model-defined pooling and optional L2 normalization
*
- * pplx-embed-v1 embeddings are intentionally not L2-normalized. Use cosine
- * similarity for comparisons, or normalize explicitly before storing in vector
- * databases that only support inner product.
+ * pplx-embed-v1 uses bidirectional attention and unnormalized mean pooling.
+ * Qwen3-Embedding uses causal attention and normalized last-token pooling.
+ * Token IDs are consumed exactly as provided; text frontends append any
+ * model-required terminal token before calling this API.
*
* Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/
@@ -145,7 +161,7 @@ EMBED_API int embed_model_encode_batch(const embed_model_t *model, embed_workspa
float *out_embeddings);
/*
- * Mean-pool packed final hidden states for a ragged batch.
+ * Apply the config's pooling mode to packed final hidden states.
*
* states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive
* token count per sequence. The states must already include final RMSNorm.
diff --git a/embed_cli.c b/embed_cli.c
index ebdb223..06b40de 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -16,6 +16,7 @@
#include <string.h>
#include <math.h>
#include <time.h>
+#include <limits.h>
static double now_ms(void)
{
@@ -94,6 +95,8 @@ typedef struct {
qwen_tokenizer_t *tok;
qwen_tokenizer_workspace_t *tok_ws;
int dim;
+ int terminal_token_id;
+ int append_terminal_token;
} engine_t;
typedef struct {
@@ -128,6 +131,22 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
out->n_tokens = 0;
return -1;
}
+ if (e->append_terminal_token) {
+ if (out->n_tokens == INT_MAX) {
+ free(out->ids);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ int *ids = (int *)realloc(
+ out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
+ if (!ids) {
+ free(out->ids);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ out->ids = ids;
+ out->ids[out->n_tokens++] = e->terminal_token_id;
+ }
if (embed_verbose >= 1) {
fprintf(stderr, "tokens (%d): ", out->n_tokens);
@@ -580,6 +599,7 @@ int main(int argc, char *argv[])
engine_t e = {0};
e.tok = tok;
t0 = now_ms();
+ const embed_config_t *config = NULL;
#ifdef USE_MLX
if (use_mlx) {
@@ -593,7 +613,8 @@ int main(int argc, char *argv[])
qwen_tokenizer_free(tok);
return 1;
}
- e.dim = embed_mlx_config(e.mlx_ctx)->hidden_size;
+ config = embed_mlx_config(e.mlx_ctx);
+ e.dim = config->hidden_size;
} else
#endif
#ifdef USE_CUDA
@@ -604,7 +625,8 @@ int main(int argc, char *argv[])
qwen_tokenizer_free(tok);
return 1;
}
- e.dim = embed_cuda_config(e.cuda_ctx)->hidden_size;
+ config = embed_cuda_config(e.cuda_ctx);
+ e.dim = config->hidden_size;
} else
#endif
{
@@ -621,8 +643,11 @@ int main(int argc, char *argv[])
qwen_tokenizer_free(tok);
return 1;
}
- e.dim = embed_model_config(e.model)->hidden_size;
+ config = embed_model_config(e.model);
+ e.dim = config->hidden_size;
}
+ e.append_terminal_token = config->append_terminal_token;
+ e.terminal_token_id = config->terminal_token_id;
if (verbose >= 1)
fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
e.dim, now_ms() - t0,
diff --git a/embed_cuda.cu b/embed_cuda.cu
index 2e4cd28..1ec0573 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -530,6 +530,45 @@ __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L)
store_act(p, (size_t)j, s[j] * inv);
}
+template <typename P_T>
+__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs,
+ int L)
+{
+ float *s = scores + (size_t)blockIdx.x * L;
+ P_T *p = probs + (size_t)blockIdx.x * L;
+ int query = blockIdx.x % L;
+ int keys = query + 1;
+ int tid = threadIdx.x;
+ __shared__ float red[256];
+
+ float m = -3.402823466e+38F;
+ for (int j = tid; j < keys; j += blockDim.x) m = fmaxf(m, s[j]);
+ red[tid] = m;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]);
+ __syncthreads();
+ }
+ m = red[0];
+ __syncthreads();
+
+ float sum = 0.0f;
+ for (int j = tid; j < keys; j += blockDim.x) {
+ float e = __expf(s[j] - m);
+ s[j] = e;
+ sum += e;
+ }
+ red[tid] = sum;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o) red[tid] += red[tid + o];
+ __syncthreads();
+ }
+ float inv = 1.0f / red[0];
+ for (int j = tid; j < L; j += blockDim.x)
+ store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f);
+}
+
__global__ static void mean_pool_kernel(float *out, const float *x,
const int *offsets, int batch,
int hidden)
@@ -547,6 +586,17 @@ __global__ static void mean_pool_kernel(float *out, const float *x,
}
}
+__global__ static void last_pool_kernel(float *out, const float *x,
+ const int *offsets, int batch,
+ int hidden)
+{
+ int b = blockIdx.x;
+ int tid = threadIdx.x;
+ int last = offsets[b + 1] - 1;
+ for (int d = tid; d < hidden; d += blockDim.x)
+ out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d];
+}
+
__global__ static void span_pool_kernel(float *out, const float *x,
const int *starts, const int *lens,
int n_spans, int hidden)
@@ -783,6 +833,7 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
int hd = c->head_dim;
int H = c->n_heads;
int total = h_offsets[batch];
+ int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
const float alpha = scale, beta = 0.0f, one = 1.0f;
int threads = 256;
@@ -823,11 +874,21 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
&beta, S + off, CUDA_R_32F, L,
n, gemm_compute(), CUBLAS_GEMM_DEFAULT));
if (out_bf16) {
- attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
+ if (causal)
+ attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores,
+ (__nv_bfloat16 *)ctx->attn_probs, L);
+ else
+ attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores,
+ (__nv_bfloat16 *)ctx->attn_probs, L);
} else {
- attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
+ if (causal)
+ attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, ctx->attn_scores, L);
+ else
+ attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, ctx->attn_scores, L);
}
if (launch_check() != 0) return -1;
/* O = P @ V, batched over (seq, head) */
@@ -857,8 +918,12 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
const __nv_bfloat16 *V =
(const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim;
__nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim;
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, P, L);
+ if (causal)
+ attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, P, L);
+ else
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, P, L);
if (launch_check() != 0) return -1;
/* O[h] = P[h] @ V[h] (row-major C = A @ B) */
CUBLAS_CHECK(cublasGemmStridedBatchedEx(
@@ -870,8 +935,12 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets,
} else {
const float *V = ctx->vexp + (size_t)start * q_dim;
float *O = ctx->attn_out + (size_t)start * q_dim;
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, ctx->attn_scores, L);
+ if (causal)
+ attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, ctx->attn_scores, L);
+ else
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, ctx->attn_scores, L);
if (launch_check() != 0) return -1;
/* O[h] = P[h] @ V[h] (row-major C = A @ B) */
if (gemm_strided_batched(
@@ -1308,8 +1377,12 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs,
return -1;
if (ensure_pooled_rows(ctx, batch) != 0)
return -1;
- mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
- ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size);
+ if (c->pooling_mode == EMBED_POOL_LAST_TOKEN)
+ last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
+ ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size);
+ else
+ mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
+ ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size);
if (launch_check() != 0) return -1;
cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
(size_t)batch * c->hidden_size * sizeof(float),
@@ -1320,6 +1393,14 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs,
fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce));
return -1;
}
+ if (c->normalize_embeddings) {
+ for (int b = 0; b < batch; b++) {
+ if (embed_l2_normalize(
+ out_embeddings + (size_t)b * c->hidden_size,
+ c->hidden_size) != 0)
+ return -1;
+ }
+ }
return 0;
}
diff --git a/embed_internal.h b/embed_internal.h
index a7365a2..60fc581 100644
--- a/embed_internal.h
+++ b/embed_internal.h
@@ -29,12 +29,18 @@ typedef struct {
const float *norm; /* [hidden] */
} embed_weights_t;
+typedef void (*embed_attention_fn)(
+ float *out, const float *Q, const float *K, const float *V,
+ const int *offsets, int batch, int n_heads, int n_kv_heads,
+ int head_dim, float scale, float *scratch, size_t scratch_bytes);
+
struct embed_model {
embed_config_t config;
embed_weights_t weights;
void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
int layer_start; /* loaded transformer range, inclusive */
int layer_end; /* loaded transformer range, exclusive */
+ embed_attention_fn attention;
};
struct embed_workspace {
diff --git a/embed_mlx.c b/embed_mlx.c
index 5de4068..cbdfa33 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -358,6 +358,16 @@ static const char *json_find_m(const char *json, const char *key)
return NULL;
}
+static int json_string_equals_m(const char *json, const char *key,
+ const char *expected)
+{
+ const char *v = json_find_m(json, key);
+ if (!v || *v != '"') return 0;
+ v++;
+ size_t len = strlen(expected);
+ return strncmp(v, expected, len) == 0 && v[len] == '"';
+}
+
static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
{
char path[1024];
@@ -394,6 +404,19 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
GF("rope_theta", rope_theta, 1000000.0f);
#undef GI
#undef GF
+ cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
+ cfg->pooling_mode = EMBED_POOL_MEAN;
+ cfg->normalize_embeddings = 0;
+ cfg->append_terminal_token = 0;
+ cfg->terminal_token_id = -1;
+ if (json_string_equals_m(buf, "model_type", "qwen3")) {
+ const char *eos = json_find_m(buf, "eos_token_id");
+ cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
+ cfg->pooling_mode = EMBED_POOL_LAST_TOKEN;
+ cfg->normalize_embeddings = 1;
+ cfg->append_terminal_token = 1;
+ cfg->terminal_token_id = eos ? atoi(eos) : -1;
+ }
cfg->q_dim = 0;
cfg->kv_dim = 0;
@@ -411,7 +434,10 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
(cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
!isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
+ !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f ||
+ (cfg->append_terminal_token &&
+ (cfg->terminal_token_id < 0 ||
+ cfg->terminal_token_id >= cfg->vocab_size))) {
fprintf(stderr, "mlx: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
@@ -782,6 +808,8 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
int q_dim = c->q_dim;
float scale = 1.0f / sqrtf((float)head_dim);
mlx_array null_arr = (mlx_array){0};
+ const char *mask_mode =
+ c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
/* For non-quantized BF16 weights, run the residual stream, norms, and
* attention in F32; matmuls downcast their operands to BF16 internally
@@ -850,7 +878,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_array attn = mlx_array_new();
mlx_fast_scaled_dot_product_attention(
- &attn, qr, kr, vt, scale, "",
+ &attn, qr, kr, vt, scale, mask_mode,
has_padding ? local_mask : null_arr, null_arr, S);
mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
@@ -1373,6 +1401,10 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
int hidden = c->hidden_size;
int max_seq = 0;
int has_padding = 0;
+ int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
+ int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN;
+ if (hidden <= 0 || c->vocab_size <= 0)
+ return -1;
for (int b = 0; b < batch; b++) {
if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1;
@@ -1383,26 +1415,58 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
for (int b = 0; b < batch; b++) {
if (inputs[b].n_tokens != max_seq) has_padding = 1;
}
+ int needs_pool_mask = has_padding && !last_token_pool;
+ int needs_attn_mask = has_padding && !causal;
size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes;
+ size_t last_index_bytes = 0;
if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 ||
mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 ||
mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 ||
mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 ||
mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 ||
- mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
+ mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 ||
+ (last_token_pool &&
+ mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
+ return -1;
+ if (ids_bytes == 0 || out_bytes == 0 ||
+ (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) ||
+ (needs_attn_mask && mask_bytes == 0) ||
+ (last_token_pool && last_index_bytes == 0))
return -1;
int *padded_ids = (int *)malloc(ids_bytes);
- float *pool_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
- float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
- float *len_data = has_padding ? (float *)malloc(len_bytes) : NULL;
- if (!padded_ids || (has_padding && (!pool_mask_data || !attn_mask_data || !len_data))) {
- free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+ float *pool_mask_data =
+ needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
+ float *attn_mask_data =
+ needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
+ float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL;
+ int *last_index_data =
+ last_token_pool ? (int *)malloc(last_index_bytes) : NULL;
+ if (!padded_ids ||
+ (needs_pool_mask && (!pool_mask_data || !len_data)) ||
+ (needs_attn_mask && !attn_mask_data)) {
+ free(padded_ids);
+ free(pool_mask_data);
+ free(attn_mask_data);
+ free(len_data);
+ free(last_index_data);
+ return -1;
+ }
+ if (last_token_pool && !last_index_data) {
+ free(padded_ids);
+ free(pool_mask_data);
+ free(attn_mask_data);
+ free(len_data);
return -1;
}
for (int b = 0; b < batch; b++) {
- if (has_padding) len_data[b] = (float)inputs[b].n_tokens;
+ if (needs_pool_mask) len_data[b] = (float)inputs[b].n_tokens;
+ if (last_token_pool) {
+ int index = inputs[b].n_tokens - 1;
+ for (int d = 0; d < hidden; d++)
+ last_index_data[(size_t)b * hidden + d] = index;
+ }
for (int t = 0; t < max_seq; t++) {
size_t idx = (size_t)b * max_seq + t;
if (t < inputs[b].n_tokens) {
@@ -1410,19 +1474,24 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
if (id < 0 || id >= c->vocab_size) {
fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
id, b, t);
- free(padded_ids); free(pool_mask_data);
- free(attn_mask_data); free(len_data);
+ free(padded_ids);
+ free(pool_mask_data);
+ free(attn_mask_data);
+ free(len_data);
+ free(last_index_data);
return -1;
}
padded_ids[idx] = id;
- if (has_padding) {
+ if (needs_pool_mask)
pool_mask_data[idx] = 1.0f;
+ if (needs_attn_mask)
attn_mask_data[idx] = 0.0f;
- }
} else {
padded_ids[idx] = 0;
- pool_mask_data[idx] = 0.0f;
- attn_mask_data[idx] = -1.0e9f;
+ if (needs_pool_mask)
+ pool_mask_data[idx] = 0.0f;
+ if (needs_attn_mask)
+ attn_mask_data[idx] = -1.0e9f;
}
}
}
@@ -1431,33 +1500,42 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
int pool_mask_shape[] = {batch, max_seq, 1};
int attn_mask_shape[] = {batch, 1, 1, max_seq};
int len_shape[] = {batch, 1};
+ int last_index_shape[] = {batch, 1, hidden};
mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- mlx_array pool_mask = has_padding
+ mlx_array pool_mask = needs_pool_mask
? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
: (mlx_array){0};
- mlx_array attn_mask = has_padding
+ mlx_array attn_mask = needs_attn_mask
? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
: (mlx_array){0};
- mlx_array lengths = has_padding
+ mlx_array lengths = needs_pool_mask
? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32)
: (mlx_array){0};
+ mlx_array last_indices = last_token_pool
+ ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
+ : (mlx_array){0};
free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+ free(last_index_data);
- if (!arr_ok(ids) || (has_padding && (!arr_ok(pool_mask) ||
- !arr_ok(attn_mask) || !arr_ok(lengths)))) {
+ if (!arr_ok(ids) ||
+ (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
+ (needs_attn_mask && !arr_ok(attn_mask)) ||
+ (last_token_pool && !arr_ok(last_indices))) {
mlx_array_free(ids); mlx_array_free(pool_mask);
- if (has_padding) mlx_array_free(attn_mask);
+ mlx_array_free(attn_mask);
mlx_array_free(lengths);
+ mlx_array_free(last_indices);
return -1;
}
- if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
+ if (needs_attn_mask && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
mlx_array attn_mask_bf16 = mlx_array_new();
mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
mlx_array_free(attn_mask);
attn_mask = attn_mask_bf16;
if (!arr_ok(attn_mask)) {
mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths);
+ mlx_array_free(last_indices);
return -1;
}
}
@@ -1468,7 +1546,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(ids);
/* 2. Transformer layers. */
- x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
+ x = mlx_forward_layers(ctx, x, batch, max_seq,
+ needs_attn_mask, attn_mask,
0, c->n_layers);
/* 3. Final RMSNorm. */
@@ -1477,7 +1556,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(x);
mlx_array emb = mlx_array_new();
- if (has_padding) {
+ if (last_token_pool) {
+ mlx_array selected = mlx_array_new();
+ mlx_take_along_axis(&selected, x_normed, last_indices, 1, S);
+ mlx_array_free(x_normed);
+ mlx_array_free(last_indices);
+ int emb_shape[] = {batch, hidden};
+ mlx_reshape(&emb, selected, emb_shape, 2, S);
+ mlx_array_free(selected);
+ mlx_array_free(pool_mask);
+ mlx_array_free(attn_mask);
+ mlx_array_free(lengths);
+ } else if (has_padding) {
/* 4. Masked mean pool over T. */
mlx_array masked = mlx_array_new();
mlx_multiply(&masked, x_normed, pool_mask, S);
@@ -1508,7 +1598,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
int rc = -1;
if (data) {
memcpy(out_embeddings, data, out_bytes);
- rc = 0;
+ if (c->normalize_embeddings) {
+ rc = 0;
+ for (int b = 0; b < batch; b++) {
+ if (embed_l2_normalize(
+ out_embeddings + (size_t)b * hidden, hidden) != 0) {
+ rc = -1;
+ break;
+ }
+ }
+ } else {
+ rc = 0;
+ }
}
mlx_array_free(emb_f32);
return rc;
diff --git a/embed_mlx.h b/embed_mlx.h
index 267c454..5f6e2c9 100644
--- a/embed_mlx.h
+++ b/embed_mlx.h
@@ -108,7 +108,7 @@ EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids,
* Compute a true padded dense batch on MLX.
*
* out_embeddings is caller-provided [batch, hidden_size].
- * Padded tokens are masked out of attention keys and mean pooling.
+ * Pooling and attention masking follow the loaded model configuration.
* Returns 0 on success, -1 on error.
*/
EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
diff --git a/embed_server.c b/embed_server.c
index 7388a4d..1d64e1d 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -205,6 +205,7 @@ static void sbuf_free(sbuf *b) {
typedef enum {
MODEL_STD_06,
MODEL_STD_4,
+ MODEL_QWEN3_06,
MODEL_CTX_06,
MODEL_CTX_4,
MODEL_LATE_06,
@@ -222,16 +223,27 @@ typedef struct {
const char *id;
model_kind kind;
int dim;
+ int min_dim;
int token_dim;
double price_per_mtok;
+ embed_attention_mode_t attention_mode;
+ embed_pooling_mode_t pooling_mode;
+ int normalize_embeddings;
} model_info;
static const model_info k_models[] = {
- {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 0, 0.004},
- {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 0, 0.030},
- {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 0, 0.008},
- {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 0, 0.050},
- {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 0.0},
+ {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, 0.004,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0},
+ {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, 0.030,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0},
+ {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, 0.0,
+ EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1},
+ {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, 0.008,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0},
+ {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, 0.050,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0},
+ {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, 0.0,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0},
};
static model_slot model_slot_for_id(const char *id) {
@@ -253,6 +265,9 @@ typedef struct {
int late_document_prefix_id;
int late_skip_ids[64];
int n_late_skip_ids;
+ int append_terminal_token;
+ int terminal_token_id;
+ int renormalize_truncated;
embed_model_t *cpu_model;
embed_workspace_t *cpu_ws;
embed_late_model_t *cpu_late_model;
@@ -1100,8 +1115,8 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail) {
return encoding->valuestring;
}
-static int dimensions_from_root(cJSON *root, cJSON *detail, int max_dim,
- const char *encoding) {
+static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim,
+ int max_dim, const char *encoding) {
int dims = max_dim;
cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions");
if (dimensions) {
@@ -1110,9 +1125,11 @@ static int dimensions_from_root(cJSON *root, cJSON *detail, int max_dim,
"type_error.integer");
} else {
dims = dimensions->valueint;
- if (dims < 128 || dims > max_dim) {
+ if (dims < min_dim || dims > max_dim) {
char msg[96];
- snprintf(msg, sizeof(msg), "dimensions must be between 128 and %d", max_dim);
+ snprintf(msg, sizeof(msg),
+ "dimensions must be between %d and %d",
+ min_dim, max_dim);
ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range");
}
}
@@ -1285,6 +1302,22 @@ static int tokenize_one(loaded_model *m, job *j, const char *text,
memset(out, 0, sizeof(*out));
return -1;
}
+ if (m->append_terminal_token) {
+ if (out->n_tokens == INT_MAX) {
+ free(out->ids);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ int *ids = realloc(
+ out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
+ if (!ids) {
+ free(out->ids);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ out->ids = ids;
+ out->ids[out->n_tokens++] = m->terminal_token_id;
+ }
return 0;
}
@@ -1402,12 +1435,32 @@ static void set_response_from_buf(job *j, sbuf *b) {
memset(b, 0, sizeof(*b));
}
-static void render_embedding_response(embedding_request *r, const float *embs) {
+static int render_embedding_response(embedding_request *r, const float *embs) {
+ const int full_dim = r->model->info->dim;
+ const float *render_embs = embs;
+ float *truncated = NULL;
+ if (r->model->renormalize_truncated && r->dims < full_dim) {
+ truncated = xmalloc(
+ (size_t)r->n_inputs * r->dims * sizeof(*truncated));
+ for (int i = 0; i < r->n_inputs; i++) {
+ float *dst = truncated + (size_t)i * r->dims;
+ memcpy(dst, embs + (size_t)i * full_dim,
+ (size_t)r->dims * sizeof(*dst));
+ if (embed_l2_normalize(dst, r->dims) != 0) {
+ free(truncated);
+ return -1;
+ }
+ }
+ render_embs = truncated;
+ }
+ int render_stride = truncated ? r->dims : full_dim;
+
char **encoded = NULL;
if (strcmp(r->encoding, "float")) {
encoded = xcalloc((size_t)r->n_inputs, sizeof(*encoded));
for (int i = 0; i < r->n_inputs; i++)
- encoded[i] = encode_embedding(embs + (size_t)i * r->model->info->dim,
+ encoded[i] = encode_embedding(render_embs +
+ (size_t)i * render_stride,
r->dims, r->encoding);
}
@@ -1419,7 +1472,7 @@ static void render_embedding_response(embedding_request *r, const float *embs) {
append_embedding_object(&b, i, encoded[i]);
else
append_float_embedding_object(&b, i,
- embs + (size_t)i * r->model->info->dim, r->dims);
+ render_embs + (size_t)i * render_stride, r->dims);
}
double cost = ((double)r->total_tokens / 1000000.0) *
r->model->info->price_per_mtok;
@@ -1432,6 +1485,8 @@ static void render_embedding_response(embedding_request *r, const float *embs) {
for (int i = 0; encoded && i < r->n_inputs; i++) free(encoded[i]);
free(encoded);
+ free(truncated);
+ return 0;
}
static int embedding_request_compatible(const embedding_request *a,
@@ -1525,7 +1580,10 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
} else {
uint64_t t0 = nstime();
- render_embedding_response(reqs[i], embs + (size_t)pos * dim);
+ if (render_embedding_response(
+ reqs[i], embs + (size_t)pos * dim) != 0)
+ job_set_error(reqs[i]->j, 500, "embedding normalization failed",
+ "server_error");
reqs[i]->j->encode_ns += nstime() - t0;
}
pos += reqs[i]->n_inputs;
@@ -1736,9 +1794,10 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind,
}
}
const char *enc = encoding_from_root(root, detail);
- int max_dim = model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN
- ? k_models[model_slot_for_id(model_id)].dim : 2560;
- int dims = dimensions_from_root(root, detail, max_dim, enc);
+ model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN;
+ int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128;
+ int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560;
+ int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc);
*out_encoding = enc;
*out_dims = dims;
return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN &&
@@ -2502,6 +2561,34 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) {
free(std_reqs);
}
+static int configure_loaded_model(loaded_model *m,
+ const embed_config_t *config,
+ int token_dim)
+{
+ if (!m || !m->info || !config ||
+ config->hidden_size != m->info->dim ||
+ token_dim != m->info->token_dim ||
+ config->attention_mode != m->info->attention_mode ||
+ config->pooling_mode != m->info->pooling_mode ||
+ config->normalize_embeddings != m->info->normalize_embeddings) {
+ if (m && m->info && config)
+ server_log("embed-server: model %s has incompatible config",
+ m->info->id);
+ return -1;
+ }
+ if (config->append_terminal_token &&
+ (config->terminal_token_id < 0 ||
+ config->terminal_token_id >= config->vocab_size))
+ return -1;
+
+ m->append_terminal_token = config->append_terminal_token;
+ m->terminal_token_id = config->terminal_token_id;
+ m->renormalize_truncated =
+ config->normalize_embeddings &&
+ config->pooling_mode == EMBED_POOL_LAST_TOKEN;
+ return 0;
+}
+
static void *worker_main(void *arg) {
http_server *s = arg;
#ifdef USE_MLX
@@ -2531,14 +2618,11 @@ static void *worker_main(void *arg) {
: embed_mlx_config(m->mlx_ctx);
int token_dim = m->info->kind == MODEL_KIND_LATE
? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0;
- if (config->hidden_size != m->info->dim ||
- token_dim != m->info->token_dim ||
+ if (configure_loaded_model(m, config, token_dim) != 0 ||
(m->info->kind == MODEL_KIND_LATE &&
(m->late_mask_id >= config->vocab_size ||
m->late_query_prefix_id >= config->vocab_size ||
m->late_document_prefix_id >= config->vocab_size))) {
- server_log("embed-server: model %s has unexpected dimension %d",
- m->info->id, config->hidden_size);
rc = 1;
break;
}
@@ -2578,10 +2662,8 @@ static void *worker_main(void *arg) {
rc = 1;
break;
}
- if (embed_cuda_config(m->cuda_ctx)->hidden_size != m->info->dim) {
- server_log("embed-server: model %s has unexpected dimension %d",
- m->info->id,
- embed_cuda_config(m->cuda_ctx)->hidden_size);
+ if (configure_loaded_model(
+ m, embed_cuda_config(m->cuda_ctx), 0) != 0) {
rc = 1;
break;
}
@@ -2665,6 +2747,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
loaded_model *m = &s->models[slot];
m->info = &k_models[slot];
m->path = xstrdup(path);
+ m->terminal_token_id = -1;
char vocab_path[1024];
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path);
m->tok = qwen_tokenizer_load(vocab_path);
@@ -2751,10 +2834,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
}
config = embed_model_config(m->cpu_model);
}
- if (config->hidden_size != m->info->dim ||
- token_dim != m->info->token_dim) {
- server_log("embed-server: model %s has unexpected dimension %d",
- m->info->id, config->hidden_size);
+ if (configure_loaded_model(m, config, token_dim) != 0) {
return -1;
}
if (m->info->kind == MODEL_KIND_LATE &&
diff --git a/qwen_kernels.c b/qwen_kernels.c
index 85871d4..dcd724c 100644
--- a/qwen_kernels.c
+++ b/qwen_kernels.c
@@ -1015,6 +1015,158 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
free(scores);
}
+static void qwen_causal_gqa_attention_packed_online_rows(
+ float *out, const float *Q, const float *K, const float *V,
+ int start, int n_heads, int n_kv_heads, int head_dim,
+ float scale, int h, int query_start, int query_end) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+
+ for (int i = query_start; i < query_end; i++) {
+ const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
+ float *o_row = out + (size_t)i * q_hidden + h * head_dim;
+
+ float max_score = -1e30f;
+ float sum_exp = 0.0f;
+ for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f;
+
+ for (int j = start; j <= i; j++) {
+ const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
+ const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
+ float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale;
+
+ if (score > max_score) {
+ float correction = expf(max_score - score);
+ sum_exp = sum_exp * correction + 1.0f;
+ qwen_vec_scale_add(o_row, v_row, correction, head_dim);
+ max_score = score;
+ } else {
+ float wt = expf(score - max_score);
+ sum_exp += wt;
+ qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ }
+ }
+
+ if (sum_exp > 0.0f)
+ qwen_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim);
+ }
+}
+
+#ifdef USE_BLAS
+static void qwen_causal_gqa_attention_packed_blas_rows(
+ float *out, const float *Q, const float *K, const float *V,
+ int start, int n_heads, int n_kv_heads, int head_dim,
+ float scale, int h, int query_start, int query_end, float *scores) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+ int rows = query_end - query_start;
+ int keys = query_end - start;
+
+ const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
+ const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
+ const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
+ float *o = out + (size_t)query_start * q_hidden + h * head_dim;
+
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
+ rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden,
+ 0.0f, scores, keys);
+ for (int r = 0; r < rows; r++) {
+ int allowed = query_start + r - start + 1;
+ float *row = scores + (size_t)r * keys;
+ for (int j = allowed; j < keys; j++) row[j] = -INFINITY;
+ }
+ qwen_softmax(scores, rows, keys);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans,
+ rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden,
+ 0.0f, o, q_hidden);
+}
+#endif
+
+static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) {
+ packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
+ int task_id = 0;
+#ifdef USE_BLAS
+ float *scores = t->scores
+ ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq
+ : NULL;
+#endif
+
+ for (int h = 0; h < t->n_heads; h++) {
+ for (int b = 0; b < t->batch; b++) {
+ int start = t->offsets[b];
+ int end = t->offsets[b + 1];
+ for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) {
+ int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE;
+ if (q1 > end) q1 = end;
+ if (task_id++ % n_threads != tid) continue;
+#ifdef USE_BLAS
+ if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) {
+ qwen_causal_gqa_attention_packed_blas_rows(
+ t->out, t->Q, t->K, t->V, start, t->n_heads,
+ t->n_kv_heads, t->head_dim, t->scale, h, q0, q1,
+ scores);
+ continue;
+ }
+#endif
+ qwen_causal_gqa_attention_packed_online_rows(
+ t->out, t->Q, t->K, t->V, start, t->n_heads,
+ t->n_kv_heads, t->head_dim, t->scale, h, q0, q1);
+ }
+ }
+ }
+}
+
+void qwen_causal_gqa_attention_packed_with_scratch(
+ float *out, const float *Q, const float *K, const float *V,
+ const int *offsets, int batch, int n_heads, int n_kv_heads,
+ int head_dim, float scale, float *scratch, size_t scratch_bytes) {
+ long long qk_work = 0;
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ qk_work += (long long)len * (len + 1) / 2 * n_heads;
+ if (len > max_seq) max_seq = len;
+ }
+
+ float *scores = NULL;
+#ifdef USE_BLAS
+ size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(
+ offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required) scores = scratch;
+#else
+ (void)scratch;
+ (void)scratch_bytes;
+#endif
+ packed_gqa_attn_task_t task = {
+ .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets,
+ .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim, .scale = scale, .scores = scores,
+ .max_seq = max_seq
+ };
+ if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096)
+ parallel_for(packed_causal_gqa_attn_worker, &task);
+ else
+ packed_causal_gqa_attn_worker(0, 1, &task);
+}
+
+void qwen_causal_gqa_attention_packed(float *out, const float *Q,
+ const float *K, const float *V,
+ const int *offsets, int batch,
+ int n_heads, int n_kv_heads,
+ int head_dim, float scale) {
+ size_t scratch_bytes =
+ qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
+ qwen_causal_gqa_attention_packed_with_scratch(
+ out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale,
+ scores, scratch_bytes);
+ free(scores);
+}
+
/* ========================================================================
* Position Embeddings
* ======================================================================== */
diff --git a/qwen_kernels.h b/qwen_kernels.h
index 71f77f1..c611194 100644
--- a/qwen_kernels.h
+++ b/qwen_kernels.h
@@ -116,6 +116,20 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch(
const int *offsets, int batch, int n_heads, int n_kv_heads,
int head_dim, float scale, float *scratch, size_t scratch_bytes);
+/*
+ * Causal GQA attention over a packed/ragged batch. Query position i attends
+ * only to positions at or before i within its sequence.
+ */
+void qwen_causal_gqa_attention_packed(float *out, const float *Q,
+ const float *K, const float *V,
+ const int *offsets, int batch,
+ int n_heads, int n_kv_heads,
+ int head_dim, float scale);
+void qwen_causal_gqa_attention_packed_with_scratch(
+ float *out, const float *Q, const float *K, const float *V,
+ const int *offsets, int batch, int n_heads, int n_kv_heads,
+ int head_dim, float scale, float *scratch, size_t scratch_bytes);
+
/* ========================================================================
* Position Embeddings
* ======================================================================== */
diff --git a/tests/test_kernels.c b/tests/test_kernels.c
index 208dd02..4a2a57d 100644
--- a/tests/test_kernels.c
+++ b/tests/test_kernels.c
@@ -95,7 +95,7 @@ static void test_rope_neox(void) {
static void reference_packed_gqa(float *out, const float *Q, const float *K,
const float *V, const int *offsets, int batch,
int n_heads, int n_kv_heads, int head_dim,
- float scale) {
+ float scale, int causal) {
int q_hidden = n_heads * head_dim;
int kv_hidden = n_kv_heads * head_dim;
int heads_per_kv = n_heads / n_kv_heads;
@@ -105,10 +105,11 @@ static void reference_packed_gqa(float *out, const float *Q, const float *K,
for (int b = 0; b < batch; b++) {
int start = offsets[b], end = offsets[b + 1];
for (int i = start; i < end; i++) {
+ int key_end = causal ? i + 1 : end;
const float *q = Q + i * q_hidden + h * head_dim;
float *o = out + i * q_hidden + h * head_dim;
float max_score = -INFINITY;
- for (int j = start; j < end; j++) {
+ for (int j = start; j < key_end; j++) {
const float *k = K + j * kv_hidden + kv_h * head_dim;
float score = 0.0f;
for (int d = 0; d < head_dim; d++) score += q[d] * k[d];
@@ -117,7 +118,7 @@ static void reference_packed_gqa(float *out, const float *Q, const float *K,
}
float denom = 0.0f;
for (int d = 0; d < head_dim; d++) o[d] = 0.0f;
- for (int j = start; j < end; j++) {
+ for (int j = start; j < key_end; j++) {
const float *k = K + j * kv_hidden + kv_h * head_dim;
const float *v = V + j * kv_hidden + kv_h * head_dim;
float score = 0.0f;
@@ -155,7 +156,7 @@ static void test_packed_gqa_attention(void) {
1.0f / sqrtf((float)head_dim));
reference_packed_gqa(want, Q, K, V, offsets, 2,
n_heads, n_kv_heads, head_dim,
- 1.0f / sqrtf((float)head_dim));
+ 1.0f / sqrtf((float)head_dim), 0);
for (int i = 0; i < total * q_hidden; i++)
expect_close("packed_gqa", got[i], want[i], 2e-6f);
@@ -186,7 +187,7 @@ static void test_packed_gqa_attention_long(void) {
1.0f / sqrtf((float)head_dim));
reference_packed_gqa(want, Q, K, V, offsets, 2,
n_heads, n_kv_heads, head_dim,
- 1.0f / sqrtf((float)head_dim));
+ 1.0f / sqrtf((float)head_dim), 0);
qwen_set_threads(1);
for (int i = 0; i < total * q_hidden; i++)
@@ -198,6 +199,43 @@ static void test_packed_gqa_attention_long(void) {
free(Q);
}
+static void test_packed_causal_gqa_attention(void) {
+ enum { total = 136, n_heads = 4, n_kv_heads = 2, head_dim = 8 };
+ enum { q_hidden = n_heads * head_dim };
+ enum { kv_hidden = n_kv_heads * head_dim };
+ const int offsets[3] = {0, 7, 136};
+ float *Q = malloc((size_t)total * q_hidden * sizeof(*Q));
+ float *K = malloc((size_t)total * kv_hidden * sizeof(*K));
+ float *V = malloc((size_t)total * kv_hidden * sizeof(*V));
+ float *got = malloc((size_t)total * q_hidden * sizeof(*got));
+ float *want = malloc((size_t)total * q_hidden * sizeof(*want));
+ if (!Q || !K || !V || !got || !want) exit(2);
+
+ for (int i = 0; i < total * q_hidden; i++)
+ Q[i] = sinf((float)i * 0.019f) - 0.2f;
+ for (int i = 0; i < total * kv_hidden; i++) {
+ K[i] = cosf((float)i * 0.013f) + 0.15f;
+ V[i] = sinf((float)i * 0.009f) - cosf((float)i * 0.015f);
+ }
+
+ qwen_set_threads(4);
+ qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2,
+ n_heads, n_kv_heads, head_dim,
+ 1.0f / sqrtf((float)head_dim));
+ reference_packed_gqa(want, Q, K, V, offsets, 2,
+ n_heads, n_kv_heads, head_dim,
+ 1.0f / sqrtf((float)head_dim), 1);
+ qwen_set_threads(1);
+
+ for (int i = 0; i < total * q_hidden; i++)
+ expect_close("packed_causal_gqa", got[i], want[i], 2e-5f);
+ free(want);
+ free(got);
+ free(V);
+ free(K);
+ free(Q);
+}
+
static void test_bf16_linear(void) {
enum { seq = 2, in_dim = 3, out_dim = 2 };
const float x[6] = {
@@ -492,6 +530,7 @@ int main(void) {
test_rope_neox();
test_packed_gqa_attention();
test_packed_gqa_attention_long();
+ test_packed_causal_gqa_attention();
test_bf16_linear();
test_bf16_qkv();
test_bf16_matvec();
diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c
new file mode 100644
index 0000000..8205ca1
--- /dev/null
+++ b/tests/test_qwen3.c
@@ -0,0 +1,92 @@
+/* tests/test_qwen3.c - hermetic Qwen3 model semantics. */
+
+#include "embed.h"
+#include "tiny_model.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static float max_abs_diff(const float *a, const float *b, int n)
+{
+ float out = 0.0f;
+ for (int i = 0; i < n; i++) {
+ float d = fabsf(a[i] - b[i]);
+ if (d > out) out = d;
+ }
+ return out;
+}
+
+int main(void)
+{
+ char dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-qwen3-test-XXXXXX",
+ getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
+ tm_dims_t dims = {4, 2, 1, 2, 8, 16};
+ if (!mkdtemp(dir) ||
+ tm_write_qwen3_model_dims(dir, "F32", &dims, 15) != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ embed_model_t *model = embed_model_load(dir);
+ embed_workspace_t *ws = embed_workspace_new(model);
+ const embed_config_t *cfg = embed_model_config(model);
+ if (!model || !ws || !cfg) {
+ fprintf(stderr, "model setup failed\n");
+ return 1;
+ }
+ if (cfg->attention_mode != EMBED_ATTENTION_CAUSAL ||
+ cfg->pooling_mode != EMBED_POOL_LAST_TOKEN ||
+ !cfg->normalize_embeddings || !cfg->append_terminal_token ||
+ cfg->terminal_token_id != 15) {
+ fprintf(stderr, "Qwen3 config semantics were not detected\n");
+ return 1;
+ }
+
+ int ids[] = {1, 2, 3, 15};
+ float states[4 * 4];
+ float embedding[4];
+ float pooled[4];
+ if (embed_model_forward_into(model, ws, ids, 4, states) != 0 ||
+ embed_model_encode_into(model, ws, ids, 4, embedding) != 0 ||
+ embed_pool_batch(cfg, states, (int[]){4}, 1, pooled) != 0) {
+ fprintf(stderr, "Qwen3 inference failed\n");
+ return 1;
+ }
+
+ float expected[4];
+ memcpy(expected, states + 3 * 4, sizeof(expected));
+ if (embed_l2_normalize(expected, 4) != 0 ||
+ max_abs_diff(embedding, expected, 4) > 1e-5f ||
+ max_abs_diff(pooled, expected, 4) > 1e-5f) {
+ fprintf(stderr, "last-token pooling mismatch\n");
+ return 1;
+ }
+
+ float norm_sq = 0.0f;
+ for (int d = 0; d < 4; d++) norm_sq += embedding[d] * embedding[d];
+ if (fabsf(norm_sq - 1.0f) > 1e-5f) {
+ fprintf(stderr, "Qwen3 embedding is not normalized: %.9g\n", norm_sq);
+ return 1;
+ }
+
+ int short_ids[] = {4, 15};
+ embed_input_t inputs[] = {{ids, 4}, {short_ids, 2}};
+ float batch[8];
+ float single[4];
+ if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 ||
+ embed_model_encode_into(model, ws, short_ids, 2, single) != 0 ||
+ max_abs_diff(batch, embedding, 4) > 1e-5f ||
+ max_abs_diff(batch + 4, single, 4) > 1e-5f) {
+ fprintf(stderr, "Qwen3 packed batch parity failed\n");
+ return 1;
+ }
+
+ embed_workspace_free(ws);
+ embed_model_free(model);
+ puts("ok: Qwen3 causal attention and last-token pooling");
+ return 0;
+}
diff --git a/tests/test_server.c b/tests/test_server.c
index 113b4d7..bb4d93b 100644
--- a/tests/test_server.c
+++ b/tests/test_server.c
@@ -289,7 +289,7 @@ static int http_req(int port, const char *method, const char *path,
}
typedef struct {
- embed_server_model_spec_t spec[3];
+ embed_server_model_spec_t spec[4];
embed_server_config_t cfg;
int rc;
} srv_ctx;
@@ -470,6 +470,51 @@ static void test_http_embeddings(int port)
free(body);
body = NULL;
+ /* Qwen3 appends its terminal token, supports 32D MRL output, and
+ * re-normalizes the truncated prefix. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"Qwen3-Embedding-0.6B\","
+ "\"input\":\"hello\",\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ cJSON *qwen_full = body ? parse_embeddings_body(body, 1) : NULL;
+ free(body);
+ body = NULL;
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"Qwen3-Embedding-0.6B\","
+ "\"input\":\"hello\",\"dimensions\":32,"
+ "\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ cJSON *qwen_mrl = body ? parse_embeddings_body(body, 1) : NULL;
+ if (qwen_full && qwen_mrl) {
+ cJSON *full_data =
+ cJSON_GetObjectItemCaseSensitive(qwen_full, "data");
+ cJSON *mrl_data =
+ cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data");
+ cJSON *full_emb = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(full_data, 0), "embedding");
+ cJSON *mrl_emb = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(mrl_data, 0), "embedding");
+ TEST_ASSERT(cJSON_IsArray(full_emb) &&
+ cJSON_GetArraySize(full_emb) == 1024);
+ TEST_ASSERT(cJSON_IsArray(mrl_emb) &&
+ cJSON_GetArraySize(mrl_emb) == 32);
+ double prefix_diff = 0.0;
+ for (int i = 0; i < 32; i++) {
+ double d = fabs(cJSON_GetArrayItem(full_emb, i)->valuedouble -
+ cJSON_GetArrayItem(mrl_emb, i)->valuedouble);
+ if (d > prefix_diff) prefix_diff = d;
+ }
+ TEST_ASSERT(prefix_diff > 0.0);
+ cJSON *usage = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "usage");
+ cJSON *total = usage
+ ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
+ TEST_ASSERT(total && total->valueint == 2);
+ }
+ cJSON_Delete(qwen_full);
+ cJSON_Delete(qwen_mrl);
+ free(body);
+ body = NULL;
+
/* Concurrent clients: more requests in flight than the micro-batch cap
* (-b 2) so the worker has to assemble several batches. */
enum { N_CONC = 6 };
@@ -713,7 +758,7 @@ static void test_http_rerank(int port)
static void test_http_server(void)
{
- char dir[1024], late_dir[1024];
+ char dir[1024], late_dir[1024], qwen_dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
if (!mkdtemp(dir)) {
@@ -728,6 +773,13 @@ static void test_http_server(void)
test_failures++;
return;
}
+ snprintf(qwen_dir, sizeof(qwen_dir), "%s/embed-srv-qwen-test-XXXXXX",
+ getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
+ if (!mkdtemp(qwen_dir)) {
+ fprintf(stderr, "FAIL: Qwen mkdtemp\n");
+ test_failures++;
+ return;
+ }
/* The 0.6b server slot requires hidden_size 1024; everything else stays
* tiny. The model vocab covers every tokenizer fixture id. */
tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE};
@@ -735,7 +787,10 @@ static void test_http_server(void)
tm_write_model_dims(dir, "F32", &dims) != 0 ||
tf_write_vocab(late_dir) != 0 ||
tm_write_model_dims(late_dir, "F32", &dims) != 0 ||
- tm_write_late_projection(late_dir, "F32", 128, 1024) != 0) {
+ tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 ||
+ tf_write_vocab(qwen_dir) != 0 ||
+ tm_write_qwen3_model_dims(qwen_dir, "F32", &dims,
+ TF_EOT_ID) != 0) {
fprintf(stderr, "FAIL: fixture write\n");
test_failures++;
return;
@@ -754,8 +809,10 @@ static void test_http_server(void)
ctx.spec[1].path = dir;
ctx.spec[2].id = "pplx-embed-v1-late-0.6b";
ctx.spec[2].path = late_dir;
+ ctx.spec[3].id = "Qwen3-Embedding-0.6B";
+ ctx.spec[3].path = qwen_dir;
ctx.cfg.models = ctx.spec;
- ctx.cfg.n_models = 3;
+ ctx.cfg.n_models = 4;
ctx.cfg.port = port;
ctx.cfg.batch_size = 2;
ctx.cfg.batch_wait_us = 1000;
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index 8c23e2c..d4ae431 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -57,6 +57,27 @@ typedef struct {
enum { TM_N_SPECS = 13 };
+static int tm_write_config(const char *dir, const tm_dims_t *d,
+ const char *model_type, int eos_token_id)
+{
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f) return -1;
+ fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
+ "\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
+ "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
+ "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0",
+ d->hidden, d->heads, d->kv_heads, d->head_dim,
+ d->intermediate, d->vocab);
+ if (model_type)
+ fprintf(f, ",\"model_type\":\"%s\",\"eos_token_id\":%d",
+ model_type, eos_token_id);
+ fputs("}", f);
+ fclose(f);
+ return 0;
+}
+
/* Write a safetensors file holding `specs` with tm_value() contents.
* dtype: "F32" or "BF16". Returns 0 on success. */
static int tm_write_safetensors(const char *path, const char *dtype,
@@ -133,21 +154,20 @@ static int tm_write_model_dims(const char *dir, const char *dtype,
};
char path[2048];
- snprintf(path, sizeof(path), "%s/config.json", dir);
- FILE *f = fopen(path, "w");
- if (!f) return -1;
- fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
- "\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
- "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
- "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}",
- d->hidden, d->heads, d->kv_heads, d->head_dim,
- d->intermediate, d->vocab);
- fclose(f);
+ if (tm_write_config(dir, d, NULL, -1) != 0) return -1;
snprintf(path, sizeof(path), "%s/model.safetensors", dir);
return tm_write_safetensors(path, dtype, specs, TM_N_SPECS);
}
+static inline int tm_write_qwen3_model_dims(const char *dir, const char *dtype,
+ const tm_dims_t *d,
+ int eos_token_id)
+{
+ if (tm_write_model_dims(dir, dtype, d) != 0) return -1;
+ return tm_write_config(dir, d, "qwen3", eos_token_id);
+}
+
/* Add the 1_Dense per-token projection head that turns a base model dir into
* a late-interaction snapshot. weight_name is "linear.weight" for a valid
* head; pass another name to synthesize a malformed head. */