diff options
| -rw-r--r-- | Makefile | 11 | ||||
| -rw-r--r-- | README.md | 24 | ||||
| -rw-r--r-- | embed.c | 111 | ||||
| -rw-r--r-- | embed.h | 48 | ||||
| -rw-r--r-- | embed_cli.c | 31 | ||||
| -rw-r--r-- | embed_cuda.cu | 101 | ||||
| -rw-r--r-- | embed_internal.h | 6 | ||||
| -rw-r--r-- | embed_mlx.c | 151 | ||||
| -rw-r--r-- | embed_mlx.h | 2 | ||||
| -rw-r--r-- | embed_server.c | 136 | ||||
| -rw-r--r-- | qwen_kernels.c | 152 | ||||
| -rw-r--r-- | qwen_kernels.h | 14 | ||||
| -rw-r--r-- | tests/test_kernels.c | 49 | ||||
| -rw-r--r-- | tests/test_qwen3.c | 92 | ||||
| -rw-r--r-- | tests/test_server.c | 65 | ||||
| -rw-r--r-- | tests/tiny_model.h | 40 |
16 files changed, 901 insertions, 132 deletions
@@ -162,6 +162,8 @@ TEST_TOKENIZER_SRCS = tests/test_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS) TEST_SAFETENSORS_SRCS = tests/test_safetensors.c qwen_safetensors.c TEST_BF16_SRCS = tests/test_bf16_model.c embed.c qwen_safetensors.c \ $(KERNEL_SRCS) +TEST_QWEN3_SRCS = tests/test_qwen3.c embed.c qwen_safetensors.c \ + $(KERNEL_SRCS) TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c qwen_tokenizer.c \ qwen_safetensors.c $(KERNEL_SRCS) TEST_LATE_SRCS = tests/test_late.c embed.c qwen_tokenizer.c \ @@ -188,6 +190,9 @@ test: $(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \ $(TEST_BF16_SRCS) -lm -lpthread ./tests/test_bf16_model + $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \ + $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + ./tests/test_qwen3 $(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \ $(TEST_WORKSPACE_SRCS) -lm -lpthread ./tests/test_workspace @@ -226,7 +231,7 @@ COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \ -fcoverage-compilation-dir=. COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \ tests/test_tokenizer tests/test_safetensors \ - tests/test_bf16_model tests/test_workspace tests/test_late \ + tests/test_bf16_model tests/test_qwen3 tests/test_workspace tests/test_late \ tests/cli_under_test tests/test_server # Report on project sources only - not the harnesses, vendored deps, or # system headers (cJSON lands under /opt or /usr otherwise). @@ -256,6 +261,9 @@ coverage: $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_bf16_model \ $(TEST_BF16_SRCS) -lm -lpthread LLVM_PROFILE_FILE=$(COV_DIR)/bf16_model.profraw ./tests/test_bf16_model + $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \ + $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + LLVM_PROFILE_FILE=$(COV_DIR)/qwen3.profraw ./tests/test_qwen3 $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \ $(TEST_WORKSPACE_SRCS) -lm -lpthread LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace @@ -381,6 +389,7 @@ clean: $(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \ tests/test_kernels_generic tests/test_kernels_blas \ tests/test_safetensors tests/test_bf16_model tests/test_server \ + tests/test_qwen3 \ tests/test_tokenizer tests/test_workspace tests/test_cli tests/test_late \ tests/cli_under_test bench/bench_tokenizer bench/bench_kernels bench/bench_model rm -rf $(COV_DIR) @@ -1,8 +1,11 @@ # embed.c -`embed.c` implements inference for -[pplx-embed](https://huggingface.co/collections/perplexity-ai/pplx-embed) -dense/contextual and late-interaction text embedding models. +`embed.c` implements inference for: + +- [pplx-embed](https://huggingface.co/collections/perplexity-ai/pplx-embed) + dense, contextual, and late-interaction models +- [Qwen3-Embedding](https://huggingface.co/collections/Qwen/qwen3-embedding) + causal, last-token pooled models Supported platforms are CUDA/cuBLAS on Linux/NVIDIA, Apple Silicon MLX, and CPU/BLAS. @@ -34,6 +37,17 @@ matrix: "Berlin is the capital of Germany." ``` +Qwen3 retrieval queries benefit from an explicit task instruction. Documents are +embedded directly: + +```bash +./embed -d ./Qwen3-Embedding-0.6B \ + $'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: What is the capital of China?' +``` + +The Qwen3 tokenizer terminal token is appended automatically by the CLI and +server. Qwen3 embeddings are L2-normalized; pplx embeddings remain unnormalized. + Pipe in lines and use `--stream` to get one JSON embedding per line: ```bash @@ -54,6 +68,7 @@ Serves the HTTP API. Takes one or more `--model ID=DIR` pairs: ```bash ./embed-server \ --model pplx-embed-v1-0.6b=./model \ + --model Qwen3-Embedding-0.6B=./qwen3-model \ --port 8000 ``` @@ -66,6 +81,9 @@ Late-interaction reranking endpoint: - `POST /v1/rerank` — late-interaction MaxSim reranking +For `Qwen3-Embedding-0.6B`, `/v1/embeddings` accepts `dimensions` from 32 +to 1024. Truncated Matryoshka embeddings are re-normalized before encoding. + Example: ```bash @@ -185,6 +185,16 @@ static double json_get_double(const char *json, const char *key, double fallback return atof(v); } +static int json_string_equals(const char *json, const char *key, + const char *expected) +{ + const char *v = json_find_key(json, key); + if (!v || *v != '"') return 0; + v++; + size_t len = strlen(expected); + return strncmp(v, expected, len) == 0 && v[len] == '"'; +} + static int parse_config(embed_config_t *cfg, const char *model_dir) { char path[1024]; @@ -225,6 +235,19 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE); cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); + cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; + cfg->pooling_mode = EMBED_POOL_MEAN; + cfg->normalize_embeddings = 0; + cfg->append_terminal_token = 0; + cfg->terminal_token_id = -1; + + if (json_string_equals(buf, "model_type", "qwen3")) { + cfg->attention_mode = EMBED_ATTENTION_CAUSAL; + cfg->pooling_mode = EMBED_POOL_LAST_TOKEN; + cfg->normalize_embeddings = 1; + cfg->append_terminal_token = 1; + cfg->terminal_token_id = json_get_int(buf, "eos_token_id", -1); + } cfg->q_dim = 0; cfg->kv_dim = 0; @@ -244,7 +267,10 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { + !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f || + (cfg->append_terminal_token && + (cfg->terminal_token_id < 0 || + cfg->terminal_token_id >= cfg->vocab_size))) { fprintf(stderr, "embed_model_load: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, @@ -259,10 +285,14 @@ static int parse_config(embed_config_t *cfg, const char *model_dir) if (embed_verbose >= 1) fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, " - "inter=%d, head_dim=%d\n", + "inter=%d, head_dim=%d, attention=%s, pooling=%s\n", cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, - cfg->intermediate_size, cfg->head_dim); + cfg->intermediate_size, cfg->head_dim, + cfg->attention_mode == EMBED_ATTENTION_CAUSAL + ? "causal" : "bidirectional", + cfg->pooling_mode == EMBED_POOL_LAST_TOKEN + ? "last-token" : "mean"); return 0; } @@ -618,6 +648,10 @@ static embed_model_t *model_load_range_ex(const char *model_dir, } model->layer_start = layer_start; model->layer_end = layer_end; + model->attention = + cfg->attention_mode == EMBED_ATTENTION_CAUSAL + ? qwen_causal_gqa_attention_packed_with_scratch + : qwen_bidirectional_gqa_attention_packed_with_scratch; /* Open safetensors (handles single file or multi-shard) */ multi_safetensors_t *ms = multi_safetensors_open(model_dir); @@ -1187,7 +1221,7 @@ static int forward_packed_slice_inplace(const embed_model_t *model, rope_cos, rope_sin, len, n_kv_heads, head_dim); } - qwen_bidirectional_gqa_attention_packed_with_scratch( + model->attention( attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, head_dim, scale, ws->attn_scores, ws->attn_scores_bytes); @@ -1223,16 +1257,18 @@ static int forward_packed_inplace(const embed_model_t *model, apply_final_norm); } -static void pool_embeddings(const embed_model_t *model, - const embed_workspace_t *ws, - const int *offsets, int batch, - float *out_embeddings) +static int pool_embeddings(const embed_model_t *model, + const embed_workspace_t *ws, + const int *offsets, int batch, + float *out_embeddings) { const embed_config_t *cfg = &model->config; int hidden = cfg->hidden_size; float eps = cfg->rms_norm_eps; const float *norm_weight = model->weights.norm; const float *x = ws->x; + if (hidden <= 0 || !norm_weight || !x) + return -1; for (int b = 0; b < batch; b++) { int start = offsets[b]; @@ -1241,23 +1277,36 @@ static void pool_embeddings(const embed_model_t *model, float *emb = out_embeddings + (size_t)b * hidden; memset(emb, 0, (size_t)hidden * sizeof(float)); - for (int i = start; i < end; i++) { - const float *row = x + (size_t)i * hidden; - + if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) { + const float *row = x + (size_t)(end - 1) * hidden; float sum_sq = 0.0f; for (int d = 0; d < hidden; d++) sum_sq += row[d] * row[d]; - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); for (int d = 0; d < hidden; d++) - emb[d] += row[d] * rms_inv * norm_weight[d]; - } + emb[d] = row[d] * rms_inv * norm_weight[d]; + } else { + for (int i = start; i < end; i++) { + const float *row = x + (size_t)i * hidden; - float inv_len = 1.0f / (float)len; - for (int d = 0; d < hidden; d++) - emb[d] *= inv_len; + float sum_sq = 0.0f; + for (int d = 0; d < hidden; d++) + sum_sq += row[d] * row[d]; + float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); + for (int d = 0; d < hidden; d++) + emb[d] += row[d] * rms_inv * norm_weight[d]; + } + float inv_len = 1.0f / (float)len; + for (int d = 0; d < hidden; d++) + emb[d] *= inv_len; + } + + if (cfg->normalize_embeddings && + embed_l2_normalize(emb, hidden) != 0) + return -1; } + return 0; } /* ======================================================================== @@ -1431,7 +1480,7 @@ int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws, int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); if (rc == 0) - pool_embeddings(model, ws, offsets, batch, out_embeddings); + rc = pool_embeddings(model, ws, offsets, batch, out_embeddings); return rc; } @@ -1451,16 +1500,25 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states, return -1; float *emb = out_embeddings + (size_t)b * cfg->hidden_size; - memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); - for (int t = 0; t < len; t++) { - const float *row = states + (size_t)(offset + t) * cfg->hidden_size; + if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) { + const float *row = + states + (size_t)(offset + len - 1) * cfg->hidden_size; + memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float)); + } else { + memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); + for (int t = 0; t < len; t++) { + const float *row = + states + (size_t)(offset + t) * cfg->hidden_size; + for (int d = 0; d < cfg->hidden_size; d++) + emb[d] += row[d]; + } + float inv_len = 1.0f / (float)len; for (int d = 0; d < cfg->hidden_size; d++) - emb[d] += row[d]; + emb[d] *= inv_len; } - - float inv_len = 1.0f / (float)len; - for (int d = 0; d < cfg->hidden_size; d++) - emb[d] *= inv_len; + if (cfg->normalize_embeddings && + embed_l2_normalize(emb, cfg->hidden_size) != 0) + return -1; offset += len; } return 0; @@ -1480,6 +1538,7 @@ float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws, if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; int hidden = model->config.hidden_size; + if (hidden <= 0) return NULL; float *emb = malloc_floats((size_t)hidden); if (!emb) return NULL; @@ -20,7 +20,7 @@ #include <stdint.h> /* ======================================================================== - * Shared constants (identical across all models) + * Shared constants and family defaults * ======================================================================== */ #define EMBED_VOCAB_SIZE 151936 @@ -32,18 +32,33 @@ * Model Configuration (populated from config.json at load time) * ======================================================================== */ +typedef enum { + EMBED_ATTENTION_BIDIRECTIONAL = 0, + EMBED_ATTENTION_CAUSAL = 1, +} embed_attention_mode_t; + +typedef enum { + EMBED_POOL_MEAN = 0, + EMBED_POOL_LAST_TOKEN = 1, +} embed_pooling_mode_t; + typedef struct { - int hidden_size; /* 0.6B: 1024 4B: 2560 */ - int n_layers; /* 0.6B: 28 4B: 36 */ - int n_heads; /* 0.6B: 16 4B: 32 */ - int n_kv_heads; /* 0.6B: 8 4B: 8 */ - int head_dim; /* always 128 */ + int hidden_size; + int n_layers; + int n_heads; + int n_kv_heads; + int head_dim; int q_dim; /* n_heads * head_dim */ int kv_dim; /* n_kv_heads * head_dim */ - int intermediate_size; /* 0.6B: 3072 4B: 9728 */ - int vocab_size; /* 151936 */ - float rms_norm_eps; /* 1e-6 */ - float rope_theta; /* 1e6 */ + int intermediate_size; + int vocab_size; + float rms_norm_eps; + float rope_theta; + embed_attention_mode_t attention_mode; + embed_pooling_mode_t pooling_mode; + int normalize_embeddings; + int append_terminal_token; + int terminal_token_id; } embed_config_t; /* ======================================================================== @@ -110,13 +125,14 @@ EMBED_API size_t embed_workspace_nbytes(const embed_workspace_t *ws); * Compute embedding for a token sequence. * * 1. Token embedding lookup - * 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU) + * 2. N transformer layers (GQA + RoPE + SwiGLU) * 3. Final RMSNorm - * 4. Mean pooling over all positions + * 4. Model-defined pooling and optional L2 normalization * - * pplx-embed-v1 embeddings are intentionally not L2-normalized. Use cosine - * similarity for comparisons, or normalize explicitly before storing in vector - * databases that only support inner product. + * pplx-embed-v1 uses bidirectional attention and unnormalized mean pooling. + * Qwen3-Embedding uses causal attention and normalized last-token pooling. + * Token IDs are consumed exactly as provided; text frontends append any + * model-required terminal token before calling this API. * * Returns malloc'd float[hidden_size] (caller frees). NULL on error. */ @@ -145,7 +161,7 @@ EMBED_API int embed_model_encode_batch(const embed_model_t *model, embed_workspa float *out_embeddings); /* - * Mean-pool packed final hidden states for a ragged batch. + * Apply the config's pooling mode to packed final hidden states. * * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive * token count per sequence. The states must already include final RMSNorm. diff --git a/embed_cli.c b/embed_cli.c index ebdb223..06b40de 100644 --- a/embed_cli.c +++ b/embed_cli.c @@ -16,6 +16,7 @@ #include <string.h> #include <math.h> #include <time.h> +#include <limits.h> static double now_ms(void) { @@ -94,6 +95,8 @@ typedef struct { qwen_tokenizer_t *tok; qwen_tokenizer_workspace_t *tok_ws; int dim; + int terminal_token_id; + int append_terminal_token; } engine_t; typedef struct { @@ -128,6 +131,22 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) out->n_tokens = 0; return -1; } + if (e->append_terminal_token) { + if (out->n_tokens == INT_MAX) { + free(out->ids); + memset(out, 0, sizeof(*out)); + return -1; + } + int *ids = (int *)realloc( + out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); + if (!ids) { + free(out->ids); + memset(out, 0, sizeof(*out)); + return -1; + } + out->ids = ids; + out->ids[out->n_tokens++] = e->terminal_token_id; + } if (embed_verbose >= 1) { fprintf(stderr, "tokens (%d): ", out->n_tokens); @@ -580,6 +599,7 @@ int main(int argc, char *argv[]) engine_t e = {0}; e.tok = tok; t0 = now_ms(); + const embed_config_t *config = NULL; #ifdef USE_MLX if (use_mlx) { @@ -593,7 +613,8 @@ int main(int argc, char *argv[]) qwen_tokenizer_free(tok); return 1; } - e.dim = embed_mlx_config(e.mlx_ctx)->hidden_size; + config = embed_mlx_config(e.mlx_ctx); + e.dim = config->hidden_size; } else #endif #ifdef USE_CUDA @@ -604,7 +625,8 @@ int main(int argc, char *argv[]) qwen_tokenizer_free(tok); return 1; } - e.dim = embed_cuda_config(e.cuda_ctx)->hidden_size; + config = embed_cuda_config(e.cuda_ctx); + e.dim = config->hidden_size; } else #endif { @@ -621,8 +643,11 @@ int main(int argc, char *argv[]) qwen_tokenizer_free(tok); return 1; } - e.dim = embed_model_config(e.model)->hidden_size; + config = embed_model_config(e.model); + e.dim = config->hidden_size; } + e.append_terminal_token = config->append_terminal_token; + e.terminal_token_id = config->terminal_token_id; if (verbose >= 1) fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0, diff --git a/embed_cuda.cu b/embed_cuda.cu index 2e4cd28..1ec0573 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -530,6 +530,45 @@ __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) store_act(p, (size_t)j, s[j] * inv); } +template <typename P_T> +__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, + int L) +{ + float *s = scores + (size_t)blockIdx.x * L; + P_T *p = probs + (size_t)blockIdx.x * L; + int query = blockIdx.x % L; + int keys = query + 1; + int tid = threadIdx.x; + __shared__ float red[256]; + + float m = -3.402823466e+38F; + for (int j = tid; j < keys; j += blockDim.x) m = fmaxf(m, s[j]); + red[tid] = m; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]); + __syncthreads(); + } + m = red[0]; + __syncthreads(); + + float sum = 0.0f; + for (int j = tid; j < keys; j += blockDim.x) { + float e = __expf(s[j] - m); + s[j] = e; + sum += e; + } + red[tid] = sum; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) red[tid] += red[tid + o]; + __syncthreads(); + } + float inv = 1.0f / red[0]; + for (int j = tid; j < L; j += blockDim.x) + store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f); +} + __global__ static void mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) @@ -547,6 +586,17 @@ __global__ static void mean_pool_kernel(float *out, const float *x, } } +__global__ static void last_pool_kernel(float *out, const float *x, + const int *offsets, int batch, + int hidden) +{ + int b = blockIdx.x; + int tid = threadIdx.x; + int last = offsets[b + 1] - 1; + for (int d = tid; d < hidden; d += blockDim.x) + out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d]; +} + __global__ static void span_pool_kernel(float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) @@ -783,6 +833,7 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, int hd = c->head_dim; int H = c->n_heads; int total = h_offsets[batch]; + int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; const float alpha = scale, beta = 0.0f, one = 1.0f; int threads = 256; @@ -823,11 +874,21 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, &beta, S + off, CUDA_R_32F, L, n, gemm_compute(), CUBLAS_GEMM_DEFAULT)); if (out_bf16) { - attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( - ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L); + if (causal) + attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, + (__nv_bfloat16 *)ctx->attn_probs, L); + else + attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, + (__nv_bfloat16 *)ctx->attn_probs, L); } else { - attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( - ctx->attn_scores, ctx->attn_scores, L); + if (causal) + attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, ctx->attn_scores, L); + else + attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, ctx->attn_scores, L); } if (launch_check() != 0) return -1; /* O = P @ V, batched over (seq, head) */ @@ -857,8 +918,12 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, const __nv_bfloat16 *V = (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim; __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim; - attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( - ctx->attn_scores, P, L); + if (causal) + attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, P, L); + else + attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, P, L); if (launch_check() != 0) return -1; /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ CUBLAS_CHECK(cublasGemmStridedBatchedEx( @@ -870,8 +935,12 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const int *h_offsets, } else { const float *V = ctx->vexp + (size_t)start * q_dim; float *O = ctx->attn_out + (size_t)start * q_dim; - attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( - ctx->attn_scores, ctx->attn_scores, L); + if (causal) + attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, ctx->attn_scores, L); + else + attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, ctx->attn_scores, L); if (launch_check() != 0) return -1; /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ if (gemm_strided_batched( @@ -1308,8 +1377,12 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, return -1; if (ensure_pooled_rows(ctx, batch) != 0) return -1; - mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>( - ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); + if (c->pooling_mode == EMBED_POOL_LAST_TOKEN) + last_pool_kernel<<<batch, 256, 0, ctx->stream>>>( + ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); + else + mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>( + ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); if (launch_check() != 0) return -1; cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, (size_t)batch * c->hidden_size * sizeof(float), @@ -1320,6 +1393,14 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce)); return -1; } + if (c->normalize_embeddings) { + for (int b = 0; b < batch; b++) { + if (embed_l2_normalize( + out_embeddings + (size_t)b * c->hidden_size, + c->hidden_size) != 0) + return -1; + } + } return 0; } diff --git a/embed_internal.h b/embed_internal.h index a7365a2..60fc581 100644 --- a/embed_internal.h +++ b/embed_internal.h @@ -29,12 +29,18 @@ typedef struct { const float *norm; /* [hidden] */ } embed_weights_t; +typedef void (*embed_attention_fn)( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes); + struct embed_model { embed_config_t config; embed_weights_t weights; void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ int layer_start; /* loaded transformer range, inclusive */ int layer_end; /* loaded transformer range, exclusive */ + embed_attention_fn attention; }; struct embed_workspace { diff --git a/embed_mlx.c b/embed_mlx.c index 5de4068..cbdfa33 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -358,6 +358,16 @@ static const char *json_find_m(const char *json, const char *key) return NULL; } +static int json_string_equals_m(const char *json, const char *key, + const char *expected) +{ + const char *v = json_find_m(json, key); + if (!v || *v != '"') return 0; + v++; + size_t len = strlen(expected); + return strncmp(v, expected, len) == 0 && v[len] == '"'; +} + static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) { char path[1024]; @@ -394,6 +404,19 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) GF("rope_theta", rope_theta, 1000000.0f); #undef GI #undef GF + cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; + cfg->pooling_mode = EMBED_POOL_MEAN; + cfg->normalize_embeddings = 0; + cfg->append_terminal_token = 0; + cfg->terminal_token_id = -1; + if (json_string_equals_m(buf, "model_type", "qwen3")) { + const char *eos = json_find_m(buf, "eos_token_id"); + cfg->attention_mode = EMBED_ATTENTION_CAUSAL; + cfg->pooling_mode = EMBED_POOL_LAST_TOKEN; + cfg->normalize_embeddings = 1; + cfg->append_terminal_token = 1; + cfg->terminal_token_id = eos ? atoi(eos) : -1; + } cfg->q_dim = 0; cfg->kv_dim = 0; @@ -411,7 +434,10 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { + !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f || + (cfg->append_terminal_token && + (cfg->terminal_token_id < 0 || + cfg->terminal_token_id >= cfg->vocab_size))) { fprintf(stderr, "mlx: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, @@ -782,6 +808,8 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, int q_dim = c->q_dim; float scale = 1.0f / sqrtf((float)head_dim); mlx_array null_arr = (mlx_array){0}; + const char *mask_mode = + c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : ""; /* For non-quantized BF16 weights, run the residual stream, norms, and * attention in F32; matmuls downcast their operands to BF16 internally @@ -850,7 +878,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_array attn = mlx_array_new(); mlx_fast_scaled_dot_product_attention( - &attn, qr, kr, vt, scale, "", + &attn, qr, kr, vt, scale, mask_mode, has_padding ? local_mask : null_arr, null_arr, S); mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); @@ -1373,6 +1401,10 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, int hidden = c->hidden_size; int max_seq = 0; int has_padding = 0; + int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; + int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN; + if (hidden <= 0 || c->vocab_size <= 0) + return -1; for (int b = 0; b < batch; b++) { if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1; @@ -1383,26 +1415,58 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, for (int b = 0; b < batch; b++) { if (inputs[b].n_tokens != max_seq) has_padding = 1; } + int needs_pool_mask = has_padding && !last_token_pool; + int needs_attn_mask = has_padding && !causal; size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes; + size_t last_index_bytes = 0; if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 || mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 || mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 || mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || - mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) + mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 || + (last_token_pool && + mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) + return -1; + if (ids_bytes == 0 || out_bytes == 0 || + (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) || + (needs_attn_mask && mask_bytes == 0) || + (last_token_pool && last_index_bytes == 0)) return -1; int *padded_ids = (int *)malloc(ids_bytes); - float *pool_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; - float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; - float *len_data = has_padding ? (float *)malloc(len_bytes) : NULL; - if (!padded_ids || (has_padding && (!pool_mask_data || !attn_mask_data || !len_data))) { - free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + float *pool_mask_data = + needs_pool_mask ? (float *)malloc(mask_bytes) : NULL; + float *attn_mask_data = + needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; + float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL; + int *last_index_data = + last_token_pool ? (int *)malloc(last_index_bytes) : NULL; + if (!padded_ids || + (needs_pool_mask && (!pool_mask_data || !len_data)) || + (needs_attn_mask && !attn_mask_data)) { + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); + free(last_index_data); + return -1; + } + if (last_token_pool && !last_index_data) { + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); return -1; } for (int b = 0; b < batch; b++) { - if (has_padding) len_data[b] = (float)inputs[b].n_tokens; + if (needs_pool_mask) len_data[b] = (float)inputs[b].n_tokens; + if (last_token_pool) { + int index = inputs[b].n_tokens - 1; + for (int d = 0; d < hidden; d++) + last_index_data[(size_t)b * hidden + d] = index; + } for (int t = 0; t < max_seq; t++) { size_t idx = (size_t)b * max_seq + t; if (t < inputs[b].n_tokens) { @@ -1410,19 +1474,24 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, if (id < 0 || id >= c->vocab_size) { fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t); - free(padded_ids); free(pool_mask_data); - free(attn_mask_data); free(len_data); + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); + free(last_index_data); return -1; } padded_ids[idx] = id; - if (has_padding) { + if (needs_pool_mask) pool_mask_data[idx] = 1.0f; + if (needs_attn_mask) attn_mask_data[idx] = 0.0f; - } } else { padded_ids[idx] = 0; - pool_mask_data[idx] = 0.0f; - attn_mask_data[idx] = -1.0e9f; + if (needs_pool_mask) + pool_mask_data[idx] = 0.0f; + if (needs_attn_mask) + attn_mask_data[idx] = -1.0e9f; } } } @@ -1431,33 +1500,42 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, int pool_mask_shape[] = {batch, max_seq, 1}; int attn_mask_shape[] = {batch, 1, 1, max_seq}; int len_shape[] = {batch, 1}; + int last_index_shape[] = {batch, 1, hidden}; mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - mlx_array pool_mask = has_padding + mlx_array pool_mask = needs_pool_mask ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) : (mlx_array){0}; - mlx_array attn_mask = has_padding + mlx_array attn_mask = needs_attn_mask ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) : (mlx_array){0}; - mlx_array lengths = has_padding + mlx_array lengths = needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0}; + mlx_array last_indices = last_token_pool + ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) + : (mlx_array){0}; free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + free(last_index_data); - if (!arr_ok(ids) || (has_padding && (!arr_ok(pool_mask) || - !arr_ok(attn_mask) || !arr_ok(lengths)))) { + if (!arr_ok(ids) || + (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) || + (needs_attn_mask && !arr_ok(attn_mask)) || + (last_token_pool && !arr_ok(last_indices))) { mlx_array_free(ids); mlx_array_free(pool_mask); - if (has_padding) mlx_array_free(attn_mask); + mlx_array_free(attn_mask); mlx_array_free(lengths); + mlx_array_free(last_indices); return -1; } - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + if (needs_attn_mask && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { mlx_array attn_mask_bf16 = mlx_array_new(); mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); mlx_array_free(attn_mask); attn_mask = attn_mask_bf16; if (!arr_ok(attn_mask)) { mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths); + mlx_array_free(last_indices); return -1; } } @@ -1468,7 +1546,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(ids); /* 2. Transformer layers. */ - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, + x = mlx_forward_layers(ctx, x, batch, max_seq, + needs_attn_mask, attn_mask, 0, c->n_layers); /* 3. Final RMSNorm. */ @@ -1477,7 +1556,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(x); mlx_array emb = mlx_array_new(); - if (has_padding) { + if (last_token_pool) { + mlx_array selected = mlx_array_new(); + mlx_take_along_axis(&selected, x_normed, last_indices, 1, S); + mlx_array_free(x_normed); + mlx_array_free(last_indices); + int emb_shape[] = {batch, hidden}; + mlx_reshape(&emb, selected, emb_shape, 2, S); + mlx_array_free(selected); + mlx_array_free(pool_mask); + mlx_array_free(attn_mask); + mlx_array_free(lengths); + } else if (has_padding) { /* 4. Masked mean pool over T. */ mlx_array masked = mlx_array_new(); mlx_multiply(&masked, x_normed, pool_mask, S); @@ -1508,7 +1598,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, int rc = -1; if (data) { memcpy(out_embeddings, data, out_bytes); - rc = 0; + if (c->normalize_embeddings) { + rc = 0; + for (int b = 0; b < batch; b++) { + if (embed_l2_normalize( + out_embeddings + (size_t)b * hidden, hidden) != 0) { + rc = -1; + break; + } + } + } else { + rc = 0; + } } mlx_array_free(emb_f32); return rc; diff --git a/embed_mlx.h b/embed_mlx.h index 267c454..5f6e2c9 100644 --- a/embed_mlx.h +++ b/embed_mlx.h @@ -108,7 +108,7 @@ EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids, * Compute a true padded dense batch on MLX. * * out_embeddings is caller-provided [batch, hidden_size]. - * Padded tokens are masked out of attention keys and mean pooling. + * Pooling and attention masking follow the loaded model configuration. * Returns 0 on success, -1 on error. */ EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, diff --git a/embed_server.c b/embed_server.c index 7388a4d..1d64e1d 100644 --- a/embed_server.c +++ b/embed_server.c @@ -205,6 +205,7 @@ static void sbuf_free(sbuf *b) { typedef enum { MODEL_STD_06, MODEL_STD_4, + MODEL_QWEN3_06, MODEL_CTX_06, MODEL_CTX_4, MODEL_LATE_06, @@ -222,16 +223,27 @@ typedef struct { const char *id; model_kind kind; int dim; + int min_dim; int token_dim; double price_per_mtok; + embed_attention_mode_t attention_mode; + embed_pooling_mode_t pooling_mode; + int normalize_embeddings; } model_info; static const model_info k_models[] = { - {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 0, 0.004}, - {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 0, 0.030}, - {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 0, 0.008}, - {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 0, 0.050}, - {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 0.0}, + {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, 0.004, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0}, + {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, 0.030, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0}, + {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, 0.0, + EMBED_ATTENTION_CAUSAL, EMBED_POOL_LAST_TOKEN, 1}, + {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, 0.008, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0}, + {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, 0.050, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0}, + {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, 0.0, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0}, }; static model_slot model_slot_for_id(const char *id) { @@ -253,6 +265,9 @@ typedef struct { int late_document_prefix_id; int late_skip_ids[64]; int n_late_skip_ids; + int append_terminal_token; + int terminal_token_id; + int renormalize_truncated; embed_model_t *cpu_model; embed_workspace_t *cpu_ws; embed_late_model_t *cpu_late_model; @@ -1100,8 +1115,8 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail) { return encoding->valuestring; } -static int dimensions_from_root(cJSON *root, cJSON *detail, int max_dim, - const char *encoding) { +static int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, + int max_dim, const char *encoding) { int dims = max_dim; cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); if (dimensions) { @@ -1110,9 +1125,11 @@ static int dimensions_from_root(cJSON *root, cJSON *detail, int max_dim, "type_error.integer"); } else { dims = dimensions->valueint; - if (dims < 128 || dims > max_dim) { + if (dims < min_dim || dims > max_dim) { char msg[96]; - snprintf(msg, sizeof(msg), "dimensions must be between 128 and %d", max_dim); + snprintf(msg, sizeof(msg), + "dimensions must be between %d and %d", + min_dim, max_dim); ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); } } @@ -1285,6 +1302,22 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, memset(out, 0, sizeof(*out)); return -1; } + if (m->append_terminal_token) { + if (out->n_tokens == INT_MAX) { + free(out->ids); + memset(out, 0, sizeof(*out)); + return -1; + } + int *ids = realloc( + out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); + if (!ids) { + free(out->ids); + memset(out, 0, sizeof(*out)); + return -1; + } + out->ids = ids; + out->ids[out->n_tokens++] = m->terminal_token_id; + } return 0; } @@ -1402,12 +1435,32 @@ static void set_response_from_buf(job *j, sbuf *b) { memset(b, 0, sizeof(*b)); } -static void render_embedding_response(embedding_request *r, const float *embs) { +static int render_embedding_response(embedding_request *r, const float *embs) { + const int full_dim = r->model->info->dim; + const float *render_embs = embs; + float *truncated = NULL; + if (r->model->renormalize_truncated && r->dims < full_dim) { + truncated = xmalloc( + (size_t)r->n_inputs * r->dims * sizeof(*truncated)); + for (int i = 0; i < r->n_inputs; i++) { + float *dst = truncated + (size_t)i * r->dims; + memcpy(dst, embs + (size_t)i * full_dim, + (size_t)r->dims * sizeof(*dst)); + if (embed_l2_normalize(dst, r->dims) != 0) { + free(truncated); + return -1; + } + } + render_embs = truncated; + } + int render_stride = truncated ? r->dims : full_dim; + char **encoded = NULL; if (strcmp(r->encoding, "float")) { encoded = xcalloc((size_t)r->n_inputs, sizeof(*encoded)); for (int i = 0; i < r->n_inputs; i++) - encoded[i] = encode_embedding(embs + (size_t)i * r->model->info->dim, + encoded[i] = encode_embedding(render_embs + + (size_t)i * render_stride, r->dims, r->encoding); } @@ -1419,7 +1472,7 @@ static void render_embedding_response(embedding_request *r, const float *embs) { append_embedding_object(&b, i, encoded[i]); else append_float_embedding_object(&b, i, - embs + (size_t)i * r->model->info->dim, r->dims); + render_embs + (size_t)i * render_stride, r->dims); } double cost = ((double)r->total_tokens / 1000000.0) * r->model->info->price_per_mtok; @@ -1432,6 +1485,8 @@ static void render_embedding_response(embedding_request *r, const float *embs) { for (int i = 0; encoded && i < r->n_inputs; i++) free(encoded[i]); free(encoded); + free(truncated); + return 0; } static int embedding_request_compatible(const embedding_request *a, @@ -1525,7 +1580,10 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); } else { uint64_t t0 = nstime(); - render_embedding_response(reqs[i], embs + (size_t)pos * dim); + if (render_embedding_response( + reqs[i], embs + (size_t)pos * dim) != 0) + job_set_error(reqs[i]->j, 500, "embedding normalization failed", + "server_error"); reqs[i]->j->encode_ns += nstime() - t0; } pos += reqs[i]->n_inputs; @@ -1736,9 +1794,10 @@ static int validate_common(cJSON *root, cJSON *detail, model_kind expected_kind, } } const char *enc = encoding_from_root(root, detail); - int max_dim = model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN - ? k_models[model_slot_for_id(model_id)].dim : 2560; - int dims = dimensions_from_root(root, detail, max_dim, enc); + model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN; + int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128; + int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560; + int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc); *out_encoding = enc; *out_dims = dims; return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && @@ -2502,6 +2561,34 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { free(std_reqs); } +static int configure_loaded_model(loaded_model *m, + const embed_config_t *config, + int token_dim) +{ + if (!m || !m->info || !config || + config->hidden_size != m->info->dim || + token_dim != m->info->token_dim || + config->attention_mode != m->info->attention_mode || + config->pooling_mode != m->info->pooling_mode || + config->normalize_embeddings != m->info->normalize_embeddings) { + if (m && m->info && config) + server_log("embed-server: model %s has incompatible config", + m->info->id); + return -1; + } + if (config->append_terminal_token && + (config->terminal_token_id < 0 || + config->terminal_token_id >= config->vocab_size)) + return -1; + + m->append_terminal_token = config->append_terminal_token; + m->terminal_token_id = config->terminal_token_id; + m->renormalize_truncated = + config->normalize_embeddings && + config->pooling_mode == EMBED_POOL_LAST_TOKEN; + return 0; +} + static void *worker_main(void *arg) { http_server *s = arg; #ifdef USE_MLX @@ -2531,14 +2618,11 @@ static void *worker_main(void *arg) { : embed_mlx_config(m->mlx_ctx); int token_dim = m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; - if (config->hidden_size != m->info->dim || - token_dim != m->info->token_dim || + if (configure_loaded_model(m, config, token_dim) != 0 || (m->info->kind == MODEL_KIND_LATE && (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size || m->late_document_prefix_id >= config->vocab_size))) { - server_log("embed-server: model %s has unexpected dimension %d", - m->info->id, config->hidden_size); rc = 1; break; } @@ -2578,10 +2662,8 @@ static void *worker_main(void *arg) { rc = 1; break; } - if (embed_cuda_config(m->cuda_ctx)->hidden_size != m->info->dim) { - server_log("embed-server: model %s has unexpected dimension %d", - m->info->id, - embed_cuda_config(m->cuda_ctx)->hidden_size); + if (configure_loaded_model( + m, embed_cuda_config(m->cuda_ctx), 0) != 0) { rc = 1; break; } @@ -2665,6 +2747,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { loaded_model *m = &s->models[slot]; m->info = &k_models[slot]; m->path = xstrdup(path); + m->terminal_token_id = -1; char vocab_path[1024]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); m->tok = qwen_tokenizer_load(vocab_path); @@ -2751,10 +2834,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { } config = embed_model_config(m->cpu_model); } - if (config->hidden_size != m->info->dim || - token_dim != m->info->token_dim) { - server_log("embed-server: model %s has unexpected dimension %d", - m->info->id, config->hidden_size); + if (configure_loaded_model(m, config, token_dim) != 0) { return -1; } if (m->info->kind == MODEL_KIND_LATE && diff --git a/qwen_kernels.c b/qwen_kernels.c index 85871d4..dcd724c 100644 --- a/qwen_kernels.c +++ b/qwen_kernels.c @@ -1015,6 +1015,158 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, free(scores); } +static void qwen_causal_gqa_attention_packed_online_rows( + float *out, const float *Q, const float *K, const float *V, + int start, int n_heads, int n_kv_heads, int head_dim, + float scale, int h, int query_start, int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = start; j <= i; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) + qwen_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); + } +} + +#ifdef USE_BLAS +static void qwen_causal_gqa_attention_packed_blas_rows( + float *out, const float *Q, const float *K, const float *V, + int start, int n_heads, int n_kv_heads, int head_dim, + float scale, int h, int query_start, int query_end, float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = query_end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden, + 0.0f, scores, keys); + for (int r = 0; r < rows; r++) { + int allowed = query_start + r - start + 1; + float *row = scores + (size_t)r * keys; + for (int j = allowed; j < keys; j++) row[j] = -INFINITY; + } + qwen_softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, + 0.0f, o, q_hidden); +} +#endif + +static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = t->scores + ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq + : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; + if (q1 > end) q1 = end; + if (task_id++ % n_threads != tid) continue; +#ifdef USE_BLAS + if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { + qwen_causal_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, + scores); + continue; + } +#endif + qwen_causal_gqa_attention_packed_online_rows( + t->out, t->Q, t->K, t->V, start, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); + } + } + } +} + +void qwen_causal_gqa_attention_packed_with_scratch( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * (len + 1) / 2 * n_heads; + if (len > max_seq) max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes( + offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = { + .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, + .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, + .head_dim = head_dim, .scale = scale, .scores = scores, + .max_seq = max_seq + }; + if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) + parallel_for(packed_causal_gqa_attn_worker, &task); + else + packed_causal_gqa_attn_worker(0, 1, &task); +} + +void qwen_causal_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale) { + size_t scratch_bytes = + qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + qwen_causal_gqa_attention_packed_with_scratch( + out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, + scores, scratch_bytes); + free(scores); +} + /* ======================================================================== * Position Embeddings * ======================================================================== */ diff --git a/qwen_kernels.h b/qwen_kernels.h index 71f77f1..c611194 100644 --- a/qwen_kernels.h +++ b/qwen_kernels.h @@ -116,6 +116,20 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch( const int *offsets, int batch, int n_heads, int n_kv_heads, int head_dim, float scale, float *scratch, size_t scratch_bytes); +/* + * Causal GQA attention over a packed/ragged batch. Query position i attends + * only to positions at or before i within its sequence. + */ +void qwen_causal_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale); +void qwen_causal_gqa_attention_packed_with_scratch( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes); + /* ======================================================================== * Position Embeddings * ======================================================================== */ diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 208dd02..4a2a57d 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -95,7 +95,7 @@ static void test_rope_neox(void) { static void reference_packed_gqa(float *out, const float *Q, const float *K, const float *V, const int *offsets, int batch, int n_heads, int n_kv_heads, int head_dim, - float scale) { + float scale, int causal) { int q_hidden = n_heads * head_dim; int kv_hidden = n_kv_heads * head_dim; int heads_per_kv = n_heads / n_kv_heads; @@ -105,10 +105,11 @@ static void reference_packed_gqa(float *out, const float *Q, const float *K, for (int b = 0; b < batch; b++) { int start = offsets[b], end = offsets[b + 1]; for (int i = start; i < end; i++) { + int key_end = causal ? i + 1 : end; const float *q = Q + i * q_hidden + h * head_dim; float *o = out + i * q_hidden + h * head_dim; float max_score = -INFINITY; - for (int j = start; j < end; j++) { + for (int j = start; j < key_end; j++) { const float *k = K + j * kv_hidden + kv_h * head_dim; float score = 0.0f; for (int d = 0; d < head_dim; d++) score += q[d] * k[d]; @@ -117,7 +118,7 @@ static void reference_packed_gqa(float *out, const float *Q, const float *K, } float denom = 0.0f; for (int d = 0; d < head_dim; d++) o[d] = 0.0f; - for (int j = start; j < end; j++) { + for (int j = start; j < key_end; j++) { const float *k = K + j * kv_hidden + kv_h * head_dim; const float *v = V + j * kv_hidden + kv_h * head_dim; float score = 0.0f; @@ -155,7 +156,7 @@ static void test_packed_gqa_attention(void) { 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, - 1.0f / sqrtf((float)head_dim)); + 1.0f / sqrtf((float)head_dim), 0); for (int i = 0; i < total * q_hidden; i++) expect_close("packed_gqa", got[i], want[i], 2e-6f); @@ -186,7 +187,7 @@ static void test_packed_gqa_attention_long(void) { 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, - 1.0f / sqrtf((float)head_dim)); + 1.0f / sqrtf((float)head_dim), 0); qwen_set_threads(1); for (int i = 0; i < total * q_hidden; i++) @@ -198,6 +199,43 @@ static void test_packed_gqa_attention_long(void) { free(Q); } +static void test_packed_causal_gqa_attention(void) { + enum { total = 136, n_heads = 4, n_kv_heads = 2, head_dim = 8 }; + enum { q_hidden = n_heads * head_dim }; + enum { kv_hidden = n_kv_heads * head_dim }; + const int offsets[3] = {0, 7, 136}; + float *Q = malloc((size_t)total * q_hidden * sizeof(*Q)); + float *K = malloc((size_t)total * kv_hidden * sizeof(*K)); + float *V = malloc((size_t)total * kv_hidden * sizeof(*V)); + float *got = malloc((size_t)total * q_hidden * sizeof(*got)); + float *want = malloc((size_t)total * q_hidden * sizeof(*want)); + if (!Q || !K || !V || !got || !want) exit(2); + + for (int i = 0; i < total * q_hidden; i++) + Q[i] = sinf((float)i * 0.019f) - 0.2f; + for (int i = 0; i < total * kv_hidden; i++) { + K[i] = cosf((float)i * 0.013f) + 0.15f; + V[i] = sinf((float)i * 0.009f) - cosf((float)i * 0.015f); + } + + qwen_set_threads(4); + qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, + n_heads, n_kv_heads, head_dim, + 1.0f / sqrtf((float)head_dim)); + reference_packed_gqa(want, Q, K, V, offsets, 2, + n_heads, n_kv_heads, head_dim, + 1.0f / sqrtf((float)head_dim), 1); + qwen_set_threads(1); + + for (int i = 0; i < total * q_hidden; i++) + expect_close("packed_causal_gqa", got[i], want[i], 2e-5f); + free(want); + free(got); + free(V); + free(K); + free(Q); +} + static void test_bf16_linear(void) { enum { seq = 2, in_dim = 3, out_dim = 2 }; const float x[6] = { @@ -492,6 +530,7 @@ int main(void) { test_rope_neox(); test_packed_gqa_attention(); test_packed_gqa_attention_long(); + test_packed_causal_gqa_attention(); test_bf16_linear(); test_bf16_qkv(); test_bf16_matvec(); diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c new file mode 100644 index 0000000..8205ca1 --- /dev/null +++ b/tests/test_qwen3.c @@ -0,0 +1,92 @@ +/* tests/test_qwen3.c - hermetic Qwen3 model semantics. */ + +#include "embed.h" +#include "tiny_model.h" + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +static float max_abs_diff(const float *a, const float *b, int n) +{ + float out = 0.0f; + for (int i = 0; i < n; i++) { + float d = fabsf(a[i] - b[i]); + if (d > out) out = d; + } + return out; +} + +int main(void) +{ + char dir[1024]; + snprintf(dir, sizeof(dir), "%s/embed-qwen3-test-XXXXXX", + getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + tm_dims_t dims = {4, 2, 1, 2, 8, 16}; + if (!mkdtemp(dir) || + tm_write_qwen3_model_dims(dir, "F32", &dims, 15) != 0) { + fprintf(stderr, "fixture creation failed\n"); + return 2; + } + + embed_model_t *model = embed_model_load(dir); + embed_workspace_t *ws = embed_workspace_new(model); + const embed_config_t *cfg = embed_model_config(model); + if (!model || !ws || !cfg) { + fprintf(stderr, "model setup failed\n"); + return 1; + } + if (cfg->attention_mode != EMBED_ATTENTION_CAUSAL || + cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || + !cfg->normalize_embeddings || !cfg->append_terminal_token || + cfg->terminal_token_id != 15) { + fprintf(stderr, "Qwen3 config semantics were not detected\n"); + return 1; + } + + int ids[] = {1, 2, 3, 15}; + float states[4 * 4]; + float embedding[4]; + float pooled[4]; + if (embed_model_forward_into(model, ws, ids, 4, states) != 0 || + embed_model_encode_into(model, ws, ids, 4, embedding) != 0 || + embed_pool_batch(cfg, states, (int[]){4}, 1, pooled) != 0) { + fprintf(stderr, "Qwen3 inference failed\n"); + return 1; + } + + float expected[4]; + memcpy(expected, states + 3 * 4, sizeof(expected)); + if (embed_l2_normalize(expected, 4) != 0 || + max_abs_diff(embedding, expected, 4) > 1e-5f || + max_abs_diff(pooled, expected, 4) > 1e-5f) { + fprintf(stderr, "last-token pooling mismatch\n"); + return 1; + } + + float norm_sq = 0.0f; + for (int d = 0; d < 4; d++) norm_sq += embedding[d] * embedding[d]; + if (fabsf(norm_sq - 1.0f) > 1e-5f) { + fprintf(stderr, "Qwen3 embedding is not normalized: %.9g\n", norm_sq); + return 1; + } + + int short_ids[] = {4, 15}; + embed_input_t inputs[] = {{ids, 4}, {short_ids, 2}}; + float batch[8]; + float single[4]; + if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 || + embed_model_encode_into(model, ws, short_ids, 2, single) != 0 || + max_abs_diff(batch, embedding, 4) > 1e-5f || + max_abs_diff(batch + 4, single, 4) > 1e-5f) { + fprintf(stderr, "Qwen3 packed batch parity failed\n"); + return 1; + } + + embed_workspace_free(ws); + embed_model_free(model); + puts("ok: Qwen3 causal attention and last-token pooling"); + return 0; +} diff --git a/tests/test_server.c b/tests/test_server.c index 113b4d7..bb4d93b 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -289,7 +289,7 @@ static int http_req(int port, const char *method, const char *path, } typedef struct { - embed_server_model_spec_t spec[3]; + embed_server_model_spec_t spec[4]; embed_server_config_t cfg; int rc; } srv_ctx; @@ -470,6 +470,51 @@ static void test_http_embeddings(int port) free(body); body = NULL; + /* Qwen3 appends its terminal token, supports 32D MRL output, and + * re-normalizes the truncated prefix. */ + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"Qwen3-Embedding-0.6B\"," + "\"input\":\"hello\",\"encoding_format\":\"float\"}", + NULL, &body) == 200); + cJSON *qwen_full = body ? parse_embeddings_body(body, 1) : NULL; + free(body); + body = NULL; + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"Qwen3-Embedding-0.6B\"," + "\"input\":\"hello\",\"dimensions\":32," + "\"encoding_format\":\"float\"}", + NULL, &body) == 200); + cJSON *qwen_mrl = body ? parse_embeddings_body(body, 1) : NULL; + if (qwen_full && qwen_mrl) { + cJSON *full_data = + cJSON_GetObjectItemCaseSensitive(qwen_full, "data"); + cJSON *mrl_data = + cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data"); + cJSON *full_emb = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(full_data, 0), "embedding"); + cJSON *mrl_emb = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(mrl_data, 0), "embedding"); + TEST_ASSERT(cJSON_IsArray(full_emb) && + cJSON_GetArraySize(full_emb) == 1024); + TEST_ASSERT(cJSON_IsArray(mrl_emb) && + cJSON_GetArraySize(mrl_emb) == 32); + double prefix_diff = 0.0; + for (int i = 0; i < 32; i++) { + double d = fabs(cJSON_GetArrayItem(full_emb, i)->valuedouble - + cJSON_GetArrayItem(mrl_emb, i)->valuedouble); + if (d > prefix_diff) prefix_diff = d; + } + TEST_ASSERT(prefix_diff > 0.0); + cJSON *usage = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "usage"); + cJSON *total = usage + ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; + TEST_ASSERT(total && total->valueint == 2); + } + cJSON_Delete(qwen_full); + cJSON_Delete(qwen_mrl); + free(body); + body = NULL; + /* Concurrent clients: more requests in flight than the micro-batch cap * (-b 2) so the worker has to assemble several batches. */ enum { N_CONC = 6 }; @@ -713,7 +758,7 @@ static void test_http_rerank(int port) static void test_http_server(void) { - char dir[1024], late_dir[1024]; + char dir[1024], late_dir[1024], qwen_dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); if (!mkdtemp(dir)) { @@ -728,6 +773,13 @@ static void test_http_server(void) test_failures++; return; } + snprintf(qwen_dir, sizeof(qwen_dir), "%s/embed-srv-qwen-test-XXXXXX", + getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + if (!mkdtemp(qwen_dir)) { + fprintf(stderr, "FAIL: Qwen mkdtemp\n"); + test_failures++; + return; + } /* The 0.6b server slot requires hidden_size 1024; everything else stays * tiny. The model vocab covers every tokenizer fixture id. */ tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE}; @@ -735,7 +787,10 @@ static void test_http_server(void) tm_write_model_dims(dir, "F32", &dims) != 0 || tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 || - tm_write_late_projection(late_dir, "F32", 128, 1024) != 0) { + tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 || + tf_write_vocab(qwen_dir) != 0 || + tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, + TF_EOT_ID) != 0) { fprintf(stderr, "FAIL: fixture write\n"); test_failures++; return; @@ -754,8 +809,10 @@ static void test_http_server(void) ctx.spec[1].path = dir; ctx.spec[2].id = "pplx-embed-v1-late-0.6b"; ctx.spec[2].path = late_dir; + ctx.spec[3].id = "Qwen3-Embedding-0.6B"; + ctx.spec[3].path = qwen_dir; ctx.cfg.models = ctx.spec; - ctx.cfg.n_models = 3; + ctx.cfg.n_models = 4; ctx.cfg.port = port; ctx.cfg.batch_size = 2; ctx.cfg.batch_wait_us = 1000; diff --git a/tests/tiny_model.h b/tests/tiny_model.h index 8c23e2c..d4ae431 100644 --- a/tests/tiny_model.h +++ b/tests/tiny_model.h @@ -57,6 +57,27 @@ typedef struct { enum { TM_N_SPECS = 13 }; +static int tm_write_config(const char *dir, const tm_dims_t *d, + const char *model_type, int eos_token_id) +{ + char path[2048]; + snprintf(path, sizeof(path), "%s/config.json", dir); + FILE *f = fopen(path, "w"); + if (!f) return -1; + fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1," + "\"num_attention_heads\":%d,\"num_key_value_heads\":%d," + "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d," + "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0", + d->hidden, d->heads, d->kv_heads, d->head_dim, + d->intermediate, d->vocab); + if (model_type) + fprintf(f, ",\"model_type\":\"%s\",\"eos_token_id\":%d", + model_type, eos_token_id); + fputs("}", f); + fclose(f); + return 0; +} + /* Write a safetensors file holding `specs` with tm_value() contents. * dtype: "F32" or "BF16". Returns 0 on success. */ static int tm_write_safetensors(const char *path, const char *dtype, @@ -133,21 +154,20 @@ static int tm_write_model_dims(const char *dir, const char *dtype, }; char path[2048]; - snprintf(path, sizeof(path), "%s/config.json", dir); - FILE *f = fopen(path, "w"); - if (!f) return -1; - fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1," - "\"num_attention_heads\":%d,\"num_key_value_heads\":%d," - "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d," - "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}", - d->hidden, d->heads, d->kv_heads, d->head_dim, - d->intermediate, d->vocab); - fclose(f); + if (tm_write_config(dir, d, NULL, -1) != 0) return -1; snprintf(path, sizeof(path), "%s/model.safetensors", dir); return tm_write_safetensors(path, dtype, specs, TM_N_SPECS); } +static inline int tm_write_qwen3_model_dims(const char *dir, const char *dtype, + const tm_dims_t *d, + int eos_token_id) +{ + if (tm_write_model_dims(dir, dtype, d) != 0) return -1; + return tm_write_config(dir, d, "qwen3", eos_token_id); +} + /* Add the 1_Dense per-token projection head that turns a base model dir into * a late-interaction snapshot. weight_name is "linear.weight" for a valid * head; pass another name to synthesize a malformed head. */ |
