aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 12:16:33 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 12:16:33 +0200
commitc6ade0a14b87053d4a19cbac121d03af48cbf515 (patch)
treefccea2d9351e310ad308a2336abf766b4884cc11
parentb5ada41fedf1ebdaff39ee0fbef76a468b26e5f7 (diff)
downloadffwd-c6ade0a14b87053d4a19cbac121d03af48cbf515.tar.gz
ffwd-c6ade0a14b87053d4a19cbac121d03af48cbf515.zip
all: support GTE-Qwen2 embeddings
-rw-r--r--Makefile13
-rw-r--r--embed.c33
-rw-r--r--embed.h2
-rw-r--r--embed_config.c140
-rw-r--r--embed_cuda.cu72
-rw-r--r--embed_internal.h3
-rw-r--r--embed_mlx.c48
-rw-r--r--embed_server.c79
-rw-r--r--tests/test_qwen2.c97
-rw-r--r--tests/test_server.c87
-rw-r--r--tests/tiny_model.h85
11 files changed, 574 insertions, 85 deletions
diff --git a/Makefile b/Makefile
index af88f55..1bd2b23 100644
--- a/Makefile
+++ b/Makefile
@@ -169,6 +169,8 @@ TEST_BF16_SRCS = tests/test_bf16_model.c embed.c embed_config.c qwen_safe
$(KERNEL_SRCS)
TEST_QWEN3_SRCS = tests/test_qwen3.c embed.c embed_config.c qwen_safetensors.c \
$(KERNEL_SRCS)
+TEST_QWEN2_SRCS = tests/test_qwen2.c embed.c embed_config.c qwen_safetensors.c \
+ $(KERNEL_SRCS)
TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c embed_config.c qwen_tokenizer.c \
qwen_safetensors.c $(KERNEL_SRCS)
TEST_LATE_SRCS = tests/test_late.c embed.c embed_config.c qwen_tokenizer.c \
@@ -198,6 +200,9 @@ test:
$(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \
$(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
./tests/test_qwen3
+ $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \
+ $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ ./tests/test_qwen2
$(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \
$(TEST_WORKSPACE_SRCS) -lm -lpthread
./tests/test_workspace
@@ -236,7 +241,8 @@ COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \
-fcoverage-compilation-dir=.
COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \
tests/test_tokenizer tests/test_safetensors \
- tests/test_bf16_model tests/test_qwen3 tests/test_workspace tests/test_late \
+ tests/test_bf16_model tests/test_qwen3 tests/test_qwen2 tests/test_workspace \
+ tests/test_late \
tests/cli_under_test tests/test_server
# Report on project sources only - not the harnesses, vendored deps, or
# system headers (cJSON lands under /opt or /usr otherwise).
@@ -269,6 +275,9 @@ coverage:
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \
$(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
LLVM_PROFILE_FILE=$(COV_DIR)/qwen3.profraw ./tests/test_qwen3
+ $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \
+ $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ LLVM_PROFILE_FILE=$(COV_DIR)/qwen2.profraw ./tests/test_qwen2
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \
$(TEST_WORKSPACE_SRCS) -lm -lpthread
LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace
@@ -397,7 +406,7 @@ clean:
$(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \
tests/test_kernels_generic tests/test_kernels_blas \
tests/test_safetensors tests/test_bf16_model tests/test_server \
- tests/test_qwen3 \
+ tests/test_qwen3 tests/test_qwen2 \
tests/test_tokenizer tests/test_workspace tests/test_cli tests/test_late \
tests/cli_under_test bench/bench_tokenizer bench/bench_kernels bench/bench_model
rm -rf $(COV_DIR)
diff --git a/embed.c b/embed.c
index c05adfc..a49aa7d 100644
--- a/embed.c
+++ b/embed.c
@@ -347,6 +347,16 @@ static void linear_qkv_weight(embed_workspace_t *ws,
linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim);
}
+static void add_bias_rows(float *x, const float *bias, int rows, int dim) {
+ if (!bias)
+ return;
+ for (int row = 0; row < rows; row++) {
+ float *dst = x + (size_t)row * dim;
+ for (int d = 0; d < dim; d++)
+ dst[d] += bias[d];
+ }
+}
+
static void linear_pair_weight(embed_workspace_t *ws,
float *a,
float *b,
@@ -576,8 +586,15 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a
LOAD2(wk, "%slayers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size);
LOAD2(wv, "%slayers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size);
LOAD2(wo, "%slayers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim);
- LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim);
- LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim);
+ if (cfg->qkv_bias) {
+ LOAD_NORM(q_bias, "%slayers.%d.self_attn.q_proj.bias", cfg->q_dim);
+ LOAD_NORM(k_bias, "%slayers.%d.self_attn.k_proj.bias", cfg->kv_dim);
+ LOAD_NORM(v_bias, "%slayers.%d.self_attn.v_proj.bias", cfg->kv_dim);
+ }
+ if (cfg->qk_norm) {
+ LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim);
+ LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim);
+ }
LOAD_NORM(input_norm, "%slayers.%d.input_layernorm.weight", cfg->hidden_size);
LOAD_NORM(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", cfg->hidden_size);
LOAD2(gate_proj, "%slayers.%d.mlp.gate_proj.weight", cfg->intermediate_size,
@@ -625,6 +642,9 @@ void embed_model_free(embed_model_t *model) {
if (model->weights.layers) {
for (int i = 0; i < model->config.n_layers; i++) {
embed_layer_t *l = &model->weights.layers[i];
+ free((void *)l->q_bias);
+ free((void *)l->k_bias);
+ free((void *)l->v_bias);
free((void *)l->q_norm);
free((void *)l->k_norm);
free((void *)l->input_norm);
@@ -1089,8 +1109,13 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq,
hidden, q_dim, kv_dim);
- qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
- qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
+ add_bias_rows(q_buf, l->q_bias, total_seq, q_dim);
+ add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim);
+ add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim);
+ if (cfg->qk_norm) {
+ qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
+ qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
+ }
for (int b = 0; b < batch; b++) {
int start = offsets[b];
diff --git a/embed.h b/embed.h
index b84f71c..ef36359 100644
--- a/embed.h
+++ b/embed.h
@@ -54,6 +54,8 @@ typedef struct {
int vocab_size;
float rms_norm_eps;
float rope_theta;
+ int qk_norm;
+ int qkv_bias;
embed_attention_mode_t attention_mode;
embed_pooling_mode_t pooling_mode;
int normalize_embeddings;
diff --git a/embed_config.c b/embed_config.c
index 5bbfe48..4a17740 100644
--- a/embed_config.c
+++ b/embed_config.c
@@ -1,11 +1,20 @@
#include "embed_config.h"
+#include <ctype.h>
+#include <errno.h>
#include <limits.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+typedef enum {
+ JSON_BOOL_MISSING = 0,
+ JSON_BOOL_FALSE = 1,
+ JSON_BOOL_TRUE = 2,
+ JSON_BOOL_INVALID = -1,
+} json_bool_t;
+
static const char *skip_ws(const char *p) {
while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')
p++;
@@ -48,6 +57,112 @@ static int json_string_equals(const char *json, const char *key, const char *exp
return strncmp(v, expected, len) == 0 && v[len] == '"';
}
+static json_bool_t json_get_bool(const char *json, const char *key) {
+ const char *v = json_find_key(json, key);
+ if (!v)
+ return JSON_BOOL_MISSING;
+ if (strncmp(v, "true", 4) == 0 && !isalnum((unsigned char)v[4]) && v[4] != '_')
+ return JSON_BOOL_TRUE;
+ if (strncmp(v, "false", 5) == 0 && !isalnum((unsigned char)v[5]) && v[5] != '_')
+ return JSON_BOOL_FALSE;
+ return JSON_BOOL_INVALID;
+}
+
+/* Return 1 with a NUL-terminated buffer, 0 when an optional file is absent,
+ * and -1 for an existing file that could not be read. */
+static int read_optional_file(const char *model_dir, const char *relative, char **out) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ fprintf(stderr, "embed_config: model path too long\n");
+ return -1;
+ }
+
+ FILE *f = fopen(path, "rb");
+ if (!f && errno == ENOENT)
+ return 0;
+ if (!f) {
+ fprintf(stderr, "embed_config: cannot open %s\n", path);
+ return -1;
+ }
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ fclose(f);
+ free(buf);
+ return -1;
+ }
+ fclose(f);
+ buf[sz] = '\0';
+ *out = buf;
+ return 1;
+}
+
+static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char *model_dir) {
+ char *buf = NULL;
+ int rc = read_optional_file(model_dir, "1_Pooling/config.json", &buf);
+ if (rc < 0)
+ return -1;
+ if (rc > 0) {
+ json_bool_t mean = json_get_bool(buf, "pooling_mode_mean_tokens");
+ json_bool_t last = json_get_bool(buf, "pooling_mode_lasttoken");
+ json_bool_t cls = json_get_bool(buf, "pooling_mode_cls_token");
+ json_bool_t max = json_get_bool(buf, "pooling_mode_max_tokens");
+ json_bool_t sqrt_len = json_get_bool(buf, "pooling_mode_mean_sqrt_len_tokens");
+ json_bool_t weighted = json_get_bool(buf, "pooling_mode_weightedmean_tokens");
+ if (mean < 0 || last < 0 || cls < 0 || max < 0 || sqrt_len < 0 || weighted < 0) {
+ fprintf(stderr, "embed_config: invalid pooling metadata\n");
+ free(buf);
+ return -1;
+ }
+ int supported = (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE);
+ int unsupported = cls == JSON_BOOL_TRUE || max == JSON_BOOL_TRUE ||
+ sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE;
+ if (supported != 1 || unsupported) {
+ fprintf(stderr, "embed_config: unsupported Sentence Transformers pooling mode\n");
+ free(buf);
+ return -1;
+ }
+ cfg->pooling_mode = last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN : EMBED_POOL_MEAN;
+ free(buf);
+ buf = NULL;
+ }
+
+ rc = read_optional_file(model_dir, "modules.json", &buf);
+ if (rc < 0)
+ return -1;
+ if (rc > 0) {
+ cfg->normalize_embeddings =
+ strstr(buf, "\"sentence_transformers.models.Normalize\"") != NULL;
+ free(buf);
+ buf = NULL;
+ }
+
+ rc = read_optional_file(model_dir, "tokenizer_config.json", &buf);
+ if (rc < 0)
+ return -1;
+ if (rc > 0) {
+ json_bool_t add_eos = json_get_bool(buf, "add_eos_token");
+ if (add_eos < 0) {
+ fprintf(stderr, "embed_config: invalid add_eos_token metadata\n");
+ free(buf);
+ return -1;
+ }
+ if (add_eos != JSON_BOOL_MISSING)
+ cfg->append_terminal_token = add_eos == JSON_BOOL_TRUE;
+ free(buf);
+ }
+ return 0;
+}
+
int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
@@ -89,6 +204,8 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
+ cfg->qk_norm = 1;
+ cfg->qkv_bias = 0;
cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
cfg->pooling_mode = EMBED_POOL_MEAN;
cfg->normalize_embeddings = 0;
@@ -99,6 +216,21 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->pooling_mode = EMBED_POOL_LAST_TOKEN;
cfg->normalize_embeddings = 1;
cfg->append_terminal_token = 1;
+ } else if (json_string_equals(buf, "model_type", "qwen2")) {
+ cfg->qk_norm = 0;
+ cfg->qkv_bias = 1;
+ cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
+ }
+
+ json_bool_t is_causal = json_get_bool(buf, "is_causal");
+ if (is_causal < 0) {
+ fprintf(stderr, "embed_config: invalid is_causal in %s\n", path);
+ free(buf);
+ return -1;
+ }
+ if (is_causal != JSON_BOOL_MISSING) {
+ cfg->attention_mode =
+ is_causal == JSON_BOOL_TRUE ? EMBED_ATTENTION_CAUSAL : EMBED_ATTENTION_BIDIRECTIONAL;
}
cfg->q_dim = 0;
@@ -110,6 +242,9 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
free(buf);
+ if (parse_sentence_transformers_metadata(cfg, model_dir) != 0)
+ return -1;
+
if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 ||
cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) ||
@@ -131,11 +266,12 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
if (embed_verbose >= 1)
fprintf(stderr,
"config: hidden=%d, layers=%d, heads=%d/%d, "
- "inter=%d, head_dim=%d, attention=%s, pooling=%s\n",
+ "inter=%d, head_dim=%d, attention=%s, pooling=%s, qk_norm=%s, qkv_bias=%s\n",
cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads,
cfg->intermediate_size, cfg->head_dim,
cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional",
- cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean");
+ cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean",
+ cfg->qk_norm ? "yes" : "no", cfg->qkv_bias ? "yes" : "no");
return 0;
}
diff --git a/embed_cuda.cu b/embed_cuda.cu
index ba78f09..3100f44 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -23,6 +23,7 @@ typedef struct {
typedef struct {
cuda_matrix_t qkv, wo;
+ float *qkv_bias;
float *q_norm;
float *k_norm;
float *input_norm;
@@ -303,6 +304,26 @@ static int load_vector(float **out, const float *src, int n) {
return 0;
}
+static int load_qkv_bias(float **out, const embed_layer_t *src, const embed_config_t *c) {
+ int n = c->q_dim + 2 * c->kv_dim;
+ float *host = (float *)malloc((size_t)n * sizeof(float));
+ if (!host)
+ return -1;
+ memcpy(host, src->q_bias, (size_t)c->q_dim * sizeof(float));
+ memcpy(host + c->q_dim, src->k_bias, (size_t)c->kv_dim * sizeof(float));
+ memcpy(host + c->q_dim + c->kv_dim, src->v_bias, (size_t)c->kv_dim * sizeof(float));
+ int rc = load_vector(out, host, n);
+ free(host);
+ return rc;
+}
+
+__global__ static void add_row_bias_kernel(float *x, const float *bias, int rows, int dim) {
+ int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ int count = rows * dim;
+ if (idx < count)
+ x[idx] += bias[idx % dim];
+}
+
__global__ static void embed_lookup_kernel(
float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
@@ -391,6 +412,7 @@ __global__ static void rms_norm_rope_qk_kernel(float *x,
int row_stride,
int q_offset,
int k_offset,
+ int qk_norm,
float eps) {
__shared__ float partial[4];
__shared__ float total;
@@ -409,29 +431,34 @@ __global__ static void rms_norm_rope_qk_kernel(float *x,
size_t base = (size_t)row * row_stride + base_offset + head * head_dim;
float v = tid < head_dim ? x[base + tid] : 0.0f;
float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f;
- float sum = v * v;
-
- for (int offset = 16; offset > 0; offset >>= 1)
- sum += __shfl_down_sync(0xffffffff, sum, offset);
- if (lane == 0)
- partial[warp] = sum;
- __syncthreads();
- if (warp == 0) {
- sum = lane < 4 ? partial[lane] : 0.0f;
+ if (qk_norm) {
+ float sum = v * v;
for (int offset = 16; offset > 0; offset >>= 1)
sum += __shfl_down_sync(0xffffffff, sum, offset);
if (lane == 0)
- total = sum;
+ partial[warp] = sum;
+ __syncthreads();
+ if (warp == 0) {
+ sum = lane < 4 ? partial[lane] : 0.0f;
+ for (int offset = 16; offset > 0; offset >>= 1)
+ sum += __shfl_down_sync(0xffffffff, sum, offset);
+ if (lane == 0)
+ total = sum;
+ }
+ __syncthreads();
}
- __syncthreads();
if (tid < head_dim) {
- float inv = rsqrtf(total / (float)head_dim + eps);
int pos = positions[row];
float c = cosv[(size_t)pos * head_dim + tid];
float s = sinv[(size_t)pos * head_dim + tid];
- float a = v * inv * weight[tid];
- float b = pair_v * inv * weight[pair_d];
+ float a = v;
+ float b = pair_v;
+ if (qk_norm) {
+ float inv = rsqrtf(total / (float)head_dim + eps);
+ a *= inv * weight[tid];
+ b *= inv * weight[pair_d];
+ }
x[base + tid] = a * c + sign * b * s;
}
}
@@ -1081,8 +1108,9 @@ static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) {
static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
return load_qkv_matrix(&dst->qkv, src, c) ||
load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
- load_vector(&dst->q_norm, src->q_norm, c->head_dim) ||
- load_vector(&dst->k_norm, src->k_norm, c->head_dim) ||
+ (c->qkv_bias && load_qkv_bias(&dst->qkv_bias, src, c)) ||
+ (c->qk_norm && load_vector(&dst->q_norm, src->q_norm, c->head_dim)) ||
+ (c->qk_norm && load_vector(&dst->k_norm, src->k_norm, c->head_dim)) ||
load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
load_gate_up_matrix(&dst->gate_up_proj, src, c) ||
@@ -1092,6 +1120,7 @@ static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_c
static void free_layer(cuda_layer_t *l) {
cuda_matrix_free(&l->qkv);
cuda_matrix_free(&l->wo);
+ cudaFree(l->qkv_bias);
cudaFree(l->q_norm);
cudaFree(l->k_norm);
cudaFree(l->input_norm);
@@ -1303,9 +1332,18 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
return -1;
}
+ if (c->qkv_bias) {
+ int count = total * qkv_dim;
+ add_row_bias_kernel<<<(count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias,
+ total, qkv_dim);
+ if (launch_check() != 0)
+ return -1;
+ }
+
rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>(
ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total,
- c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->rms_norm_eps);
+ c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->qk_norm,
+ c->rms_norm_eps);
if (launch_check() != 0)
return -1;
diff --git a/embed_internal.h b/embed_internal.h
index 7d6977c..0536692 100644
--- a/embed_internal.h
+++ b/embed_internal.h
@@ -14,6 +14,9 @@ typedef struct {
embed_weight_ref_t wk; /* [kv_dim, hidden] */
embed_weight_ref_t wv; /* [kv_dim, hidden] */
embed_weight_ref_t wo; /* [hidden, q_dim] */
+ const float *q_bias; /* [q_dim], optional */
+ const float *k_bias; /* [kv_dim], optional */
+ const float *v_bias; /* [kv_dim], optional */
const float *q_norm; /* [head_dim] */
const float *k_norm; /* [head_dim] */
const float *input_norm; /* [hidden] */
diff --git a/embed_mlx.c b/embed_mlx.c
index de82f83..d85dc0d 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -28,6 +28,7 @@ typedef struct {
typedef struct {
mlx_linear_t wq, wk, wv, wo;
+ mlx_array q_bias, k_bias, v_bias;
mlx_array q_norm, k_norm;
mlx_array input_norm, post_attn_norm;
mlx_linear_t gate_proj, up_proj, down_proj;
@@ -469,8 +470,15 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
goto fail; \
} while (0)
- LD(q_norm, "%slayers.%d.self_attn.q_norm.weight", hd);
- LD(k_norm, "%slayers.%d.self_attn.k_norm.weight", hd);
+ if (c->qkv_bias) {
+ LD(q_bias, "%slayers.%d.self_attn.q_proj.bias", qd);
+ LD(k_bias, "%slayers.%d.self_attn.k_proj.bias", kvd);
+ LD(v_bias, "%slayers.%d.self_attn.v_proj.bias", kvd);
+ }
+ if (c->qk_norm) {
+ LD(q_norm, "%slayers.%d.self_attn.q_norm.weight", hd);
+ LD(k_norm, "%slayers.%d.self_attn.k_norm.weight", hd);
+ }
LD(input_norm, "%slayers.%d.input_layernorm.weight", h);
LD(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", h);
#undef LD
@@ -509,6 +517,9 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) {
for (int i = layer_start; i < layer_end; i++) {
mlx_layer_t *l = &ctx->layers[i];
+ upcast_f32_inplace(&l->q_bias, ctx->stream);
+ upcast_f32_inplace(&l->k_bias, ctx->stream);
+ upcast_f32_inplace(&l->v_bias, ctx->stream);
upcast_f32_inplace(&l->q_norm, ctx->stream);
upcast_f32_inplace(&l->k_norm, ctx->stream);
upcast_f32_inplace(&l->input_norm, ctx->stream);
@@ -576,6 +587,9 @@ static void free_mlx_layer(mlx_layer_t *l) {
free_linear(&l->wk);
free_linear(&l->wv);
free_linear(&l->wo);
+ mlx_array_free(l->q_bias);
+ mlx_array_free(l->k_bias);
+ mlx_array_free(l->v_bias);
mlx_array_free(l->q_norm);
mlx_array_free(l->k_norm);
mlx_array_free(l->input_norm);
@@ -733,6 +747,20 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
mlx_array k_flat = linear(xn, &l->wk, S);
mlx_array v_flat = linear(xn, &l->wv, S);
mlx_array_free(xn);
+ if (c->qkv_bias) {
+ mlx_array biased = mlx_array_new();
+ mlx_add(&biased, q_flat, l->q_bias, S);
+ mlx_array_free(q_flat);
+ q_flat = biased;
+ biased = mlx_array_new();
+ mlx_add(&biased, k_flat, l->k_bias, S);
+ mlx_array_free(k_flat);
+ k_flat = biased;
+ biased = mlx_array_new();
+ mlx_add(&biased, v_flat, l->v_bias, S);
+ mlx_array_free(v_flat);
+ v_flat = biased;
+ }
int q_shape[] = {batch, max_seq, n_heads, head_dim};
int k_shape[] = {batch, max_seq, n_kv_heads, head_dim};
@@ -746,12 +774,16 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
mlx_array_free(k_flat);
mlx_array_free(v_flat);
- mlx_array qn = mlx_array_new();
- mlx_array kn = mlx_array_new();
- mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
- mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
- mlx_array_free(q);
- mlx_array_free(k);
+ mlx_array qn = q;
+ mlx_array kn = k;
+ if (c->qk_norm) {
+ qn = mlx_array_new();
+ kn = mlx_array_new();
+ mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
+ mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
+ mlx_array_free(q);
+ mlx_array_free(k);
+ }
/* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */
int perm[] = {0, 2, 1, 3};
diff --git a/embed_server.c b/embed_server.c
index cf3a9ec..b62bcfb 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -66,13 +66,14 @@
#define EMBED_LATE_MASK_TOKEN_ID 151642
#define EMBED_LATE_QUERY_PREFIX_ID 151669
#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
-/* Qwen3/DashScope `text_type: query` prepends this default retrieval instruction
- * in the Qwen3-Embedding format ("Instruct: {task}\nQuery:{query}"); documents
- * and the default pass through unchanged. The task text matches the
- * Qwen3-Embedding reference default; callers needing a custom instruction still
- * prepend their own text and leave text_type at "document". */
+/* `text_type: query` prepends the model's published retrieval instruction.
+ * Documents and the default pass through unchanged. Callers needing a custom
+ * instruction prepend it themselves and leave text_type at "document". */
#define EMBED_QWEN3_QUERY_INSTRUCT \
"Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"
+#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \
+ "Instruct: Given a web search query, retrieve relevant passages that answer the " \
+ "query\nQuery: "
/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
* short-request throughput over 8 with no long-document penalty, while 128
* was no faster and inflated long-document tail latency. */
@@ -227,6 +228,7 @@ typedef enum {
MODEL_QWEN3_06,
MODEL_QWEN3_4,
MODEL_QWEN3_8,
+ MODEL_GTE_QWEN2_15,
MODEL_CTX_06,
MODEL_CTX_4,
MODEL_LATE_06,
@@ -239,10 +241,10 @@ typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } mod
/* Which embedding API a model speaks. pplx-embed models follow the Perplexity
* API: the canonical output is the tanh int8 quantization, so the default
* encoding is base64_int8 and "float" is the int8-decoded view (int8/128).
- * Qwen3-Embedding models follow the Qwen/DashScope API, which is OpenAI-
- * compatible: "float" is the true float32 vector and "base64" is base64 of
- * little-endian float32. int8 quantization underuses the range of Qwen3's
- * L2-normalized vectors, so it is never the Qwen3 wire format. */
+ * Instruction embedding models follow the OpenAI-compatible API: "float" is
+ * the true float32 vector and "base64" is base64 of little-endian float32.
+ * int8 quantization underuses the range of L2-normalized vectors, so it is not
+ * used for these models. */
typedef enum {
EMBED_API_PERPLEXITY = 0,
EMBED_API_OPENAI = 1,
@@ -258,25 +260,28 @@ typedef struct {
embed_pooling_mode_t pooling_mode;
int normalize_embeddings;
embedding_api_t api;
+ const char *query_instruct;
} model_info;
static const model_info k_models[] = {
{"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
{"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
{"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
{"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
{"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL,
- EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI},
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT},
+ {"gte-Qwen2-1.5B-instruct", MODEL_KIND_STANDARD, 1536, 1536, 0, EMBED_ATTENTION_BIDIRECTIONAL,
+ EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_GTE_QWEN2_QUERY_INSTRUCT},
{"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0,
- EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
{"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
{"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL,
- EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY},
+ EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
};
static model_slot model_slot_for_id(const char *id) {
@@ -1242,18 +1247,16 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_
return v;
}
-/* Parse the Qwen3/DashScope `text_type` hint. Returns 1 when the query
- * retrieval instruction should be prepended, 0 otherwise (document / absent).
- * The field belongs to the OpenAI/DashScope (Qwen3) family only; on a Perplexity
- * model it is rejected, since pplx-embed has no instruction concept. Invalid
- * values are reported into detail and treated as the no-op default. */
-static int text_type_is_query(cJSON *root, cJSON *detail, embedding_api_t api) {
+/* Parse the `text_type` hint. Returns 1 when the model's query instruction
+ * should be prepended, 0 otherwise (document / absent). Models without a
+ * published query instruction reject the field. */
+static int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) {
cJSON *tt = cJSON_GetObjectItemCaseSensitive(root, "text_type");
if (!tt)
return 0;
- if (api != EMBED_API_OPENAI) {
+ if (!query_instruct) {
ve_add(detail, "[\"body\",\"text_type\"]",
- "text_type is only supported for Qwen3-Embedding models", "extra_fields");
+ "text_type is only supported for instruction embedding models", "extra_fields");
return 0;
}
if (!cJSON_IsString(tt) || !tt->valuestring) {
@@ -1497,22 +1500,20 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *ou
return 0;
}
-/* Tokenize one embedding input, prepending the Qwen3 query instruction when
- * query_instruct is set. The instruction is part of the text the model sees, so
- * it is tokenized in one pass with the input (and the terminal token, if any, is
- * appended by tokenize_one as usual). */
-static int
-tokenize_input(loaded_model *m, job *j, const char *text, int query_instruct, token_buf *out) {
+/* Tokenize one embedding input, optionally prepending the model's query
+ * instruction. The instruction and input are tokenized in one pass. */
+static int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
if (!query_instruct)
return tokenize_one(m, j, text, out);
- size_t plen = strlen(EMBED_QWEN3_QUERY_INSTRUCT);
+ size_t plen = strlen(query_instruct);
size_t tlen = strlen(text);
char *buf = (char *)malloc(plen + tlen + 1);
if (!buf) {
memset(out, 0, sizeof(*out));
return -1;
}
- memcpy(buf, EMBED_QWEN3_QUERY_INSTRUCT, plen);
+ memcpy(buf, query_instruct, plen);
memcpy(buf + plen, text, tlen + 1);
int rc = tokenize_one(m, j, buf, out);
free(buf);
@@ -2003,15 +2004,17 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embed
out->dims = dims;
out->encoding = encoding;
- /* Resolve the API family from the requested model (independent of load
- * state) to validate the Qwen3/DashScope text_type hint. query_instruct is
- * applied to every input in the request. */
+ /* Resolve request behavior from the model registry independent of load
+ * state. The query instruction, when requested, applies to every input. */
cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring)
? model_slot_for_id(model_item->valuestring)
: MODEL_UNKNOWN;
- embedding_api_t api = api_slot != MODEL_UNKNOWN ? k_models[api_slot].api : EMBED_API_PERPLEXITY;
- int query_instruct = text_type_is_query(root, detail, api);
+ const char *query_instruct =
+ api_slot != MODEL_UNKNOWN &&
+ text_type_is_query(root, detail, k_models[api_slot].query_instruct)
+ ? k_models[api_slot].query_instruct
+ : NULL;
cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
int n_inputs = 0;
diff --git a/tests/test_qwen2.c b/tests/test_qwen2.c
new file mode 100644
index 0000000..4795f7f
--- /dev/null
+++ b/tests/test_qwen2.c
@@ -0,0 +1,97 @@
+/* tests/test_qwen2.c - hermetic GTE-Qwen2 block and task semantics. */
+
+#include "embed.h"
+#include "tiny_model.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static float max_abs_diff(const float *a, const float *b, int n) {
+ float out = 0.0f;
+ for (int i = 0; i < n; i++) {
+ float d = fabsf(a[i] - b[i]);
+ if (d > out)
+ out = d;
+ }
+ return out;
+}
+
+int main(void) {
+ const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
+ char dir[1024], zero_dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-qwen2-test-XXXXXX", tmp);
+ snprintf(zero_dir, sizeof(zero_dir), "%s/embed-qwen2-zero-bias-test-XXXXXX", tmp);
+ tm_dims_t dims = {4, 2, 1, 2, 8, 16};
+ if (!mkdtemp(dir) || !mkdtemp(zero_dir) ||
+ tm_write_qwen2_model_dims(dir, "F32", &dims, 15, 0) != 0 ||
+ tm_write_qwen2_model_dims(zero_dir, "F32", &dims, 15, 1) != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ embed_model_t *model = embed_model_load(dir);
+ embed_model_t *zero_model = embed_model_load(zero_dir);
+ embed_workspace_t *ws = embed_workspace_new(model);
+ embed_workspace_t *zero_ws = embed_workspace_new(zero_model);
+ const embed_config_t *cfg = embed_model_config(model);
+ if (!model || !zero_model || !ws || !zero_ws || !cfg) {
+ fprintf(stderr, "model setup failed\n");
+ return 1;
+ }
+ if (cfg->qk_norm || !cfg->qkv_bias || cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL ||
+ cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || !cfg->normalize_embeddings ||
+ !cfg->append_terminal_token) {
+ fprintf(stderr, "Qwen2 config or Sentence Transformers metadata was not detected\n");
+ return 1;
+ }
+
+ int ids[] = {1, 2, 15};
+ float states[3 * 4], embedding[4], pooled[4], zero_embedding[4];
+ if (embed_model_forward_into(model, ws, ids, 3, states) != 0 ||
+ embed_model_encode_into(model, ws, ids, 3, embedding) != 0 ||
+ embed_model_encode_into(zero_model, zero_ws, ids, 3, zero_embedding) != 0 ||
+ embed_pool_batch(cfg, states, (int[]){3}, 1, pooled) != 0) {
+ fprintf(stderr, "Qwen2 inference failed\n");
+ return 1;
+ }
+
+ float expected[4];
+ memcpy(expected, states + 2 * 4, sizeof(expected));
+ if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f ||
+ max_abs_diff(pooled, expected, 4) > 1e-5f) {
+ fprintf(stderr, "last-token pooling mismatch\n");
+ return 1;
+ }
+ if (max_abs_diff(embedding, zero_embedding, 4) < 1e-4f) {
+ fprintf(stderr, "QKV projection bias did not affect the embedding\n");
+ return 1;
+ }
+
+ int changed_ids[] = {1, 3, 15};
+ float changed_states[3 * 4];
+ if (embed_model_forward_into(model, ws, changed_ids, 3, changed_states) != 0 ||
+ max_abs_diff(states, changed_states, 4) < 1e-6f) {
+ fprintf(stderr, "bidirectional attention did not affect the first token\n");
+ return 1;
+ }
+
+ int short_ids[] = {4, 15};
+ embed_input_t inputs[] = {{ids, 3}, {short_ids, 2}};
+ float batch[8], single[4];
+ if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 ||
+ embed_model_encode_into(model, ws, short_ids, 2, single) != 0 ||
+ max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) {
+ fprintf(stderr, "Qwen2 packed batch parity failed\n");
+ return 1;
+ }
+
+ embed_workspace_free(zero_ws);
+ embed_workspace_free(ws);
+ embed_model_free(zero_model);
+ embed_model_free(model);
+ puts("ok: Qwen2 bias, bidirectional attention, and GTE pooling semantics");
+ return 0;
+}
diff --git a/tests/test_server.c b/tests/test_server.c
index ce9668e..9ebb8f2 100644
--- a/tests/test_server.c
+++ b/tests/test_server.c
@@ -146,41 +146,54 @@ static void test_encoding_from_root_family(void) {
static void test_text_type(void) {
cJSON *detail = cJSON_CreateArray();
- /* Absent: no-op for both families, no error. */
+ /* Absent: no-op for both model types, no error. */
cJSON *empty = cJSON_CreateObject();
- TEST_ASSERT(text_type_is_query(empty, detail, EMBED_API_OPENAI) == 0);
- TEST_ASSERT(text_type_is_query(empty, detail, EMBED_API_PERPLEXITY) == 0);
+ TEST_ASSERT(text_type_is_query(empty, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0);
+ TEST_ASSERT(text_type_is_query(empty, detail, NULL) == 0);
cJSON_Delete(empty);
TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
/* Qwen3: query -> 1, document -> 0, both accepted. */
cJSON *q = cJSON_CreateObject();
cJSON_AddStringToObject(q, "text_type", "query");
- TEST_ASSERT(text_type_is_query(q, detail, EMBED_API_OPENAI) == 1);
+ TEST_ASSERT(text_type_is_query(q, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 1);
cJSON_Delete(q);
cJSON *d = cJSON_CreateObject();
cJSON_AddStringToObject(d, "text_type", "document");
- TEST_ASSERT(text_type_is_query(d, detail, EMBED_API_OPENAI) == 0);
+ TEST_ASSERT(text_type_is_query(d, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0);
cJSON_Delete(d);
TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
/* Invalid enum value on Qwen3 is reported. */
cJSON *bad = cJSON_CreateObject();
cJSON_AddStringToObject(bad, "text_type", "passage");
- TEST_ASSERT(text_type_is_query(bad, detail, EMBED_API_OPENAI) == 0);
+ TEST_ASSERT(text_type_is_query(bad, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0);
TEST_ASSERT(cJSON_GetArraySize(detail) == 1);
cJSON_Delete(bad);
/* text_type on a Perplexity-family model is rejected. */
cJSON *p = cJSON_CreateObject();
cJSON_AddStringToObject(p, "text_type", "query");
- TEST_ASSERT(text_type_is_query(p, detail, EMBED_API_PERPLEXITY) == 0);
+ TEST_ASSERT(text_type_is_query(p, detail, NULL) == 0);
TEST_ASSERT(cJSON_GetArraySize(detail) == 2);
cJSON_Delete(p);
cJSON_Delete(detail);
}
+static void test_model_registry(void) {
+ model_slot slot = model_slot_for_id("gte-Qwen2-1.5B-instruct");
+ TEST_ASSERT(slot == MODEL_GTE_QWEN2_15);
+ if (slot == MODEL_UNKNOWN)
+ return;
+ TEST_ASSERT(k_models[slot].dim == 1536);
+ TEST_ASSERT(k_models[slot].min_dim == 1536);
+ TEST_ASSERT(k_models[slot].attention_mode == EMBED_ATTENTION_BIDIRECTIONAL);
+ TEST_ASSERT(k_models[slot].pooling_mode == EMBED_POOL_LAST_TOKEN);
+ TEST_ASSERT(k_models[slot].normalize_embeddings == 1);
+ TEST_ASSERT(strcmp(k_models[slot].query_instruct, EMBED_GTE_QWEN2_QUERY_INSTRUCT) == 0);
+}
+
static void test_collect_job_batch_deadline(void) {
http_server s;
memset(&s, 0, sizeof(s));
@@ -410,7 +423,7 @@ static int http_req(int port,
}
typedef struct {
- embed_server_model_spec_t spec[4];
+ embed_server_model_spec_t spec[5];
embed_server_config_t cfg;
int rc;
} srv_ctx;
@@ -657,6 +670,45 @@ static void test_http_embeddings(int port) {
cJSON_Delete(tt_query);
cJSON_Delete(tt_manual);
+ /* GTE-Qwen2 uses the same retrieval task text but publishes a space after
+ * "Query:". Verify the registry-specific prompt through the full path. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"gte-Qwen2-1.5B-instruct\",\"input\":\"hello\","
+ "\"text_type\":\"query\",\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ cJSON *gte_query = body ? parse_embeddings_body(body, 1) : NULL;
+ free(body);
+ body = NULL;
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"gte-Qwen2-1.5B-instruct\",\"input\":\"Instruct: Given a "
+ "web search query, retrieve relevant passages that answer the "
+ "query\\nQuery: hello\",\"text_type\":\"document\","
+ "\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ cJSON *gte_manual = body ? parse_embeddings_body(body, 1) : NULL;
+ free(body);
+ body = NULL;
+ if (gte_query && gte_manual) {
+ cJSON *qe = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(gte_query, "data"), 0),
+ "embedding");
+ cJSON *me = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(gte_manual, "data"), 0),
+ "embedding");
+ TEST_ASSERT(cJSON_IsArray(qe) && cJSON_IsArray(me));
+ TEST_ASSERT(cJSON_GetArraySize(qe) == 1536 && cJSON_GetArraySize(me) == 1536);
+ double gte_diff = 0.0;
+ for (int i = 0; i < cJSON_GetArraySize(qe); i++) {
+ double d = fabs(cJSON_GetArrayItem(qe, i)->valuedouble -
+ cJSON_GetArrayItem(me, i)->valuedouble);
+ if (d > gte_diff)
+ gte_diff = d;
+ }
+ TEST_ASSERT(gte_diff < 1e-6);
+ }
+ cJSON_Delete(gte_query);
+ cJSON_Delete(gte_manual);
+
/* text_type is rejected on a Perplexity-family model, and bad values 422. */
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello\","
@@ -897,7 +949,7 @@ static void test_http_rerank(int port) {
}
static void test_http_server(void) {
- char dir[1024], late_dir[1024], qwen_dir[1024];
+ char dir[1024], late_dir[1024], qwen_dir[1024], gte_dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
if (!mkdtemp(dir)) {
@@ -919,14 +971,24 @@ static void test_http_server(void) {
test_failures++;
return;
}
+ snprintf(gte_dir, sizeof(gte_dir), "%s/embed-srv-gte-test-XXXXXX",
+ getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
+ if (!mkdtemp(gte_dir)) {
+ fprintf(stderr, "FAIL: GTE mkdtemp\n");
+ test_failures++;
+ return;
+ }
/* The 0.6b server slot requires hidden_size 1024; everything else stays
* tiny. The model vocab covers every tokenizer fixture id. */
tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE};
+ tm_dims_t gte_dims = {1536, 2, 1, 64, 8, TF_VOCAB_SIZE};
if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 ||
tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 ||
tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 ||
tf_write_vocab(qwen_dir) != 0 ||
- tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0) {
+ tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0 ||
+ tf_write_vocab(gte_dir) != 0 ||
+ tm_write_qwen2_model_dims(gte_dir, "F32", &gte_dims, TF_EOT_ID, 0) != 0) {
fprintf(stderr, "FAIL: fixture write\n");
test_failures++;
return;
@@ -947,8 +1009,10 @@ static void test_http_server(void) {
ctx.spec[2].path = late_dir;
ctx.spec[3].id = "Qwen3-Embedding-0.6B";
ctx.spec[3].path = qwen_dir;
+ ctx.spec[4].id = "gte-Qwen2-1.5B-instruct";
+ ctx.spec[4].path = gte_dir;
ctx.cfg.models = ctx.spec;
- ctx.cfg.n_models = 4;
+ ctx.cfg.n_models = 5;
ctx.cfg.port = port;
ctx.cfg.batch_size = 2;
ctx.cfg.batch_wait_us = 1000;
@@ -992,6 +1056,7 @@ int main(void) {
test_append_embedding_value_float();
test_encoding_from_root_family();
test_text_type();
+ test_model_registry();
test_collect_job_batch_deadline();
test_collect_job_batch_zero_wait();
test_http_server();
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index d2534d2..95496b7 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -76,8 +76,12 @@ tm_write_config(const char *dir, const tm_dims_t *d, const char *model_type, int
/* Write a safetensors file holding `specs` with tm_value() contents.
* dtype: "F32" or "BF16". Returns 0 on success. */
-static int tm_write_safetensors_with_prefix(
- const char *path, const char *dtype, const tm_spec_t *specs, int n_specs, const char *prefix) {
+static int tm_write_safetensors_with_prefix_options(const char *path,
+ const char *dtype,
+ const tm_spec_t *specs,
+ int n_specs,
+ const char *prefix,
+ int zero_bias) {
int bf16 = strcmp(dtype, "BF16") == 0;
size_t esize = bf16 ? 2 : 4;
@@ -117,7 +121,7 @@ static int tm_write_safetensors_with_prefix(
const tm_spec_t *s = &specs[t];
size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1);
for (size_t i = 0; i < n; i++) {
- float v = tm_value(s->name, i);
+ float v = zero_bias && strstr(s->name, ".bias") ? 0.0f : tm_value(s->name, i);
if (bf16) {
uint16_t h = tm_f32_to_bf16(v);
fwrite(&h, sizeof(h), 1, f);
@@ -130,6 +134,11 @@ static int tm_write_safetensors_with_prefix(
return 0;
}
+static int tm_write_safetensors_with_prefix(
+ const char *path, const char *dtype, const tm_spec_t *specs, int n_specs, const char *prefix) {
+ return tm_write_safetensors_with_prefix_options(path, dtype, specs, n_specs, prefix, 0);
+}
+
static int
tm_write_safetensors(const char *path, const char *dtype, const tm_spec_t *specs, int n_specs) {
return tm_write_safetensors_with_prefix(path, dtype, specs, n_specs, "");
@@ -184,6 +193,76 @@ static inline int tm_write_qwen3_model_dims(const char *dir,
return tm_write_config(dir, d, "qwen3", eos_token_id);
}
+static inline int tm_write_qwen2_model_dims(
+ const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) {
+ int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
+ const tm_spec_t specs[] = {
+ {"embed_tokens.weight", d->vocab, d->hidden},
+ {"layers.0.self_attn.q_proj.weight", q, d->hidden},
+ {"layers.0.self_attn.q_proj.bias", q, 0},
+ {"layers.0.self_attn.k_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.k_proj.bias", kv, 0},
+ {"layers.0.self_attn.v_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.v_proj.bias", kv, 0},
+ {"layers.0.self_attn.o_proj.weight", d->hidden, q},
+ {"layers.0.input_layernorm.weight", d->hidden, 0},
+ {"layers.0.post_attention_layernorm.weight", d->hidden, 0},
+ {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate},
+ {"norm.weight", d->hidden, 0},
+ };
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fprintf(f,
+ "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
+ "\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
+ "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
+ "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0,"
+ "\"model_type\":\"qwen2\",\"eos_token_id\":%d,\"is_causal\":false}",
+ d->hidden, d->heads, d->kv_heads, d->head_dim, d->intermediate, d->vocab, eos_token_id);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/model.safetensors", dir);
+ if (tm_write_safetensors_with_prefix_options(
+ path, dtype, specs, (int)(sizeof(specs) / sizeof(specs[0])), "model.", zero_bias) != 0)
+ return -1;
+
+ snprintf(path, sizeof(path), "%s/1_Pooling", dir);
+ if (mkdir(path, 0755) != 0 && errno != EEXIST)
+ return -1;
+ snprintf(path, sizeof(path), "%s/1_Pooling/config.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fputs("{\"pooling_mode_cls_token\":false,\"pooling_mode_mean_tokens\":false,"
+ "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false,"
+ "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":true}",
+ f);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/modules.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fputs("[{\"type\":\"sentence_transformers.models.Transformer\"},"
+ "{\"type\":\"sentence_transformers.models.Pooling\"},"
+ "{\"type\":\"sentence_transformers.models.Normalize\"}]",
+ f);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/tokenizer_config.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fputs("{\"add_eos_token\":true}", f);
+ fclose(f);
+ return 0;
+}
+
/* Add the 1_Dense per-token projection head that turns a base model dir into
* a late-interaction snapshot. weight_name is "linear.weight" for a valid
* head; pass another name to synthesize a malformed head. */