diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 12:16:33 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 12:16:33 +0200 |
| commit | c6ade0a14b87053d4a19cbac121d03af48cbf515 (patch) | |
| tree | fccea2d9351e310ad308a2336abf766b4884cc11 | |
| parent | b5ada41fedf1ebdaff39ee0fbef76a468b26e5f7 (diff) | |
| download | ffwd-c6ade0a14b87053d4a19cbac121d03af48cbf515.tar.gz ffwd-c6ade0a14b87053d4a19cbac121d03af48cbf515.zip | |
all: support GTE-Qwen2 embeddings
| -rw-r--r-- | Makefile | 13 | ||||
| -rw-r--r-- | embed.c | 33 | ||||
| -rw-r--r-- | embed.h | 2 | ||||
| -rw-r--r-- | embed_config.c | 140 | ||||
| -rw-r--r-- | embed_cuda.cu | 72 | ||||
| -rw-r--r-- | embed_internal.h | 3 | ||||
| -rw-r--r-- | embed_mlx.c | 48 | ||||
| -rw-r--r-- | embed_server.c | 79 | ||||
| -rw-r--r-- | tests/test_qwen2.c | 97 | ||||
| -rw-r--r-- | tests/test_server.c | 87 | ||||
| -rw-r--r-- | tests/tiny_model.h | 85 |
11 files changed, 574 insertions, 85 deletions
@@ -169,6 +169,8 @@ TEST_BF16_SRCS = tests/test_bf16_model.c embed.c embed_config.c qwen_safe $(KERNEL_SRCS) TEST_QWEN3_SRCS = tests/test_qwen3.c embed.c embed_config.c qwen_safetensors.c \ $(KERNEL_SRCS) +TEST_QWEN2_SRCS = tests/test_qwen2.c embed.c embed_config.c qwen_safetensors.c \ + $(KERNEL_SRCS) TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c embed_config.c qwen_tokenizer.c \ qwen_safetensors.c $(KERNEL_SRCS) TEST_LATE_SRCS = tests/test_late.c embed.c embed_config.c qwen_tokenizer.c \ @@ -198,6 +200,9 @@ test: $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \ $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) ./tests/test_qwen3 + $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \ + $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + ./tests/test_qwen2 $(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \ $(TEST_WORKSPACE_SRCS) -lm -lpthread ./tests/test_workspace @@ -236,7 +241,8 @@ COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \ -fcoverage-compilation-dir=. COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \ tests/test_tokenizer tests/test_safetensors \ - tests/test_bf16_model tests/test_qwen3 tests/test_workspace tests/test_late \ + tests/test_bf16_model tests/test_qwen3 tests/test_qwen2 tests/test_workspace \ + tests/test_late \ tests/cli_under_test tests/test_server # Report on project sources only - not the harnesses, vendored deps, or # system headers (cJSON lands under /opt or /usr otherwise). @@ -269,6 +275,9 @@ coverage: $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \ $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) LLVM_PROFILE_FILE=$(COV_DIR)/qwen3.profraw ./tests/test_qwen3 + $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \ + $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + LLVM_PROFILE_FILE=$(COV_DIR)/qwen2.profraw ./tests/test_qwen2 $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \ $(TEST_WORKSPACE_SRCS) -lm -lpthread LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace @@ -397,7 +406,7 @@ clean: $(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \ tests/test_kernels_generic tests/test_kernels_blas \ tests/test_safetensors tests/test_bf16_model tests/test_server \ - tests/test_qwen3 \ + tests/test_qwen3 tests/test_qwen2 \ tests/test_tokenizer tests/test_workspace tests/test_cli tests/test_late \ tests/cli_under_test bench/bench_tokenizer bench/bench_kernels bench/bench_model rm -rf $(COV_DIR) @@ -347,6 +347,16 @@ static void linear_qkv_weight(embed_workspace_t *ws, linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim); } +static void add_bias_rows(float *x, const float *bias, int rows, int dim) { + if (!bias) + return; + for (int row = 0; row < rows; row++) { + float *dst = x + (size_t)row * dim; + for (int d = 0; d < dim; d++) + dst[d] += bias[d]; + } +} + static void linear_pair_weight(embed_workspace_t *ws, float *a, float *b, @@ -576,8 +586,15 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a LOAD2(wk, "%slayers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size); LOAD2(wv, "%slayers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size); LOAD2(wo, "%slayers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim); - LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim); - LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim); + if (cfg->qkv_bias) { + LOAD_NORM(q_bias, "%slayers.%d.self_attn.q_proj.bias", cfg->q_dim); + LOAD_NORM(k_bias, "%slayers.%d.self_attn.k_proj.bias", cfg->kv_dim); + LOAD_NORM(v_bias, "%slayers.%d.self_attn.v_proj.bias", cfg->kv_dim); + } + if (cfg->qk_norm) { + LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim); + LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim); + } LOAD_NORM(input_norm, "%slayers.%d.input_layernorm.weight", cfg->hidden_size); LOAD_NORM(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", cfg->hidden_size); LOAD2(gate_proj, "%slayers.%d.mlp.gate_proj.weight", cfg->intermediate_size, @@ -625,6 +642,9 @@ void embed_model_free(embed_model_t *model) { if (model->weights.layers) { for (int i = 0; i < model->config.n_layers; i++) { embed_layer_t *l = &model->weights.layers[i]; + free((void *)l->q_bias); + free((void *)l->k_bias); + free((void *)l->v_bias); free((void *)l->q_norm); free((void *)l->k_norm); free((void *)l->input_norm); @@ -1089,8 +1109,13 @@ static int forward_packed_slice_inplace(const embed_model_t *model, linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, hidden, q_dim, kv_dim); - qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); - qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); + add_bias_rows(q_buf, l->q_bias, total_seq, q_dim); + add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim); + add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim); + if (cfg->qk_norm) { + qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); + qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); + } for (int b = 0; b < batch; b++) { int start = offsets[b]; @@ -54,6 +54,8 @@ typedef struct { int vocab_size; float rms_norm_eps; float rope_theta; + int qk_norm; + int qkv_bias; embed_attention_mode_t attention_mode; embed_pooling_mode_t pooling_mode; int normalize_embeddings; diff --git a/embed_config.c b/embed_config.c index 5bbfe48..4a17740 100644 --- a/embed_config.c +++ b/embed_config.c @@ -1,11 +1,20 @@ #include "embed_config.h" +#include <ctype.h> +#include <errno.h> #include <limits.h> #include <math.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +typedef enum { + JSON_BOOL_MISSING = 0, + JSON_BOOL_FALSE = 1, + JSON_BOOL_TRUE = 2, + JSON_BOOL_INVALID = -1, +} json_bool_t; + static const char *skip_ws(const char *p) { while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; @@ -48,6 +57,112 @@ static int json_string_equals(const char *json, const char *key, const char *exp return strncmp(v, expected, len) == 0 && v[len] == '"'; } +static json_bool_t json_get_bool(const char *json, const char *key) { + const char *v = json_find_key(json, key); + if (!v) + return JSON_BOOL_MISSING; + if (strncmp(v, "true", 4) == 0 && !isalnum((unsigned char)v[4]) && v[4] != '_') + return JSON_BOOL_TRUE; + if (strncmp(v, "false", 5) == 0 && !isalnum((unsigned char)v[5]) && v[5] != '_') + return JSON_BOOL_FALSE; + return JSON_BOOL_INVALID; +} + +/* Return 1 with a NUL-terminated buffer, 0 when an optional file is absent, + * and -1 for an existing file that could not be read. */ +static int read_optional_file(const char *model_dir, const char *relative, char **out) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "embed_config: model path too long\n"); + return -1; + } + + FILE *f = fopen(path, "rb"); + if (!f && errno == ENOENT) + return 0; + if (!f) { + fprintf(stderr, "embed_config: cannot open %s\n", path); + return -1; + } + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + fclose(f); + free(buf); + return -1; + } + fclose(f); + buf[sz] = '\0'; + *out = buf; + return 1; +} + +static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char *model_dir) { + char *buf = NULL; + int rc = read_optional_file(model_dir, "1_Pooling/config.json", &buf); + if (rc < 0) + return -1; + if (rc > 0) { + json_bool_t mean = json_get_bool(buf, "pooling_mode_mean_tokens"); + json_bool_t last = json_get_bool(buf, "pooling_mode_lasttoken"); + json_bool_t cls = json_get_bool(buf, "pooling_mode_cls_token"); + json_bool_t max = json_get_bool(buf, "pooling_mode_max_tokens"); + json_bool_t sqrt_len = json_get_bool(buf, "pooling_mode_mean_sqrt_len_tokens"); + json_bool_t weighted = json_get_bool(buf, "pooling_mode_weightedmean_tokens"); + if (mean < 0 || last < 0 || cls < 0 || max < 0 || sqrt_len < 0 || weighted < 0) { + fprintf(stderr, "embed_config: invalid pooling metadata\n"); + free(buf); + return -1; + } + int supported = (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE); + int unsupported = cls == JSON_BOOL_TRUE || max == JSON_BOOL_TRUE || + sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE; + if (supported != 1 || unsupported) { + fprintf(stderr, "embed_config: unsupported Sentence Transformers pooling mode\n"); + free(buf); + return -1; + } + cfg->pooling_mode = last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN : EMBED_POOL_MEAN; + free(buf); + buf = NULL; + } + + rc = read_optional_file(model_dir, "modules.json", &buf); + if (rc < 0) + return -1; + if (rc > 0) { + cfg->normalize_embeddings = + strstr(buf, "\"sentence_transformers.models.Normalize\"") != NULL; + free(buf); + buf = NULL; + } + + rc = read_optional_file(model_dir, "tokenizer_config.json", &buf); + if (rc < 0) + return -1; + if (rc > 0) { + json_bool_t add_eos = json_get_bool(buf, "add_eos_token"); + if (add_eos < 0) { + fprintf(stderr, "embed_config: invalid add_eos_token metadata\n"); + free(buf); + return -1; + } + if (add_eos != JSON_BOOL_MISSING) + cfg->append_terminal_token = add_eos == JSON_BOOL_TRUE; + free(buf); + } + return 0; +} + int embed_config_parse(embed_config_t *cfg, const char *model_dir) { char path[1024]; int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); @@ -89,6 +204,8 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE); cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); + cfg->qk_norm = 1; + cfg->qkv_bias = 0; cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; cfg->pooling_mode = EMBED_POOL_MEAN; cfg->normalize_embeddings = 0; @@ -99,6 +216,21 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->pooling_mode = EMBED_POOL_LAST_TOKEN; cfg->normalize_embeddings = 1; cfg->append_terminal_token = 1; + } else if (json_string_equals(buf, "model_type", "qwen2")) { + cfg->qk_norm = 0; + cfg->qkv_bias = 1; + cfg->attention_mode = EMBED_ATTENTION_CAUSAL; + } + + json_bool_t is_causal = json_get_bool(buf, "is_causal"); + if (is_causal < 0) { + fprintf(stderr, "embed_config: invalid is_causal in %s\n", path); + free(buf); + return -1; + } + if (is_causal != JSON_BOOL_MISSING) { + cfg->attention_mode = + is_causal == JSON_BOOL_TRUE ? EMBED_ATTENTION_CAUSAL : EMBED_ATTENTION_BIDIRECTIONAL; } cfg->q_dim = 0; @@ -110,6 +242,9 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { free(buf); + if (parse_sentence_transformers_metadata(cfg, model_dir) != 0) + return -1; + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || @@ -131,11 +266,12 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { if (embed_verbose >= 1) fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, " - "inter=%d, head_dim=%d, attention=%s, pooling=%s\n", + "inter=%d, head_dim=%d, attention=%s, pooling=%s, qk_norm=%s, qkv_bias=%s\n", cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->intermediate_size, cfg->head_dim, cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional", - cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean"); + cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean", + cfg->qk_norm ? "yes" : "no", cfg->qkv_bias ? "yes" : "no"); return 0; } diff --git a/embed_cuda.cu b/embed_cuda.cu index ba78f09..3100f44 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -23,6 +23,7 @@ typedef struct { typedef struct { cuda_matrix_t qkv, wo; + float *qkv_bias; float *q_norm; float *k_norm; float *input_norm; @@ -303,6 +304,26 @@ static int load_vector(float **out, const float *src, int n) { return 0; } +static int load_qkv_bias(float **out, const embed_layer_t *src, const embed_config_t *c) { + int n = c->q_dim + 2 * c->kv_dim; + float *host = (float *)malloc((size_t)n * sizeof(float)); + if (!host) + return -1; + memcpy(host, src->q_bias, (size_t)c->q_dim * sizeof(float)); + memcpy(host + c->q_dim, src->k_bias, (size_t)c->kv_dim * sizeof(float)); + memcpy(host + c->q_dim + c->kv_dim, src->v_bias, (size_t)c->kv_dim * sizeof(float)); + int rc = load_vector(out, host, n); + free(host); + return rc; +} + +__global__ static void add_row_bias_kernel(float *x, const float *bias, int rows, int dim) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int count = rows * dim; + if (idx < count) + x[idx] += bias[idx % dim]; +} + __global__ static void embed_lookup_kernel( float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) { int idx = blockIdx.x * blockDim.x + threadIdx.x; @@ -391,6 +412,7 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, int row_stride, int q_offset, int k_offset, + int qk_norm, float eps) { __shared__ float partial[4]; __shared__ float total; @@ -409,29 +431,34 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, size_t base = (size_t)row * row_stride + base_offset + head * head_dim; float v = tid < head_dim ? x[base + tid] : 0.0f; float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f; - float sum = v * v; - - for (int offset = 16; offset > 0; offset >>= 1) - sum += __shfl_down_sync(0xffffffff, sum, offset); - if (lane == 0) - partial[warp] = sum; - __syncthreads(); - if (warp == 0) { - sum = lane < 4 ? partial[lane] : 0.0f; + if (qk_norm) { + float sum = v * v; for (int offset = 16; offset > 0; offset >>= 1) sum += __shfl_down_sync(0xffffffff, sum, offset); if (lane == 0) - total = sum; + partial[warp] = sum; + __syncthreads(); + if (warp == 0) { + sum = lane < 4 ? partial[lane] : 0.0f; + for (int offset = 16; offset > 0; offset >>= 1) + sum += __shfl_down_sync(0xffffffff, sum, offset); + if (lane == 0) + total = sum; + } + __syncthreads(); } - __syncthreads(); if (tid < head_dim) { - float inv = rsqrtf(total / (float)head_dim + eps); int pos = positions[row]; float c = cosv[(size_t)pos * head_dim + tid]; float s = sinv[(size_t)pos * head_dim + tid]; - float a = v * inv * weight[tid]; - float b = pair_v * inv * weight[pair_d]; + float a = v; + float b = pair_v; + if (qk_norm) { + float inv = rsqrtf(total / (float)head_dim + eps); + a *= inv * weight[tid]; + b *= inv * weight[pair_d]; + } x[base + tid] = a * c + sign * b * s; } } @@ -1081,8 +1108,9 @@ static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) { static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) { return load_qkv_matrix(&dst->qkv, src, c) || load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || - load_vector(&dst->q_norm, src->q_norm, c->head_dim) || - load_vector(&dst->k_norm, src->k_norm, c->head_dim) || + (c->qkv_bias && load_qkv_bias(&dst->qkv_bias, src, c)) || + (c->qk_norm && load_vector(&dst->q_norm, src->q_norm, c->head_dim)) || + (c->qk_norm && load_vector(&dst->k_norm, src->k_norm, c->head_dim)) || load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || load_gate_up_matrix(&dst->gate_up_proj, src, c) || @@ -1092,6 +1120,7 @@ static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_c static void free_layer(cuda_layer_t *l) { cuda_matrix_free(&l->qkv); cuda_matrix_free(&l->wo); + cudaFree(l->qkv_bias); cudaFree(l->q_norm); cudaFree(l->k_norm); cudaFree(l->input_norm); @@ -1303,9 +1332,18 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs return -1; } + if (c->qkv_bias) { + int count = total * qkv_dim; + add_row_bias_kernel<<<(count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias, + total, qkv_dim); + if (launch_check() != 0) + return -1; + } + rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>( ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total, - c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->rms_norm_eps); + c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->qk_norm, + c->rms_norm_eps); if (launch_check() != 0) return -1; diff --git a/embed_internal.h b/embed_internal.h index 7d6977c..0536692 100644 --- a/embed_internal.h +++ b/embed_internal.h @@ -14,6 +14,9 @@ typedef struct { embed_weight_ref_t wk; /* [kv_dim, hidden] */ embed_weight_ref_t wv; /* [kv_dim, hidden] */ embed_weight_ref_t wo; /* [hidden, q_dim] */ + const float *q_bias; /* [q_dim], optional */ + const float *k_bias; /* [kv_dim], optional */ + const float *v_bias; /* [kv_dim], optional */ const float *q_norm; /* [head_dim] */ const float *k_norm; /* [head_dim] */ const float *input_norm; /* [hidden] */ diff --git a/embed_mlx.c b/embed_mlx.c index de82f83..d85dc0d 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -28,6 +28,7 @@ typedef struct { typedef struct { mlx_linear_t wq, wk, wv, wo; + mlx_array q_bias, k_bias, v_bias; mlx_array q_norm, k_norm; mlx_array input_norm, post_attn_norm; mlx_linear_t gate_proj, up_proj, down_proj; @@ -469,8 +470,15 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, goto fail; \ } while (0) - LD(q_norm, "%slayers.%d.self_attn.q_norm.weight", hd); - LD(k_norm, "%slayers.%d.self_attn.k_norm.weight", hd); + if (c->qkv_bias) { + LD(q_bias, "%slayers.%d.self_attn.q_proj.bias", qd); + LD(k_bias, "%slayers.%d.self_attn.k_proj.bias", kvd); + LD(v_bias, "%slayers.%d.self_attn.v_proj.bias", kvd); + } + if (c->qk_norm) { + LD(q_norm, "%slayers.%d.self_attn.q_norm.weight", hd); + LD(k_norm, "%slayers.%d.self_attn.k_norm.weight", hd); + } LD(input_norm, "%slayers.%d.input_layernorm.weight", h); LD(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", h); #undef LD @@ -509,6 +517,9 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) { for (int i = layer_start; i < layer_end; i++) { mlx_layer_t *l = &ctx->layers[i]; + upcast_f32_inplace(&l->q_bias, ctx->stream); + upcast_f32_inplace(&l->k_bias, ctx->stream); + upcast_f32_inplace(&l->v_bias, ctx->stream); upcast_f32_inplace(&l->q_norm, ctx->stream); upcast_f32_inplace(&l->k_norm, ctx->stream); upcast_f32_inplace(&l->input_norm, ctx->stream); @@ -576,6 +587,9 @@ static void free_mlx_layer(mlx_layer_t *l) { free_linear(&l->wk); free_linear(&l->wv); free_linear(&l->wo); + mlx_array_free(l->q_bias); + mlx_array_free(l->k_bias); + mlx_array_free(l->v_bias); mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); mlx_array_free(l->input_norm); @@ -733,6 +747,20 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array k_flat = linear(xn, &l->wk, S); mlx_array v_flat = linear(xn, &l->wv, S); mlx_array_free(xn); + if (c->qkv_bias) { + mlx_array biased = mlx_array_new(); + mlx_add(&biased, q_flat, l->q_bias, S); + mlx_array_free(q_flat); + q_flat = biased; + biased = mlx_array_new(); + mlx_add(&biased, k_flat, l->k_bias, S); + mlx_array_free(k_flat); + k_flat = biased; + biased = mlx_array_new(); + mlx_add(&biased, v_flat, l->v_bias, S); + mlx_array_free(v_flat); + v_flat = biased; + } int q_shape[] = {batch, max_seq, n_heads, head_dim}; int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; @@ -746,12 +774,16 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array_free(k_flat); mlx_array_free(v_flat); - mlx_array qn = mlx_array_new(); - mlx_array kn = mlx_array_new(); - mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); - mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); - mlx_array_free(q); - mlx_array_free(k); + mlx_array qn = q; + mlx_array kn = k; + if (c->qk_norm) { + qn = mlx_array_new(); + kn = mlx_array_new(); + mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); + mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); + mlx_array_free(q); + mlx_array_free(k); + } /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ int perm[] = {0, 2, 1, 3}; diff --git a/embed_server.c b/embed_server.c index cf3a9ec..b62bcfb 100644 --- a/embed_server.c +++ b/embed_server.c @@ -66,13 +66,14 @@ #define EMBED_LATE_MASK_TOKEN_ID 151642 #define EMBED_LATE_QUERY_PREFIX_ID 151669 #define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 -/* Qwen3/DashScope `text_type: query` prepends this default retrieval instruction - * in the Qwen3-Embedding format ("Instruct: {task}\nQuery:{query}"); documents - * and the default pass through unchanged. The task text matches the - * Qwen3-Embedding reference default; callers needing a custom instruction still - * prepend their own text and leave text_type at "document". */ +/* `text_type: query` prepends the model's published retrieval instruction. + * Documents and the default pass through unchanged. Callers needing a custom + * instruction prepend it themselves and leave text_type at "document". */ #define EMBED_QWEN3_QUERY_INSTRUCT \ "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:" +#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \ + "Instruct: Given a web search query, retrieve relevant passages that answer the " \ + "query\nQuery: " /* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent * short-request throughput over 8 with no long-document penalty, while 128 * was no faster and inflated long-document tail latency. */ @@ -227,6 +228,7 @@ typedef enum { MODEL_QWEN3_06, MODEL_QWEN3_4, MODEL_QWEN3_8, + MODEL_GTE_QWEN2_15, MODEL_CTX_06, MODEL_CTX_4, MODEL_LATE_06, @@ -239,10 +241,10 @@ typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } mod /* Which embedding API a model speaks. pplx-embed models follow the Perplexity * API: the canonical output is the tanh int8 quantization, so the default * encoding is base64_int8 and "float" is the int8-decoded view (int8/128). - * Qwen3-Embedding models follow the Qwen/DashScope API, which is OpenAI- - * compatible: "float" is the true float32 vector and "base64" is base64 of - * little-endian float32. int8 quantization underuses the range of Qwen3's - * L2-normalized vectors, so it is never the Qwen3 wire format. */ + * Instruction embedding models follow the OpenAI-compatible API: "float" is + * the true float32 vector and "base64" is base64 of little-endian float32. + * int8 quantization underuses the range of L2-normalized vectors, so it is not + * used for these models. */ typedef enum { EMBED_API_PERPLEXITY = 0, EMBED_API_OPENAI = 1, @@ -258,25 +260,28 @@ typedef struct { embed_pooling_mode_t pooling_mode; int normalize_embeddings; embedding_api_t api; + const char *query_instruct; } model_info; static const model_info k_models[] = { {"pplx-embed-v1-0.6b", MODEL_KIND_STANDARD, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"pplx-embed-v1-4b", MODEL_KIND_STANDARD, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"Qwen3-Embedding-0.6B", MODEL_KIND_STANDARD, 1024, 32, 0, EMBED_ATTENTION_CAUSAL, - EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, {"Qwen3-Embedding-4B", MODEL_KIND_STANDARD, 2560, 32, 0, EMBED_ATTENTION_CAUSAL, - EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, {"Qwen3-Embedding-8B", MODEL_KIND_STANDARD, 4096, 32, 0, EMBED_ATTENTION_CAUSAL, - EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI}, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_QWEN3_QUERY_INSTRUCT}, + {"gte-Qwen2-1.5B-instruct", MODEL_KIND_STANDARD, 1536, 1536, 0, EMBED_ATTENTION_BIDIRECTIONAL, + EMBED_POOL_LAST_TOKEN, 1, EMBED_API_OPENAI, EMBED_GTE_QWEN2_QUERY_INSTRUCT}, {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, - EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"pplx-embed-v1-late-0.6b", MODEL_KIND_LATE, 1024, 128, 128, EMBED_ATTENTION_BIDIRECTIONAL, - EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY}, + EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, }; static model_slot model_slot_for_id(const char *id) { @@ -1242,18 +1247,16 @@ static const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_ return v; } -/* Parse the Qwen3/DashScope `text_type` hint. Returns 1 when the query - * retrieval instruction should be prepended, 0 otherwise (document / absent). - * The field belongs to the OpenAI/DashScope (Qwen3) family only; on a Perplexity - * model it is rejected, since pplx-embed has no instruction concept. Invalid - * values are reported into detail and treated as the no-op default. */ -static int text_type_is_query(cJSON *root, cJSON *detail, embedding_api_t api) { +/* Parse the `text_type` hint. Returns 1 when the model's query instruction + * should be prepended, 0 otherwise (document / absent). Models without a + * published query instruction reject the field. */ +static int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) { cJSON *tt = cJSON_GetObjectItemCaseSensitive(root, "text_type"); if (!tt) return 0; - if (api != EMBED_API_OPENAI) { + if (!query_instruct) { ve_add(detail, "[\"body\",\"text_type\"]", - "text_type is only supported for Qwen3-Embedding models", "extra_fields"); + "text_type is only supported for instruction embedding models", "extra_fields"); return 0; } if (!cJSON_IsString(tt) || !tt->valuestring) { @@ -1497,22 +1500,20 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *ou return 0; } -/* Tokenize one embedding input, prepending the Qwen3 query instruction when - * query_instruct is set. The instruction is part of the text the model sees, so - * it is tokenized in one pass with the input (and the terminal token, if any, is - * appended by tokenize_one as usual). */ -static int -tokenize_input(loaded_model *m, job *j, const char *text, int query_instruct, token_buf *out) { +/* Tokenize one embedding input, optionally prepending the model's query + * instruction. The instruction and input are tokenized in one pass. */ +static int tokenize_input( + loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) { if (!query_instruct) return tokenize_one(m, j, text, out); - size_t plen = strlen(EMBED_QWEN3_QUERY_INSTRUCT); + size_t plen = strlen(query_instruct); size_t tlen = strlen(text); char *buf = (char *)malloc(plen + tlen + 1); if (!buf) { memset(out, 0, sizeof(*out)); return -1; } - memcpy(buf, EMBED_QWEN3_QUERY_INSTRUCT, plen); + memcpy(buf, query_instruct, plen); memcpy(buf + plen, text, tlen + 1); int rc = tokenize_one(m, j, buf, out); free(buf); @@ -2003,15 +2004,17 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embed out->dims = dims; out->encoding = encoding; - /* Resolve the API family from the requested model (independent of load - * state) to validate the Qwen3/DashScope text_type hint. query_instruct is - * applied to every input in the request. */ + /* Resolve request behavior from the model registry independent of load + * state. The query instruction, when requested, applies to every input. */ cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring) ? model_slot_for_id(model_item->valuestring) : MODEL_UNKNOWN; - embedding_api_t api = api_slot != MODEL_UNKNOWN ? k_models[api_slot].api : EMBED_API_PERPLEXITY; - int query_instruct = text_type_is_query(root, detail, api); + const char *query_instruct = + api_slot != MODEL_UNKNOWN && + text_type_is_query(root, detail, k_models[api_slot].query_instruct) + ? k_models[api_slot].query_instruct + : NULL; cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); int n_inputs = 0; diff --git a/tests/test_qwen2.c b/tests/test_qwen2.c new file mode 100644 index 0000000..4795f7f --- /dev/null +++ b/tests/test_qwen2.c @@ -0,0 +1,97 @@ +/* tests/test_qwen2.c - hermetic GTE-Qwen2 block and task semantics. */ + +#include "embed.h" +#include "tiny_model.h" + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +static float max_abs_diff(const float *a, const float *b, int n) { + float out = 0.0f; + for (int i = 0; i < n; i++) { + float d = fabsf(a[i] - b[i]); + if (d > out) + out = d; + } + return out; +} + +int main(void) { + const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"; + char dir[1024], zero_dir[1024]; + snprintf(dir, sizeof(dir), "%s/embed-qwen2-test-XXXXXX", tmp); + snprintf(zero_dir, sizeof(zero_dir), "%s/embed-qwen2-zero-bias-test-XXXXXX", tmp); + tm_dims_t dims = {4, 2, 1, 2, 8, 16}; + if (!mkdtemp(dir) || !mkdtemp(zero_dir) || + tm_write_qwen2_model_dims(dir, "F32", &dims, 15, 0) != 0 || + tm_write_qwen2_model_dims(zero_dir, "F32", &dims, 15, 1) != 0) { + fprintf(stderr, "fixture creation failed\n"); + return 2; + } + + embed_model_t *model = embed_model_load(dir); + embed_model_t *zero_model = embed_model_load(zero_dir); + embed_workspace_t *ws = embed_workspace_new(model); + embed_workspace_t *zero_ws = embed_workspace_new(zero_model); + const embed_config_t *cfg = embed_model_config(model); + if (!model || !zero_model || !ws || !zero_ws || !cfg) { + fprintf(stderr, "model setup failed\n"); + return 1; + } + if (cfg->qk_norm || !cfg->qkv_bias || cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL || + cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || !cfg->normalize_embeddings || + !cfg->append_terminal_token) { + fprintf(stderr, "Qwen2 config or Sentence Transformers metadata was not detected\n"); + return 1; + } + + int ids[] = {1, 2, 15}; + float states[3 * 4], embedding[4], pooled[4], zero_embedding[4]; + if (embed_model_forward_into(model, ws, ids, 3, states) != 0 || + embed_model_encode_into(model, ws, ids, 3, embedding) != 0 || + embed_model_encode_into(zero_model, zero_ws, ids, 3, zero_embedding) != 0 || + embed_pool_batch(cfg, states, (int[]){3}, 1, pooled) != 0) { + fprintf(stderr, "Qwen2 inference failed\n"); + return 1; + } + + float expected[4]; + memcpy(expected, states + 2 * 4, sizeof(expected)); + if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f || + max_abs_diff(pooled, expected, 4) > 1e-5f) { + fprintf(stderr, "last-token pooling mismatch\n"); + return 1; + } + if (max_abs_diff(embedding, zero_embedding, 4) < 1e-4f) { + fprintf(stderr, "QKV projection bias did not affect the embedding\n"); + return 1; + } + + int changed_ids[] = {1, 3, 15}; + float changed_states[3 * 4]; + if (embed_model_forward_into(model, ws, changed_ids, 3, changed_states) != 0 || + max_abs_diff(states, changed_states, 4) < 1e-6f) { + fprintf(stderr, "bidirectional attention did not affect the first token\n"); + return 1; + } + + int short_ids[] = {4, 15}; + embed_input_t inputs[] = {{ids, 3}, {short_ids, 2}}; + float batch[8], single[4]; + if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 || + embed_model_encode_into(model, ws, short_ids, 2, single) != 0 || + max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) { + fprintf(stderr, "Qwen2 packed batch parity failed\n"); + return 1; + } + + embed_workspace_free(zero_ws); + embed_workspace_free(ws); + embed_model_free(zero_model); + embed_model_free(model); + puts("ok: Qwen2 bias, bidirectional attention, and GTE pooling semantics"); + return 0; +} diff --git a/tests/test_server.c b/tests/test_server.c index ce9668e..9ebb8f2 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -146,41 +146,54 @@ static void test_encoding_from_root_family(void) { static void test_text_type(void) { cJSON *detail = cJSON_CreateArray(); - /* Absent: no-op for both families, no error. */ + /* Absent: no-op for both model types, no error. */ cJSON *empty = cJSON_CreateObject(); - TEST_ASSERT(text_type_is_query(empty, detail, EMBED_API_OPENAI) == 0); - TEST_ASSERT(text_type_is_query(empty, detail, EMBED_API_PERPLEXITY) == 0); + TEST_ASSERT(text_type_is_query(empty, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0); + TEST_ASSERT(text_type_is_query(empty, detail, NULL) == 0); cJSON_Delete(empty); TEST_ASSERT(cJSON_GetArraySize(detail) == 0); /* Qwen3: query -> 1, document -> 0, both accepted. */ cJSON *q = cJSON_CreateObject(); cJSON_AddStringToObject(q, "text_type", "query"); - TEST_ASSERT(text_type_is_query(q, detail, EMBED_API_OPENAI) == 1); + TEST_ASSERT(text_type_is_query(q, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 1); cJSON_Delete(q); cJSON *d = cJSON_CreateObject(); cJSON_AddStringToObject(d, "text_type", "document"); - TEST_ASSERT(text_type_is_query(d, detail, EMBED_API_OPENAI) == 0); + TEST_ASSERT(text_type_is_query(d, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0); cJSON_Delete(d); TEST_ASSERT(cJSON_GetArraySize(detail) == 0); /* Invalid enum value on Qwen3 is reported. */ cJSON *bad = cJSON_CreateObject(); cJSON_AddStringToObject(bad, "text_type", "passage"); - TEST_ASSERT(text_type_is_query(bad, detail, EMBED_API_OPENAI) == 0); + TEST_ASSERT(text_type_is_query(bad, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0); TEST_ASSERT(cJSON_GetArraySize(detail) == 1); cJSON_Delete(bad); /* text_type on a Perplexity-family model is rejected. */ cJSON *p = cJSON_CreateObject(); cJSON_AddStringToObject(p, "text_type", "query"); - TEST_ASSERT(text_type_is_query(p, detail, EMBED_API_PERPLEXITY) == 0); + TEST_ASSERT(text_type_is_query(p, detail, NULL) == 0); TEST_ASSERT(cJSON_GetArraySize(detail) == 2); cJSON_Delete(p); cJSON_Delete(detail); } +static void test_model_registry(void) { + model_slot slot = model_slot_for_id("gte-Qwen2-1.5B-instruct"); + TEST_ASSERT(slot == MODEL_GTE_QWEN2_15); + if (slot == MODEL_UNKNOWN) + return; + TEST_ASSERT(k_models[slot].dim == 1536); + TEST_ASSERT(k_models[slot].min_dim == 1536); + TEST_ASSERT(k_models[slot].attention_mode == EMBED_ATTENTION_BIDIRECTIONAL); + TEST_ASSERT(k_models[slot].pooling_mode == EMBED_POOL_LAST_TOKEN); + TEST_ASSERT(k_models[slot].normalize_embeddings == 1); + TEST_ASSERT(strcmp(k_models[slot].query_instruct, EMBED_GTE_QWEN2_QUERY_INSTRUCT) == 0); +} + static void test_collect_job_batch_deadline(void) { http_server s; memset(&s, 0, sizeof(s)); @@ -410,7 +423,7 @@ static int http_req(int port, } typedef struct { - embed_server_model_spec_t spec[4]; + embed_server_model_spec_t spec[5]; embed_server_config_t cfg; int rc; } srv_ctx; @@ -657,6 +670,45 @@ static void test_http_embeddings(int port) { cJSON_Delete(tt_query); cJSON_Delete(tt_manual); + /* GTE-Qwen2 uses the same retrieval task text but publishes a space after + * "Query:". Verify the registry-specific prompt through the full path. */ + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"gte-Qwen2-1.5B-instruct\",\"input\":\"hello\"," + "\"text_type\":\"query\",\"encoding_format\":\"float\"}", + NULL, &body) == 200); + cJSON *gte_query = body ? parse_embeddings_body(body, 1) : NULL; + free(body); + body = NULL; + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"gte-Qwen2-1.5B-instruct\",\"input\":\"Instruct: Given a " + "web search query, retrieve relevant passages that answer the " + "query\\nQuery: hello\",\"text_type\":\"document\"," + "\"encoding_format\":\"float\"}", + NULL, &body) == 200); + cJSON *gte_manual = body ? parse_embeddings_body(body, 1) : NULL; + free(body); + body = NULL; + if (gte_query && gte_manual) { + cJSON *qe = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(gte_query, "data"), 0), + "embedding"); + cJSON *me = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(gte_manual, "data"), 0), + "embedding"); + TEST_ASSERT(cJSON_IsArray(qe) && cJSON_IsArray(me)); + TEST_ASSERT(cJSON_GetArraySize(qe) == 1536 && cJSON_GetArraySize(me) == 1536); + double gte_diff = 0.0; + for (int i = 0; i < cJSON_GetArraySize(qe); i++) { + double d = fabs(cJSON_GetArrayItem(qe, i)->valuedouble - + cJSON_GetArrayItem(me, i)->valuedouble); + if (d > gte_diff) + gte_diff = d; + } + TEST_ASSERT(gte_diff < 1e-6); + } + cJSON_Delete(gte_query); + cJSON_Delete(gte_manual); + /* text_type is rejected on a Perplexity-family model, and bad values 422. */ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello\"," @@ -897,7 +949,7 @@ static void test_http_rerank(int port) { } static void test_http_server(void) { - char dir[1024], late_dir[1024], qwen_dir[1024]; + char dir[1024], late_dir[1024], qwen_dir[1024], gte_dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); if (!mkdtemp(dir)) { @@ -919,14 +971,24 @@ static void test_http_server(void) { test_failures++; return; } + snprintf(gte_dir, sizeof(gte_dir), "%s/embed-srv-gte-test-XXXXXX", + getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + if (!mkdtemp(gte_dir)) { + fprintf(stderr, "FAIL: GTE mkdtemp\n"); + test_failures++; + return; + } /* The 0.6b server slot requires hidden_size 1024; everything else stays * tiny. The model vocab covers every tokenizer fixture id. */ tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE}; + tm_dims_t gte_dims = {1536, 2, 1, 64, 8, TF_VOCAB_SIZE}; if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 || tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 || tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 || tf_write_vocab(qwen_dir) != 0 || - tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0) { + tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0 || + tf_write_vocab(gte_dir) != 0 || + tm_write_qwen2_model_dims(gte_dir, "F32", >e_dims, TF_EOT_ID, 0) != 0) { fprintf(stderr, "FAIL: fixture write\n"); test_failures++; return; @@ -947,8 +1009,10 @@ static void test_http_server(void) { ctx.spec[2].path = late_dir; ctx.spec[3].id = "Qwen3-Embedding-0.6B"; ctx.spec[3].path = qwen_dir; + ctx.spec[4].id = "gte-Qwen2-1.5B-instruct"; + ctx.spec[4].path = gte_dir; ctx.cfg.models = ctx.spec; - ctx.cfg.n_models = 4; + ctx.cfg.n_models = 5; ctx.cfg.port = port; ctx.cfg.batch_size = 2; ctx.cfg.batch_wait_us = 1000; @@ -992,6 +1056,7 @@ int main(void) { test_append_embedding_value_float(); test_encoding_from_root_family(); test_text_type(); + test_model_registry(); test_collect_job_batch_deadline(); test_collect_job_batch_zero_wait(); test_http_server(); diff --git a/tests/tiny_model.h b/tests/tiny_model.h index d2534d2..95496b7 100644 --- a/tests/tiny_model.h +++ b/tests/tiny_model.h @@ -76,8 +76,12 @@ tm_write_config(const char *dir, const tm_dims_t *d, const char *model_type, int /* Write a safetensors file holding `specs` with tm_value() contents. * dtype: "F32" or "BF16". Returns 0 on success. */ -static int tm_write_safetensors_with_prefix( - const char *path, const char *dtype, const tm_spec_t *specs, int n_specs, const char *prefix) { +static int tm_write_safetensors_with_prefix_options(const char *path, + const char *dtype, + const tm_spec_t *specs, + int n_specs, + const char *prefix, + int zero_bias) { int bf16 = strcmp(dtype, "BF16") == 0; size_t esize = bf16 ? 2 : 4; @@ -117,7 +121,7 @@ static int tm_write_safetensors_with_prefix( const tm_spec_t *s = &specs[t]; size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1); for (size_t i = 0; i < n; i++) { - float v = tm_value(s->name, i); + float v = zero_bias && strstr(s->name, ".bias") ? 0.0f : tm_value(s->name, i); if (bf16) { uint16_t h = tm_f32_to_bf16(v); fwrite(&h, sizeof(h), 1, f); @@ -130,6 +134,11 @@ static int tm_write_safetensors_with_prefix( return 0; } +static int tm_write_safetensors_with_prefix( + const char *path, const char *dtype, const tm_spec_t *specs, int n_specs, const char *prefix) { + return tm_write_safetensors_with_prefix_options(path, dtype, specs, n_specs, prefix, 0); +} + static int tm_write_safetensors(const char *path, const char *dtype, const tm_spec_t *specs, int n_specs) { return tm_write_safetensors_with_prefix(path, dtype, specs, n_specs, ""); @@ -184,6 +193,76 @@ static inline int tm_write_qwen3_model_dims(const char *dir, return tm_write_config(dir, d, "qwen3", eos_token_id); } +static inline int tm_write_qwen2_model_dims( + const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) { + int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim; + const tm_spec_t specs[] = { + {"embed_tokens.weight", d->vocab, d->hidden}, + {"layers.0.self_attn.q_proj.weight", q, d->hidden}, + {"layers.0.self_attn.q_proj.bias", q, 0}, + {"layers.0.self_attn.k_proj.weight", kv, d->hidden}, + {"layers.0.self_attn.k_proj.bias", kv, 0}, + {"layers.0.self_attn.v_proj.weight", kv, d->hidden}, + {"layers.0.self_attn.v_proj.bias", kv, 0}, + {"layers.0.self_attn.o_proj.weight", d->hidden, q}, + {"layers.0.input_layernorm.weight", d->hidden, 0}, + {"layers.0.post_attention_layernorm.weight", d->hidden, 0}, + {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden}, + {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden}, + {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate}, + {"norm.weight", d->hidden, 0}, + }; + char path[2048]; + snprintf(path, sizeof(path), "%s/config.json", dir); + FILE *f = fopen(path, "w"); + if (!f) + return -1; + fprintf(f, + "{\"hidden_size\":%d,\"num_hidden_layers\":1," + "\"num_attention_heads\":%d,\"num_key_value_heads\":%d," + "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d," + "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0," + "\"model_type\":\"qwen2\",\"eos_token_id\":%d,\"is_causal\":false}", + d->hidden, d->heads, d->kv_heads, d->head_dim, d->intermediate, d->vocab, eos_token_id); + fclose(f); + + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + if (tm_write_safetensors_with_prefix_options( + path, dtype, specs, (int)(sizeof(specs) / sizeof(specs[0])), "model.", zero_bias) != 0) + return -1; + + snprintf(path, sizeof(path), "%s/1_Pooling", dir); + if (mkdir(path, 0755) != 0 && errno != EEXIST) + return -1; + snprintf(path, sizeof(path), "%s/1_Pooling/config.json", dir); + f = fopen(path, "w"); + if (!f) + return -1; + fputs("{\"pooling_mode_cls_token\":false,\"pooling_mode_mean_tokens\":false," + "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false," + "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":true}", + f); + fclose(f); + + snprintf(path, sizeof(path), "%s/modules.json", dir); + f = fopen(path, "w"); + if (!f) + return -1; + fputs("[{\"type\":\"sentence_transformers.models.Transformer\"}," + "{\"type\":\"sentence_transformers.models.Pooling\"}," + "{\"type\":\"sentence_transformers.models.Normalize\"}]", + f); + fclose(f); + + snprintf(path, sizeof(path), "%s/tokenizer_config.json", dir); + f = fopen(path, "w"); + if (!f) + return -1; + fputs("{\"add_eos_token\":true}", f); + fclose(f); + return 0; +} + /* Add the 1_Dense per-token projection head that turns a base model dir into * a late-interaction snapshot. weight_name is "linear.weight" for a valid * head; pass another name to synthesize a malformed head. */ |
