aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 12:16:33 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 12:16:33 +0200
commitc6ade0a14b87053d4a19cbac121d03af48cbf515 (patch)
treefccea2d9351e310ad308a2336abf766b4884cc11 /tests
parentb5ada41fedf1ebdaff39ee0fbef76a468b26e5f7 (diff)
downloadffwd-c6ade0a14b87053d4a19cbac121d03af48cbf515.tar.gz
ffwd-c6ade0a14b87053d4a19cbac121d03af48cbf515.zip
all: support GTE-Qwen2 embeddings
Diffstat (limited to 'tests')
-rw-r--r--tests/test_qwen2.c97
-rw-r--r--tests/test_server.c87
-rw-r--r--tests/tiny_model.h85
3 files changed, 255 insertions, 14 deletions
diff --git a/tests/test_qwen2.c b/tests/test_qwen2.c
new file mode 100644
index 0000000..4795f7f
--- /dev/null
+++ b/tests/test_qwen2.c
@@ -0,0 +1,97 @@
+/* tests/test_qwen2.c - hermetic GTE-Qwen2 block and task semantics. */
+
+#include "embed.h"
+#include "tiny_model.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static float max_abs_diff(const float *a, const float *b, int n) {
+ float out = 0.0f;
+ for (int i = 0; i < n; i++) {
+ float d = fabsf(a[i] - b[i]);
+ if (d > out)
+ out = d;
+ }
+ return out;
+}
+
+int main(void) {
+ const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
+ char dir[1024], zero_dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-qwen2-test-XXXXXX", tmp);
+ snprintf(zero_dir, sizeof(zero_dir), "%s/embed-qwen2-zero-bias-test-XXXXXX", tmp);
+ tm_dims_t dims = {4, 2, 1, 2, 8, 16};
+ if (!mkdtemp(dir) || !mkdtemp(zero_dir) ||
+ tm_write_qwen2_model_dims(dir, "F32", &dims, 15, 0) != 0 ||
+ tm_write_qwen2_model_dims(zero_dir, "F32", &dims, 15, 1) != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ embed_model_t *model = embed_model_load(dir);
+ embed_model_t *zero_model = embed_model_load(zero_dir);
+ embed_workspace_t *ws = embed_workspace_new(model);
+ embed_workspace_t *zero_ws = embed_workspace_new(zero_model);
+ const embed_config_t *cfg = embed_model_config(model);
+ if (!model || !zero_model || !ws || !zero_ws || !cfg) {
+ fprintf(stderr, "model setup failed\n");
+ return 1;
+ }
+ if (cfg->qk_norm || !cfg->qkv_bias || cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL ||
+ cfg->pooling_mode != EMBED_POOL_LAST_TOKEN || !cfg->normalize_embeddings ||
+ !cfg->append_terminal_token) {
+ fprintf(stderr, "Qwen2 config or Sentence Transformers metadata was not detected\n");
+ return 1;
+ }
+
+ int ids[] = {1, 2, 15};
+ float states[3 * 4], embedding[4], pooled[4], zero_embedding[4];
+ if (embed_model_forward_into(model, ws, ids, 3, states) != 0 ||
+ embed_model_encode_into(model, ws, ids, 3, embedding) != 0 ||
+ embed_model_encode_into(zero_model, zero_ws, ids, 3, zero_embedding) != 0 ||
+ embed_pool_batch(cfg, states, (int[]){3}, 1, pooled) != 0) {
+ fprintf(stderr, "Qwen2 inference failed\n");
+ return 1;
+ }
+
+ float expected[4];
+ memcpy(expected, states + 2 * 4, sizeof(expected));
+ if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f ||
+ max_abs_diff(pooled, expected, 4) > 1e-5f) {
+ fprintf(stderr, "last-token pooling mismatch\n");
+ return 1;
+ }
+ if (max_abs_diff(embedding, zero_embedding, 4) < 1e-4f) {
+ fprintf(stderr, "QKV projection bias did not affect the embedding\n");
+ return 1;
+ }
+
+ int changed_ids[] = {1, 3, 15};
+ float changed_states[3 * 4];
+ if (embed_model_forward_into(model, ws, changed_ids, 3, changed_states) != 0 ||
+ max_abs_diff(states, changed_states, 4) < 1e-6f) {
+ fprintf(stderr, "bidirectional attention did not affect the first token\n");
+ return 1;
+ }
+
+ int short_ids[] = {4, 15};
+ embed_input_t inputs[] = {{ids, 3}, {short_ids, 2}};
+ float batch[8], single[4];
+ if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 ||
+ embed_model_encode_into(model, ws, short_ids, 2, single) != 0 ||
+ max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) {
+ fprintf(stderr, "Qwen2 packed batch parity failed\n");
+ return 1;
+ }
+
+ embed_workspace_free(zero_ws);
+ embed_workspace_free(ws);
+ embed_model_free(zero_model);
+ embed_model_free(model);
+ puts("ok: Qwen2 bias, bidirectional attention, and GTE pooling semantics");
+ return 0;
+}
diff --git a/tests/test_server.c b/tests/test_server.c
index ce9668e..9ebb8f2 100644
--- a/tests/test_server.c
+++ b/tests/test_server.c
@@ -146,41 +146,54 @@ static void test_encoding_from_root_family(void) {
static void test_text_type(void) {
cJSON *detail = cJSON_CreateArray();
- /* Absent: no-op for both families, no error. */
+ /* Absent: no-op for both model types, no error. */
cJSON *empty = cJSON_CreateObject();
- TEST_ASSERT(text_type_is_query(empty, detail, EMBED_API_OPENAI) == 0);
- TEST_ASSERT(text_type_is_query(empty, detail, EMBED_API_PERPLEXITY) == 0);
+ TEST_ASSERT(text_type_is_query(empty, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0);
+ TEST_ASSERT(text_type_is_query(empty, detail, NULL) == 0);
cJSON_Delete(empty);
TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
/* Qwen3: query -> 1, document -> 0, both accepted. */
cJSON *q = cJSON_CreateObject();
cJSON_AddStringToObject(q, "text_type", "query");
- TEST_ASSERT(text_type_is_query(q, detail, EMBED_API_OPENAI) == 1);
+ TEST_ASSERT(text_type_is_query(q, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 1);
cJSON_Delete(q);
cJSON *d = cJSON_CreateObject();
cJSON_AddStringToObject(d, "text_type", "document");
- TEST_ASSERT(text_type_is_query(d, detail, EMBED_API_OPENAI) == 0);
+ TEST_ASSERT(text_type_is_query(d, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0);
cJSON_Delete(d);
TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
/* Invalid enum value on Qwen3 is reported. */
cJSON *bad = cJSON_CreateObject();
cJSON_AddStringToObject(bad, "text_type", "passage");
- TEST_ASSERT(text_type_is_query(bad, detail, EMBED_API_OPENAI) == 0);
+ TEST_ASSERT(text_type_is_query(bad, detail, EMBED_QWEN3_QUERY_INSTRUCT) == 0);
TEST_ASSERT(cJSON_GetArraySize(detail) == 1);
cJSON_Delete(bad);
/* text_type on a Perplexity-family model is rejected. */
cJSON *p = cJSON_CreateObject();
cJSON_AddStringToObject(p, "text_type", "query");
- TEST_ASSERT(text_type_is_query(p, detail, EMBED_API_PERPLEXITY) == 0);
+ TEST_ASSERT(text_type_is_query(p, detail, NULL) == 0);
TEST_ASSERT(cJSON_GetArraySize(detail) == 2);
cJSON_Delete(p);
cJSON_Delete(detail);
}
+static void test_model_registry(void) {
+ model_slot slot = model_slot_for_id("gte-Qwen2-1.5B-instruct");
+ TEST_ASSERT(slot == MODEL_GTE_QWEN2_15);
+ if (slot == MODEL_UNKNOWN)
+ return;
+ TEST_ASSERT(k_models[slot].dim == 1536);
+ TEST_ASSERT(k_models[slot].min_dim == 1536);
+ TEST_ASSERT(k_models[slot].attention_mode == EMBED_ATTENTION_BIDIRECTIONAL);
+ TEST_ASSERT(k_models[slot].pooling_mode == EMBED_POOL_LAST_TOKEN);
+ TEST_ASSERT(k_models[slot].normalize_embeddings == 1);
+ TEST_ASSERT(strcmp(k_models[slot].query_instruct, EMBED_GTE_QWEN2_QUERY_INSTRUCT) == 0);
+}
+
static void test_collect_job_batch_deadline(void) {
http_server s;
memset(&s, 0, sizeof(s));
@@ -410,7 +423,7 @@ static int http_req(int port,
}
typedef struct {
- embed_server_model_spec_t spec[4];
+ embed_server_model_spec_t spec[5];
embed_server_config_t cfg;
int rc;
} srv_ctx;
@@ -657,6 +670,45 @@ static void test_http_embeddings(int port) {
cJSON_Delete(tt_query);
cJSON_Delete(tt_manual);
+ /* GTE-Qwen2 uses the same retrieval task text but publishes a space after
+ * "Query:". Verify the registry-specific prompt through the full path. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"gte-Qwen2-1.5B-instruct\",\"input\":\"hello\","
+ "\"text_type\":\"query\",\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ cJSON *gte_query = body ? parse_embeddings_body(body, 1) : NULL;
+ free(body);
+ body = NULL;
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"gte-Qwen2-1.5B-instruct\",\"input\":\"Instruct: Given a "
+ "web search query, retrieve relevant passages that answer the "
+ "query\\nQuery: hello\",\"text_type\":\"document\","
+ "\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ cJSON *gte_manual = body ? parse_embeddings_body(body, 1) : NULL;
+ free(body);
+ body = NULL;
+ if (gte_query && gte_manual) {
+ cJSON *qe = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(gte_query, "data"), 0),
+ "embedding");
+ cJSON *me = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(gte_manual, "data"), 0),
+ "embedding");
+ TEST_ASSERT(cJSON_IsArray(qe) && cJSON_IsArray(me));
+ TEST_ASSERT(cJSON_GetArraySize(qe) == 1536 && cJSON_GetArraySize(me) == 1536);
+ double gte_diff = 0.0;
+ for (int i = 0; i < cJSON_GetArraySize(qe); i++) {
+ double d = fabs(cJSON_GetArrayItem(qe, i)->valuedouble -
+ cJSON_GetArrayItem(me, i)->valuedouble);
+ if (d > gte_diff)
+ gte_diff = d;
+ }
+ TEST_ASSERT(gte_diff < 1e-6);
+ }
+ cJSON_Delete(gte_query);
+ cJSON_Delete(gte_manual);
+
/* text_type is rejected on a Perplexity-family model, and bad values 422. */
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
"{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello\","
@@ -897,7 +949,7 @@ static void test_http_rerank(int port) {
}
static void test_http_server(void) {
- char dir[1024], late_dir[1024], qwen_dir[1024];
+ char dir[1024], late_dir[1024], qwen_dir[1024], gte_dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
if (!mkdtemp(dir)) {
@@ -919,14 +971,24 @@ static void test_http_server(void) {
test_failures++;
return;
}
+ snprintf(gte_dir, sizeof(gte_dir), "%s/embed-srv-gte-test-XXXXXX",
+ getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
+ if (!mkdtemp(gte_dir)) {
+ fprintf(stderr, "FAIL: GTE mkdtemp\n");
+ test_failures++;
+ return;
+ }
/* The 0.6b server slot requires hidden_size 1024; everything else stays
* tiny. The model vocab covers every tokenizer fixture id. */
tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE};
+ tm_dims_t gte_dims = {1536, 2, 1, 64, 8, TF_VOCAB_SIZE};
if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 ||
tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 ||
tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 ||
tf_write_vocab(qwen_dir) != 0 ||
- tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0) {
+ tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0 ||
+ tf_write_vocab(gte_dir) != 0 ||
+ tm_write_qwen2_model_dims(gte_dir, "F32", &gte_dims, TF_EOT_ID, 0) != 0) {
fprintf(stderr, "FAIL: fixture write\n");
test_failures++;
return;
@@ -947,8 +1009,10 @@ static void test_http_server(void) {
ctx.spec[2].path = late_dir;
ctx.spec[3].id = "Qwen3-Embedding-0.6B";
ctx.spec[3].path = qwen_dir;
+ ctx.spec[4].id = "gte-Qwen2-1.5B-instruct";
+ ctx.spec[4].path = gte_dir;
ctx.cfg.models = ctx.spec;
- ctx.cfg.n_models = 4;
+ ctx.cfg.n_models = 5;
ctx.cfg.port = port;
ctx.cfg.batch_size = 2;
ctx.cfg.batch_wait_us = 1000;
@@ -992,6 +1056,7 @@ int main(void) {
test_append_embedding_value_float();
test_encoding_from_root_family();
test_text_type();
+ test_model_registry();
test_collect_job_batch_deadline();
test_collect_job_batch_zero_wait();
test_http_server();
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index d2534d2..95496b7 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -76,8 +76,12 @@ tm_write_config(const char *dir, const tm_dims_t *d, const char *model_type, int
/* Write a safetensors file holding `specs` with tm_value() contents.
* dtype: "F32" or "BF16". Returns 0 on success. */
-static int tm_write_safetensors_with_prefix(
- const char *path, const char *dtype, const tm_spec_t *specs, int n_specs, const char *prefix) {
+static int tm_write_safetensors_with_prefix_options(const char *path,
+ const char *dtype,
+ const tm_spec_t *specs,
+ int n_specs,
+ const char *prefix,
+ int zero_bias) {
int bf16 = strcmp(dtype, "BF16") == 0;
size_t esize = bf16 ? 2 : 4;
@@ -117,7 +121,7 @@ static int tm_write_safetensors_with_prefix(
const tm_spec_t *s = &specs[t];
size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1);
for (size_t i = 0; i < n; i++) {
- float v = tm_value(s->name, i);
+ float v = zero_bias && strstr(s->name, ".bias") ? 0.0f : tm_value(s->name, i);
if (bf16) {
uint16_t h = tm_f32_to_bf16(v);
fwrite(&h, sizeof(h), 1, f);
@@ -130,6 +134,11 @@ static int tm_write_safetensors_with_prefix(
return 0;
}
+static int tm_write_safetensors_with_prefix(
+ const char *path, const char *dtype, const tm_spec_t *specs, int n_specs, const char *prefix) {
+ return tm_write_safetensors_with_prefix_options(path, dtype, specs, n_specs, prefix, 0);
+}
+
static int
tm_write_safetensors(const char *path, const char *dtype, const tm_spec_t *specs, int n_specs) {
return tm_write_safetensors_with_prefix(path, dtype, specs, n_specs, "");
@@ -184,6 +193,76 @@ static inline int tm_write_qwen3_model_dims(const char *dir,
return tm_write_config(dir, d, "qwen3", eos_token_id);
}
+static inline int tm_write_qwen2_model_dims(
+ const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) {
+ int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
+ const tm_spec_t specs[] = {
+ {"embed_tokens.weight", d->vocab, d->hidden},
+ {"layers.0.self_attn.q_proj.weight", q, d->hidden},
+ {"layers.0.self_attn.q_proj.bias", q, 0},
+ {"layers.0.self_attn.k_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.k_proj.bias", kv, 0},
+ {"layers.0.self_attn.v_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.v_proj.bias", kv, 0},
+ {"layers.0.self_attn.o_proj.weight", d->hidden, q},
+ {"layers.0.input_layernorm.weight", d->hidden, 0},
+ {"layers.0.post_attention_layernorm.weight", d->hidden, 0},
+ {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate},
+ {"norm.weight", d->hidden, 0},
+ };
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fprintf(f,
+ "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
+ "\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
+ "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
+ "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0,"
+ "\"model_type\":\"qwen2\",\"eos_token_id\":%d,\"is_causal\":false}",
+ d->hidden, d->heads, d->kv_heads, d->head_dim, d->intermediate, d->vocab, eos_token_id);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/model.safetensors", dir);
+ if (tm_write_safetensors_with_prefix_options(
+ path, dtype, specs, (int)(sizeof(specs) / sizeof(specs[0])), "model.", zero_bias) != 0)
+ return -1;
+
+ snprintf(path, sizeof(path), "%s/1_Pooling", dir);
+ if (mkdir(path, 0755) != 0 && errno != EEXIST)
+ return -1;
+ snprintf(path, sizeof(path), "%s/1_Pooling/config.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fputs("{\"pooling_mode_cls_token\":false,\"pooling_mode_mean_tokens\":false,"
+ "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false,"
+ "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":true}",
+ f);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/modules.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fputs("[{\"type\":\"sentence_transformers.models.Transformer\"},"
+ "{\"type\":\"sentence_transformers.models.Pooling\"},"
+ "{\"type\":\"sentence_transformers.models.Normalize\"}]",
+ f);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/tokenizer_config.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fputs("{\"add_eos_token\":true}", f);
+ fclose(f);
+ return 0;
+}
+
/* Add the 1_Dense per-token projection head that turns a base model dir into
* a late-interaction snapshot. weight_name is "linear.weight" for a valid
* head; pass another name to synthesize a malformed head. */