aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 13:47:23 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 13:53:31 +0200
commit458563c163a885e336ef3365922af4d17e578f56 (patch)
tree52c7515d418e60f4923a69b5eade794a221cc5b4 /tests
parentd58e523c14356ac8a29480528dbceb19290b18c1 (diff)
downloadffwd-458563c163a885e336ef3365922af4d17e578f56.tar.gz
ffwd-458563c163a885e336ef3365922af4d17e578f56.zip
all: add CLS first-token pooling
Diffstat (limited to 'tests')
-rw-r--r--tests/test_cls.c89
-rw-r--r--tests/tiny_model.h27
2 files changed, 110 insertions, 6 deletions
diff --git a/tests/test_cls.c b/tests/test_cls.c
new file mode 100644
index 0000000..fc03bb8
--- /dev/null
+++ b/tests/test_cls.c
@@ -0,0 +1,89 @@
+/* tests/test_cls.c - hermetic CLS (first-token) pooling across the load path,
+ * the encode path, and the public pooling API. */
+
+#include "embed.h"
+#include "tiny_model.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static float max_abs_diff(const float *a, const float *b, int n) {
+ float out = 0.0f;
+ for (int i = 0; i < n; i++) {
+ float d = fabsf(a[i] - b[i]);
+ if (d > out)
+ out = d;
+ }
+ return out;
+}
+
+int main(void) {
+ const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
+ char dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-cls-test-XXXXXX", tmp);
+ tm_dims_t dims = {4, 2, 1, 2, 8, 16};
+ if (!mkdtemp(dir) || tm_write_qwen2_model_pooling(dir, "F32", &dims, 15, 0, "cls") != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ embed_model_t *model = embed_model_load(dir);
+ embed_workspace_t *ws = embed_workspace_new(model);
+ const embed_config_t *cfg = embed_model_config(model);
+ if (!model || !ws || !cfg) {
+ fprintf(stderr, "model setup failed\n");
+ return 1;
+ }
+ if (cfg->pooling_mode != EMBED_POOL_CLS ||
+ cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL || !cfg->normalize_embeddings) {
+ fprintf(stderr, "CLS pooling metadata was not detected\n");
+ return 1;
+ }
+
+ int ids[] = {1, 2, 15};
+ float states[3 * 4], embedding[4], pooled[4];
+ if (embed_model_forward_into(model, ws, ids, 3, states) != 0 ||
+ embed_model_encode_into(model, ws, ids, 3, embedding) != 0 ||
+ embed_pool_batch(cfg, states, (int[]){3}, 1, pooled) != 0) {
+ fprintf(stderr, "CLS inference failed\n");
+ return 1;
+ }
+
+ /* CLS pools the first token's (already final-normed) state. Both the encode
+ * path and the public pooling API must return that L2-normalized vector. */
+ float expected[4];
+ memcpy(expected, states, sizeof(expected));
+ if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f ||
+ max_abs_diff(pooled, expected, 4) > 1e-5f) {
+ fprintf(stderr, "CLS pooling did not return the normalized first token\n");
+ return 1;
+ }
+
+ /* Guard against a symmetric fixture: the first token must differ from the
+ * last, so selecting the wrong end would fail this test. */
+ float last[4];
+ memcpy(last, states + 2 * 4, sizeof(last));
+ if (embed_l2_normalize(last, 4) != 0 || max_abs_diff(expected, last, 4) < 1e-4f) {
+ fprintf(stderr, "fixture too symmetric: first and last tokens coincide\n");
+ return 1;
+ }
+
+ /* Packed batch parity: each row's CLS vector equals its singleton CLS. */
+ int short_ids[] = {4, 15};
+ embed_input_t inputs[] = {{ids, 3}, {short_ids, 2}};
+ float batch[8], single[4];
+ if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 ||
+ embed_model_encode_into(model, ws, short_ids, 2, single) != 0 ||
+ max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) {
+ fprintf(stderr, "CLS packed batch parity failed\n");
+ return 1;
+ }
+
+ embed_workspace_free(ws);
+ embed_model_free(model);
+ puts("ok: CLS first-token pooling and packed batch parity");
+ return 0;
+}
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index 95496b7..e0d000b 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -193,8 +193,15 @@ static inline int tm_write_qwen3_model_dims(const char *dir,
return tm_write_config(dir, d, "qwen3", eos_token_id);
}
-static inline int tm_write_qwen2_model_dims(
- const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) {
+/* Write a tiny qwen2-style model whose Sentence Transformers pooling is one of
+ * "cls", "mean", or "lasttoken". The thin tm_write_qwen2_model_dims wrapper
+ * below keeps the historical last-token default for existing callers. */
+static inline int tm_write_qwen2_model_pooling(const char *dir,
+ const char *dtype,
+ const tm_dims_t *d,
+ int eos_token_id,
+ int zero_bias,
+ const char *pooling_key) {
int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
const tm_spec_t specs[] = {
{"embed_tokens.weight", d->vocab, d->hidden},
@@ -238,10 +245,13 @@ static inline int tm_write_qwen2_model_dims(
f = fopen(path, "w");
if (!f)
return -1;
- fputs("{\"pooling_mode_cls_token\":false,\"pooling_mode_mean_tokens\":false,"
- "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false,"
- "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":true}",
- f);
+ fprintf(f,
+ "{\"pooling_mode_cls_token\":%s,\"pooling_mode_mean_tokens\":%s,"
+ "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false,"
+ "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":%s}",
+ strcmp(pooling_key, "cls") == 0 ? "true" : "false",
+ strcmp(pooling_key, "mean") == 0 ? "true" : "false",
+ strcmp(pooling_key, "lasttoken") == 0 ? "true" : "false");
fclose(f);
snprintf(path, sizeof(path), "%s/modules.json", dir);
@@ -263,6 +273,11 @@ static inline int tm_write_qwen2_model_dims(
return 0;
}
+static inline int tm_write_qwen2_model_dims(
+ const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) {
+ return tm_write_qwen2_model_pooling(dir, dtype, d, eos_token_id, zero_bias, "lasttoken");
+}
+
/* Add the 1_Dense per-token projection head that turns a base model dir into
* a late-interaction snapshot. weight_name is "linear.weight" for a valid
* head; pass another name to synthesize a malformed head. */