aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 13:47:23 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 13:53:31 +0200
commit458563c163a885e336ef3365922af4d17e578f56 (patch)
tree52c7515d418e60f4923a69b5eade794a221cc5b4
parentd58e523c14356ac8a29480528dbceb19290b18c1 (diff)
downloadffwd-458563c163a885e336ef3365922af4d17e578f56.tar.gz
ffwd-458563c163a885e336ef3365922af4d17e578f56.zip
all: add CLS first-token pooling
-rw-r--r--.gitignore1
-rw-r--r--Makefile5
-rw-r--r--embed.c14
-rw-r--r--embed.h1
-rw-r--r--embed_config.c15
-rw-r--r--embed_cuda.cu13
-rw-r--r--embed_mlx.c27
-rw-r--r--tests/test_cls.c89
-rw-r--r--tests/tiny_model.h27
9 files changed, 165 insertions, 27 deletions
diff --git a/.gitignore b/.gitignore
index f7ebe6b..4b5e3ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ tests/test_late
tests/test_qwen2
tests/test_qwen3
tests/test_cli
+tests/test_cls
tests/cli_under_test
bench/bench_tokenizer
bench/bench_kernels
diff --git a/Makefile b/Makefile
index 1bd2b23..76b9f57 100644
--- a/Makefile
+++ b/Makefile
@@ -171,6 +171,8 @@ TEST_QWEN3_SRCS = tests/test_qwen3.c embed.c embed_config.c qwen_safetenso
$(KERNEL_SRCS)
TEST_QWEN2_SRCS = tests/test_qwen2.c embed.c embed_config.c qwen_safetensors.c \
$(KERNEL_SRCS)
+TEST_CLS_SRCS = tests/test_cls.c embed.c embed_config.c qwen_safetensors.c \
+ $(KERNEL_SRCS)
TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c embed_config.c qwen_tokenizer.c \
qwen_safetensors.c $(KERNEL_SRCS)
TEST_LATE_SRCS = tests/test_late.c embed.c embed_config.c qwen_tokenizer.c \
@@ -203,6 +205,9 @@ test:
$(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \
$(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
./tests/test_qwen2
+ $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \
+ $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ ./tests/test_cls
$(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \
$(TEST_WORKSPACE_SRCS) -lm -lpthread
./tests/test_workspace
diff --git a/embed.c b/embed.c
index a49aa7d..51e85a5 100644
--- a/embed.c
+++ b/embed.c
@@ -1178,8 +1178,11 @@ static int pool_embeddings(const embed_model_t *model,
float *emb = out_embeddings + (size_t)b * hidden;
memset(emb, 0, (size_t)hidden * sizeof(float));
- if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) {
- const float *row = x + (size_t)(end - 1) * hidden;
+ if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) {
+ /* Single-token pooling: CLS takes the first token, last-token the
+ * final one. The model's final RMSNorm is applied to that one row. */
+ int idx = cfg->pooling_mode == EMBED_POOL_CLS ? start : end - 1;
+ const float *row = x + (size_t)idx * hidden;
float sum_sq = 0.0f;
for (int d = 0; d < hidden; d++)
sum_sq += row[d] * row[d];
@@ -1502,8 +1505,11 @@ int embed_pool_batch(const embed_config_t *cfg,
return -1;
float *emb = out_embeddings + (size_t)b * cfg->hidden_size;
- if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) {
- const float *row = states + (size_t)(offset + len - 1) * cfg->hidden_size;
+ if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) {
+ /* CLS takes the first token, last-token the final one; these states
+ * already carry the model's final RMSNorm. */
+ int idx = cfg->pooling_mode == EMBED_POOL_CLS ? offset : offset + len - 1;
+ const float *row = states + (size_t)idx * cfg->hidden_size;
memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float));
} else {
memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float));
diff --git a/embed.h b/embed.h
index ef36359..197ee7f 100644
--- a/embed.h
+++ b/embed.h
@@ -40,6 +40,7 @@ typedef enum {
typedef enum {
EMBED_POOL_MEAN = 0,
EMBED_POOL_LAST_TOKEN = 1,
+ EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */
} embed_pooling_mode_t;
typedef struct {
diff --git a/embed_config.c b/embed_config.c
index 4a17740..fc07334 100644
--- a/embed_config.c
+++ b/embed_config.c
@@ -123,15 +123,18 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char
free(buf);
return -1;
}
- int supported = (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE);
- int unsupported = cls == JSON_BOOL_TRUE || max == JSON_BOOL_TRUE ||
- sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE;
+ int supported =
+ (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE) + (cls == JSON_BOOL_TRUE);
+ int unsupported =
+ max == JSON_BOOL_TRUE || sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE;
if (supported != 1 || unsupported) {
fprintf(stderr, "embed_config: unsupported Sentence Transformers pooling mode\n");
free(buf);
return -1;
}
- cfg->pooling_mode = last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN : EMBED_POOL_MEAN;
+ cfg->pooling_mode = cls == JSON_BOOL_TRUE ? EMBED_POOL_CLS
+ : last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN
+ : EMBED_POOL_MEAN;
free(buf);
buf = NULL;
}
@@ -270,7 +273,9 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads,
cfg->intermediate_size, cfg->head_dim,
cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional",
- cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean",
+ cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token"
+ : cfg->pooling_mode == EMBED_POOL_CLS ? "cls"
+ : "mean",
cfg->qk_norm ? "yes" : "no", cfg->qkv_bias ? "yes" : "no");
return 0;
diff --git a/embed_cuda.cu b/embed_cuda.cu
index 3100f44..0005a41 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -618,6 +618,16 @@ last_pool_kernel(float *out, const float *x, const int *offsets, int batch, int
out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d];
}
+/* CLS pooling: gather the first token of each packed sequence. */
+__global__ static void
+first_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
+ int b = blockIdx.x;
+ int tid = threadIdx.x;
+ int first = offsets[b];
+ for (int d = tid; d < hidden; d += blockDim.x)
+ out[(size_t)b * hidden + d] = x[(size_t)first * hidden + d];
+}
+
__global__ static void span_pool_kernel(
float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) {
int s = blockIdx.x;
@@ -1420,6 +1430,9 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
if (c->pooling_mode == EMBED_POOL_LAST_TOKEN)
last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
batch, c->hidden_size);
+ else if (c->pooling_mode == EMBED_POOL_CLS)
+ first_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
+ batch, c->hidden_size);
else
mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
batch, c->hidden_size);
diff --git a/embed_mlx.c b/embed_mlx.c
index d85dc0d..cebfbc6 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -1600,7 +1600,10 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
int max_seq = 0;
int has_padding = 0;
int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
- int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN;
+ int cls_pool = c->pooling_mode == EMBED_POOL_CLS;
+ /* CLS and last-token are both single-token pooling: gather one row per
+ * sequence (the first for CLS, the last otherwise) rather than mean-pool. */
+ int single_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN || cls_pool;
if (hidden <= 0 || c->vocab_size <= 0)
return -1;
@@ -1617,7 +1620,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
if (inputs[b].n_tokens != max_seq)
has_padding = 1;
}
- int needs_pool_mask = has_padding && !last_token_pool;
+ int needs_pool_mask = has_padding && !single_token_pool;
int needs_attn_mask = has_padding && !causal;
size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes;
@@ -1628,17 +1631,17 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 ||
mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 ||
mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 ||
- (last_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
+ (single_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
return -1;
if (ids_bytes == 0 || out_bytes == 0 ||
(needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) ||
- (needs_attn_mask && mask_bytes == 0) || (last_token_pool && last_index_bytes == 0))
+ (needs_attn_mask && mask_bytes == 0) || (single_token_pool && last_index_bytes == 0))
return -1;
int *padded_ids = (int *)malloc(ids_bytes);
float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL;
- int *last_index_data = last_token_pool ? (int *)malloc(last_index_bytes) : NULL;
+ int *last_index_data = single_token_pool ? (int *)malloc(last_index_bytes) : NULL;
if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) ||
(needs_attn_mask && !attn_mask_data)) {
free(padded_ids);
@@ -1648,7 +1651,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
free(last_index_data);
return -1;
}
- if (last_token_pool && !last_index_data) {
+ if (single_token_pool && !last_index_data) {
free(padded_ids);
free(pool_mask_data);
free(attn_mask_data);
@@ -1659,8 +1662,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
for (int b = 0; b < batch; b++) {
if (needs_pool_mask)
len_data[b] = (float)inputs[b].n_tokens;
- if (last_token_pool) {
- int index = inputs[b].n_tokens - 1;
+ if (single_token_pool) {
+ int index = cls_pool ? 0 : inputs[b].n_tokens - 1;
for (int d = 0; d < hidden; d++)
last_index_data[(size_t)b * hidden + d] = index;
}
@@ -1708,8 +1711,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array lengths =
needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0};
mlx_array last_indices =
- last_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
- : (mlx_array){0};
+ single_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
+ : (mlx_array){0};
free(padded_ids);
free(pool_mask_data);
free(attn_mask_data);
@@ -1717,7 +1720,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
free(last_index_data);
if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
- (needs_attn_mask && !arr_ok(attn_mask)) || (last_token_pool && !arr_ok(last_indices))) {
+ (needs_attn_mask && !arr_ok(attn_mask)) || (single_token_pool && !arr_ok(last_indices))) {
mlx_array_free(ids);
mlx_array_free(pool_mask);
mlx_array_free(attn_mask);
@@ -1753,7 +1756,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(x);
mlx_array emb = mlx_array_new();
- if (last_token_pool) {
+ if (single_token_pool) {
mlx_array selected = mlx_array_new();
mlx_take_along_axis(&selected, x_normed, last_indices, 1, S);
mlx_array_free(x_normed);
diff --git a/tests/test_cls.c b/tests/test_cls.c
new file mode 100644
index 0000000..fc03bb8
--- /dev/null
+++ b/tests/test_cls.c
@@ -0,0 +1,89 @@
+/* tests/test_cls.c - hermetic CLS (first-token) pooling across the load path,
+ * the encode path, and the public pooling API. */
+
+#include "embed.h"
+#include "tiny_model.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static float max_abs_diff(const float *a, const float *b, int n) {
+ float out = 0.0f;
+ for (int i = 0; i < n; i++) {
+ float d = fabsf(a[i] - b[i]);
+ if (d > out)
+ out = d;
+ }
+ return out;
+}
+
+int main(void) {
+ const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
+ char dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-cls-test-XXXXXX", tmp);
+ tm_dims_t dims = {4, 2, 1, 2, 8, 16};
+ if (!mkdtemp(dir) || tm_write_qwen2_model_pooling(dir, "F32", &dims, 15, 0, "cls") != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ embed_model_t *model = embed_model_load(dir);
+ embed_workspace_t *ws = embed_workspace_new(model);
+ const embed_config_t *cfg = embed_model_config(model);
+ if (!model || !ws || !cfg) {
+ fprintf(stderr, "model setup failed\n");
+ return 1;
+ }
+ if (cfg->pooling_mode != EMBED_POOL_CLS ||
+ cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL || !cfg->normalize_embeddings) {
+ fprintf(stderr, "CLS pooling metadata was not detected\n");
+ return 1;
+ }
+
+ int ids[] = {1, 2, 15};
+ float states[3 * 4], embedding[4], pooled[4];
+ if (embed_model_forward_into(model, ws, ids, 3, states) != 0 ||
+ embed_model_encode_into(model, ws, ids, 3, embedding) != 0 ||
+ embed_pool_batch(cfg, states, (int[]){3}, 1, pooled) != 0) {
+ fprintf(stderr, "CLS inference failed\n");
+ return 1;
+ }
+
+ /* CLS pools the first token's (already final-normed) state. Both the encode
+ * path and the public pooling API must return that L2-normalized vector. */
+ float expected[4];
+ memcpy(expected, states, sizeof(expected));
+ if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f ||
+ max_abs_diff(pooled, expected, 4) > 1e-5f) {
+ fprintf(stderr, "CLS pooling did not return the normalized first token\n");
+ return 1;
+ }
+
+ /* Guard against a symmetric fixture: the first token must differ from the
+ * last, so selecting the wrong end would fail this test. */
+ float last[4];
+ memcpy(last, states + 2 * 4, sizeof(last));
+ if (embed_l2_normalize(last, 4) != 0 || max_abs_diff(expected, last, 4) < 1e-4f) {
+ fprintf(stderr, "fixture too symmetric: first and last tokens coincide\n");
+ return 1;
+ }
+
+ /* Packed batch parity: each row's CLS vector equals its singleton CLS. */
+ int short_ids[] = {4, 15};
+ embed_input_t inputs[] = {{ids, 3}, {short_ids, 2}};
+ float batch[8], single[4];
+ if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 ||
+ embed_model_encode_into(model, ws, short_ids, 2, single) != 0 ||
+ max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) {
+ fprintf(stderr, "CLS packed batch parity failed\n");
+ return 1;
+ }
+
+ embed_workspace_free(ws);
+ embed_model_free(model);
+ puts("ok: CLS first-token pooling and packed batch parity");
+ return 0;
+}
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index 95496b7..e0d000b 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -193,8 +193,15 @@ static inline int tm_write_qwen3_model_dims(const char *dir,
return tm_write_config(dir, d, "qwen3", eos_token_id);
}
-static inline int tm_write_qwen2_model_dims(
- const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) {
+/* Write a tiny qwen2-style model whose Sentence Transformers pooling is one of
+ * "cls", "mean", or "lasttoken". The thin tm_write_qwen2_model_dims wrapper
+ * below keeps the historical last-token default for existing callers. */
+static inline int tm_write_qwen2_model_pooling(const char *dir,
+ const char *dtype,
+ const tm_dims_t *d,
+ int eos_token_id,
+ int zero_bias,
+ const char *pooling_key) {
int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
const tm_spec_t specs[] = {
{"embed_tokens.weight", d->vocab, d->hidden},
@@ -238,10 +245,13 @@ static inline int tm_write_qwen2_model_dims(
f = fopen(path, "w");
if (!f)
return -1;
- fputs("{\"pooling_mode_cls_token\":false,\"pooling_mode_mean_tokens\":false,"
- "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false,"
- "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":true}",
- f);
+ fprintf(f,
+ "{\"pooling_mode_cls_token\":%s,\"pooling_mode_mean_tokens\":%s,"
+ "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false,"
+ "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":%s}",
+ strcmp(pooling_key, "cls") == 0 ? "true" : "false",
+ strcmp(pooling_key, "mean") == 0 ? "true" : "false",
+ strcmp(pooling_key, "lasttoken") == 0 ? "true" : "false");
fclose(f);
snprintf(path, sizeof(path), "%s/modules.json", dir);
@@ -263,6 +273,11 @@ static inline int tm_write_qwen2_model_dims(
return 0;
}
+static inline int tm_write_qwen2_model_dims(
+ const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) {
+ return tm_write_qwen2_model_pooling(dir, dtype, d, eos_token_id, zero_bias, "lasttoken");
+}
+
/* Add the 1_Dense per-token projection head that turns a base model dir into
* a late-interaction snapshot. weight_name is "linear.weight" for a valid
* head; pass another name to synthesize a malformed head. */