diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 13:47:23 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 13:53:31 +0200 |
| commit | 458563c163a885e336ef3365922af4d17e578f56 (patch) | |
| tree | 52c7515d418e60f4923a69b5eade794a221cc5b4 | |
| parent | d58e523c14356ac8a29480528dbceb19290b18c1 (diff) | |
| download | ffwd-458563c163a885e336ef3365922af4d17e578f56.tar.gz ffwd-458563c163a885e336ef3365922af4d17e578f56.zip | |
all: add CLS first-token pooling
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | embed.c | 14 | ||||
| -rw-r--r-- | embed.h | 1 | ||||
| -rw-r--r-- | embed_config.c | 15 | ||||
| -rw-r--r-- | embed_cuda.cu | 13 | ||||
| -rw-r--r-- | embed_mlx.c | 27 | ||||
| -rw-r--r-- | tests/test_cls.c | 89 | ||||
| -rw-r--r-- | tests/tiny_model.h | 27 |
9 files changed, 165 insertions, 27 deletions
@@ -12,6 +12,7 @@ tests/test_late tests/test_qwen2 tests/test_qwen3 tests/test_cli +tests/test_cls tests/cli_under_test bench/bench_tokenizer bench/bench_kernels @@ -171,6 +171,8 @@ TEST_QWEN3_SRCS = tests/test_qwen3.c embed.c embed_config.c qwen_safetenso $(KERNEL_SRCS) TEST_QWEN2_SRCS = tests/test_qwen2.c embed.c embed_config.c qwen_safetensors.c \ $(KERNEL_SRCS) +TEST_CLS_SRCS = tests/test_cls.c embed.c embed_config.c qwen_safetensors.c \ + $(KERNEL_SRCS) TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c embed_config.c qwen_tokenizer.c \ qwen_safetensors.c $(KERNEL_SRCS) TEST_LATE_SRCS = tests/test_late.c embed.c embed_config.c qwen_tokenizer.c \ @@ -203,6 +205,9 @@ test: $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \ $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) ./tests/test_qwen2 + $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \ + $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + ./tests/test_cls $(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \ $(TEST_WORKSPACE_SRCS) -lm -lpthread ./tests/test_workspace @@ -1178,8 +1178,11 @@ static int pool_embeddings(const embed_model_t *model, float *emb = out_embeddings + (size_t)b * hidden; memset(emb, 0, (size_t)hidden * sizeof(float)); - if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) { - const float *row = x + (size_t)(end - 1) * hidden; + if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) { + /* Single-token pooling: CLS takes the first token, last-token the + * final one. The model's final RMSNorm is applied to that one row. */ + int idx = cfg->pooling_mode == EMBED_POOL_CLS ? start : end - 1; + const float *row = x + (size_t)idx * hidden; float sum_sq = 0.0f; for (int d = 0; d < hidden; d++) sum_sq += row[d] * row[d]; @@ -1502,8 +1505,11 @@ int embed_pool_batch(const embed_config_t *cfg, return -1; float *emb = out_embeddings + (size_t)b * cfg->hidden_size; - if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) { - const float *row = states + (size_t)(offset + len - 1) * cfg->hidden_size; + if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) { + /* CLS takes the first token, last-token the final one; these states + * already carry the model's final RMSNorm. */ + int idx = cfg->pooling_mode == EMBED_POOL_CLS ? offset : offset + len - 1; + const float *row = states + (size_t)idx * cfg->hidden_size; memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float)); } else { memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); @@ -40,6 +40,7 @@ typedef enum { typedef enum { EMBED_POOL_MEAN = 0, EMBED_POOL_LAST_TOKEN = 1, + EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */ } embed_pooling_mode_t; typedef struct { diff --git a/embed_config.c b/embed_config.c index 4a17740..fc07334 100644 --- a/embed_config.c +++ b/embed_config.c @@ -123,15 +123,18 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char free(buf); return -1; } - int supported = (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE); - int unsupported = cls == JSON_BOOL_TRUE || max == JSON_BOOL_TRUE || - sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE; + int supported = + (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE) + (cls == JSON_BOOL_TRUE); + int unsupported = + max == JSON_BOOL_TRUE || sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE; if (supported != 1 || unsupported) { fprintf(stderr, "embed_config: unsupported Sentence Transformers pooling mode\n"); free(buf); return -1; } - cfg->pooling_mode = last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN : EMBED_POOL_MEAN; + cfg->pooling_mode = cls == JSON_BOOL_TRUE ? EMBED_POOL_CLS + : last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN + : EMBED_POOL_MEAN; free(buf); buf = NULL; } @@ -270,7 +273,9 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->intermediate_size, cfg->head_dim, cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional", - cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean", + cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" + : cfg->pooling_mode == EMBED_POOL_CLS ? "cls" + : "mean", cfg->qk_norm ? "yes" : "no", cfg->qkv_bias ? "yes" : "no"); return 0; diff --git a/embed_cuda.cu b/embed_cuda.cu index 3100f44..0005a41 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -618,6 +618,16 @@ last_pool_kernel(float *out, const float *x, const int *offsets, int batch, int out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d]; } +/* CLS pooling: gather the first token of each packed sequence. */ +__global__ static void +first_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) { + int b = blockIdx.x; + int tid = threadIdx.x; + int first = offsets[b]; + for (int d = tid; d < hidden; d += blockDim.x) + out[(size_t)b * hidden + d] = x[(size_t)first * hidden + d]; +} + __global__ static void span_pool_kernel( float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) { int s = blockIdx.x; @@ -1420,6 +1430,9 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, if (c->pooling_mode == EMBED_POOL_LAST_TOKEN) last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); + else if (c->pooling_mode == EMBED_POOL_CLS) + first_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, + batch, c->hidden_size); else mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); diff --git a/embed_mlx.c b/embed_mlx.c index d85dc0d..cebfbc6 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -1600,7 +1600,10 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, int max_seq = 0; int has_padding = 0; int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; - int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN; + int cls_pool = c->pooling_mode == EMBED_POOL_CLS; + /* CLS and last-token are both single-token pooling: gather one row per + * sequence (the first for CLS, the last otherwise) rather than mean-pool. */ + int single_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN || cls_pool; if (hidden <= 0 || c->vocab_size <= 0) return -1; @@ -1617,7 +1620,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, if (inputs[b].n_tokens != max_seq) has_padding = 1; } - int needs_pool_mask = has_padding && !last_token_pool; + int needs_pool_mask = has_padding && !single_token_pool; int needs_attn_mask = has_padding && !causal; size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes; @@ -1628,17 +1631,17 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 || - (last_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) + (single_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) return -1; if (ids_bytes == 0 || out_bytes == 0 || (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) || - (needs_attn_mask && mask_bytes == 0) || (last_token_pool && last_index_bytes == 0)) + (needs_attn_mask && mask_bytes == 0) || (single_token_pool && last_index_bytes == 0)) return -1; int *padded_ids = (int *)malloc(ids_bytes); float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL; float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL; - int *last_index_data = last_token_pool ? (int *)malloc(last_index_bytes) : NULL; + int *last_index_data = single_token_pool ? (int *)malloc(last_index_bytes) : NULL; if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) || (needs_attn_mask && !attn_mask_data)) { free(padded_ids); @@ -1648,7 +1651,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, free(last_index_data); return -1; } - if (last_token_pool && !last_index_data) { + if (single_token_pool && !last_index_data) { free(padded_ids); free(pool_mask_data); free(attn_mask_data); @@ -1659,8 +1662,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, for (int b = 0; b < batch; b++) { if (needs_pool_mask) len_data[b] = (float)inputs[b].n_tokens; - if (last_token_pool) { - int index = inputs[b].n_tokens - 1; + if (single_token_pool) { + int index = cls_pool ? 0 : inputs[b].n_tokens - 1; for (int d = 0; d < hidden; d++) last_index_data[(size_t)b * hidden + d] = index; } @@ -1708,8 +1711,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array lengths = needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0}; mlx_array last_indices = - last_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) - : (mlx_array){0}; + single_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) + : (mlx_array){0}; free(padded_ids); free(pool_mask_data); free(attn_mask_data); @@ -1717,7 +1720,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, free(last_index_data); if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) || - (needs_attn_mask && !arr_ok(attn_mask)) || (last_token_pool && !arr_ok(last_indices))) { + (needs_attn_mask && !arr_ok(attn_mask)) || (single_token_pool && !arr_ok(last_indices))) { mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(attn_mask); @@ -1753,7 +1756,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(x); mlx_array emb = mlx_array_new(); - if (last_token_pool) { + if (single_token_pool) { mlx_array selected = mlx_array_new(); mlx_take_along_axis(&selected, x_normed, last_indices, 1, S); mlx_array_free(x_normed); diff --git a/tests/test_cls.c b/tests/test_cls.c new file mode 100644 index 0000000..fc03bb8 --- /dev/null +++ b/tests/test_cls.c @@ -0,0 +1,89 @@ +/* tests/test_cls.c - hermetic CLS (first-token) pooling across the load path, + * the encode path, and the public pooling API. */ + +#include "embed.h" +#include "tiny_model.h" + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +static float max_abs_diff(const float *a, const float *b, int n) { + float out = 0.0f; + for (int i = 0; i < n; i++) { + float d = fabsf(a[i] - b[i]); + if (d > out) + out = d; + } + return out; +} + +int main(void) { + const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"; + char dir[1024]; + snprintf(dir, sizeof(dir), "%s/embed-cls-test-XXXXXX", tmp); + tm_dims_t dims = {4, 2, 1, 2, 8, 16}; + if (!mkdtemp(dir) || tm_write_qwen2_model_pooling(dir, "F32", &dims, 15, 0, "cls") != 0) { + fprintf(stderr, "fixture creation failed\n"); + return 2; + } + + embed_model_t *model = embed_model_load(dir); + embed_workspace_t *ws = embed_workspace_new(model); + const embed_config_t *cfg = embed_model_config(model); + if (!model || !ws || !cfg) { + fprintf(stderr, "model setup failed\n"); + return 1; + } + if (cfg->pooling_mode != EMBED_POOL_CLS || + cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL || !cfg->normalize_embeddings) { + fprintf(stderr, "CLS pooling metadata was not detected\n"); + return 1; + } + + int ids[] = {1, 2, 15}; + float states[3 * 4], embedding[4], pooled[4]; + if (embed_model_forward_into(model, ws, ids, 3, states) != 0 || + embed_model_encode_into(model, ws, ids, 3, embedding) != 0 || + embed_pool_batch(cfg, states, (int[]){3}, 1, pooled) != 0) { + fprintf(stderr, "CLS inference failed\n"); + return 1; + } + + /* CLS pools the first token's (already final-normed) state. Both the encode + * path and the public pooling API must return that L2-normalized vector. */ + float expected[4]; + memcpy(expected, states, sizeof(expected)); + if (embed_l2_normalize(expected, 4) != 0 || max_abs_diff(embedding, expected, 4) > 1e-5f || + max_abs_diff(pooled, expected, 4) > 1e-5f) { + fprintf(stderr, "CLS pooling did not return the normalized first token\n"); + return 1; + } + + /* Guard against a symmetric fixture: the first token must differ from the + * last, so selecting the wrong end would fail this test. */ + float last[4]; + memcpy(last, states + 2 * 4, sizeof(last)); + if (embed_l2_normalize(last, 4) != 0 || max_abs_diff(expected, last, 4) < 1e-4f) { + fprintf(stderr, "fixture too symmetric: first and last tokens coincide\n"); + return 1; + } + + /* Packed batch parity: each row's CLS vector equals its singleton CLS. */ + int short_ids[] = {4, 15}; + embed_input_t inputs[] = {{ids, 3}, {short_ids, 2}}; + float batch[8], single[4]; + if (embed_model_encode_batch(model, ws, inputs, 2, batch) != 0 || + embed_model_encode_into(model, ws, short_ids, 2, single) != 0 || + max_abs_diff(batch, embedding, 4) > 1e-5f || max_abs_diff(batch + 4, single, 4) > 1e-5f) { + fprintf(stderr, "CLS packed batch parity failed\n"); + return 1; + } + + embed_workspace_free(ws); + embed_model_free(model); + puts("ok: CLS first-token pooling and packed batch parity"); + return 0; +} diff --git a/tests/tiny_model.h b/tests/tiny_model.h index 95496b7..e0d000b 100644 --- a/tests/tiny_model.h +++ b/tests/tiny_model.h @@ -193,8 +193,15 @@ static inline int tm_write_qwen3_model_dims(const char *dir, return tm_write_config(dir, d, "qwen3", eos_token_id); } -static inline int tm_write_qwen2_model_dims( - const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) { +/* Write a tiny qwen2-style model whose Sentence Transformers pooling is one of + * "cls", "mean", or "lasttoken". The thin tm_write_qwen2_model_dims wrapper + * below keeps the historical last-token default for existing callers. */ +static inline int tm_write_qwen2_model_pooling(const char *dir, + const char *dtype, + const tm_dims_t *d, + int eos_token_id, + int zero_bias, + const char *pooling_key) { int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim; const tm_spec_t specs[] = { {"embed_tokens.weight", d->vocab, d->hidden}, @@ -238,10 +245,13 @@ static inline int tm_write_qwen2_model_dims( f = fopen(path, "w"); if (!f) return -1; - fputs("{\"pooling_mode_cls_token\":false,\"pooling_mode_mean_tokens\":false," - "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false," - "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":true}", - f); + fprintf(f, + "{\"pooling_mode_cls_token\":%s,\"pooling_mode_mean_tokens\":%s," + "\"pooling_mode_max_tokens\":false,\"pooling_mode_mean_sqrt_len_tokens\":false," + "\"pooling_mode_weightedmean_tokens\":false,\"pooling_mode_lasttoken\":%s}", + strcmp(pooling_key, "cls") == 0 ? "true" : "false", + strcmp(pooling_key, "mean") == 0 ? "true" : "false", + strcmp(pooling_key, "lasttoken") == 0 ? "true" : "false"); fclose(f); snprintf(path, sizeof(path), "%s/modules.json", dir); @@ -263,6 +273,11 @@ static inline int tm_write_qwen2_model_dims( return 0; } +static inline int tm_write_qwen2_model_dims( + const char *dir, const char *dtype, const tm_dims_t *d, int eos_token_id, int zero_bias) { + return tm_write_qwen2_model_pooling(dir, dtype, d, eos_token_id, zero_bias, "lasttoken"); +} + /* Add the 1_Dense per-token projection head that turns a base model dir into * a late-interaction snapshot. weight_name is "linear.weight" for a valid * head; pass another name to synthesize a malformed head. */ |
