diff options
Diffstat (limited to 'src/embed.c')
| -rw-r--r-- | src/embed.c | 1338 |
1 files changed, 0 insertions, 1338 deletions
diff --git a/src/embed.c b/src/embed.c deleted file mode 100644 index baa9d11..0000000 --- a/src/embed.c +++ /dev/null @@ -1,1338 +0,0 @@ -#include "embed.h" -#include "engine.h" -#include "alloc.h" -#include "config.h" -#include "internal.h" -#include "dtype.h" -#include "kernels.h" -#include "safetensors.h" - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <math.h> -#include <limits.h> - -/* ======================================================================== - * Globals - * ======================================================================== */ - -int embed_verbose = 0; - -#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16 -#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4 - -/* ======================================================================== - * Model-directory probes - * ======================================================================== */ - -static int model_dir_has_file(const char *model_dir, const char *rel) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) - return 0; - FILE *f = fopen(path, "rb"); - if (!f) - return 0; - fclose(f); - return 1; -} - -int model_dir_has_late_projection(const char *model_dir) { - return model_dir_has_file(model_dir, "1_Dense/config.json") && - model_dir_has_file(model_dir, "1_Dense/model.safetensors"); -} - -/* ======================================================================== - * Weight loading (direct mmap pointers into safetensors) - * ======================================================================== */ - -static size_t dtype_size(safetensor_dtype_t dtype) { - switch (dtype) { - case DTYPE_F32: - return sizeof(float); - case DTYPE_BF16: - return sizeof(uint16_t); - default: - return 0; - } -} - -static const char *dtype_name(safetensor_dtype_t dtype) { - switch (dtype) { - case DTYPE_F32: - return "F32"; - case DTYPE_F16: - return "F16"; - case DTYPE_BF16: - return "BF16"; - case DTYPE_I32: - return "I32"; - case DTYPE_I64: - return "I64"; - case DTYPE_BOOL: - return "BOOL"; - default: - return "UNKNOWN"; - } -} - -int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, - const int64_t *shape, - int ndim, - int bf16_ok) { - if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name, - dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (t->shape[i] != shape[i]) { - fprintf(stderr, - "embed: bad shape for %s at dim %d: " - "got %lld, expected %lld\n", - name, i, (long long)t->shape[i], (long long)shape[i]); - return 0; - } - if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "embed: shape too large for %s\n", name); - return 0; - } - numel *= (size_t)shape[i]; - } - - size_t elem_size = dtype_size(t->dtype); - if (elem_size == 0) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype)); - return 0; - } - - size_t bytes; - if (mul_size(numel, elem_size, &bytes) != 0) { - fprintf(stderr, "embed: tensor data too large for %s\n", name); - return 0; - } - if (t->data_size != bytes) { - fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, - bytes); - return 0; - } - - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "embed: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static embed_weight_ref_t -load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { - embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL}; - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "embed: tensor not found: %s\n", name); - return ref; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) - return ref; - - ref.dtype = t->dtype; - ref.data = safetensors_data(sf, t); - return ref; -} - -static float * -load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "embed: tensor not found: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) - return NULL; - return safetensors_get_f32(sf, t); -} - -static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { - for (int i = 0; i < n; i++) - dst[i] = embed_bf16_to_f32(src[i]); -} - -static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) { - if (w->dtype == DTYPE_F32) { - const float *src = (const float *)w->data + row * (size_t)row_width; - memcpy(dst, src, (size_t)row_width * sizeof(float)); - } else { - const uint16_t *src = (const uint16_t *)w->data + row * (size_t)row_width; - bf16_row_to_f32(dst, src, row_width); - } -} - -static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) { - if (ws->bf16_widen_count >= count) - return 0; - float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float)); - if (!p) - return -1; - ws->bf16_widen = p; - ws->bf16_widen_count = count; - return 0; -} - -void linear_nobias_weight(embed_workspace_t *ws, - float *y, - const float *x, - const embed_weight_ref_t *w, - int seq_len, - int in_dim, - int out_dim) { - if (w->dtype == DTYPE_F32) { - embed_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); - return; - } - if (seq_len > 16) { - /* Widen the weight matrix once into workspace scratch and use the - * BLAS-backed F32 path; below that the fused BF16 row kernels win. */ - size_t count = (size_t)out_dim * (size_t)in_dim; - if (ensure_bf16_widen(ws, count) == 0) { - embed_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count); - embed_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); - return; - } - } - embed_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim); -} - -static void linear_qkv_weight(embed_workspace_t *ws, - float *q, - float *k, - float *v, - const float *x, - const embed_weight_ref_t *wq, - const embed_weight_ref_t *wk, - const embed_weight_ref_t *wv, - int seq_len, - int in_dim, - int q_dim, - int kv_dim) { - if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 && - seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) { - embed_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, - (const uint16_t *)wk->data, (const uint16_t *)wv->data, - seq_len, in_dim, q_dim, kv_dim); - return; - } - - linear_nobias_weight(ws, q, x, wq, seq_len, in_dim, q_dim); - linear_nobias_weight(ws, k, x, wk, seq_len, in_dim, kv_dim); - linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim); -} - -static void add_bias_rows(float *x, const float *bias, int rows, int dim) { - if (!bias) - return; - for (int row = 0; row < rows; row++) { - float *dst = x + (size_t)row * dim; - for (int d = 0; d < dim; d++) - dst[d] += bias[d]; - } -} - -static void linear_pair_weight(embed_workspace_t *ws, - float *a, - float *b, - const float *x, - const embed_weight_ref_t *wa, - const embed_weight_ref_t *wb, - int seq_len, - int in_dim, - int a_dim, - int b_dim) { - if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && - seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) { - embed_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, - (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim); - return; - } - - linear_nobias_weight(ws, a, x, wa, seq_len, in_dim, a_dim); - linear_nobias_weight(ws, b, x, wb, seq_len, in_dim, b_dim); -} - -/* ======================================================================== - * Working buffer management - * ======================================================================== */ - -static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) { - if (seq <= ws->buf_seq_cap) - return 0; - - int cap; - if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) - return -1; - -#define R(ptr, n) \ - do { \ - if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ - return -1; \ - } while (0) - - R(ws->x, c->hidden_size); - R(ws->x_norm, c->hidden_size); - R(ws->q, c->q_dim); - R(ws->k, c->kv_dim); - R(ws->v, c->kv_dim); - R(ws->attn_out, c->q_dim); - R(ws->proj_out, c->hidden_size); - R(ws->ffn_gate, c->intermediate_size); - R(ws->ffn_up, c->intermediate_size); - -#undef R - - ws->buf_seq_cap = cap; - return 0; -} - -static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) { - size_t bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (bytes <= ws->attn_scores_bytes) - return; - - void *p = realloc(ws->attn_scores, bytes); - if (!p) - return; - ws->attn_scores = (float *)p; - ws->attn_scores_bytes = bytes; -} - -/* ======================================================================== - * RoPE cache - * ======================================================================== */ - -static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) { - if (n_pos <= ws->rope_cache_cap) - return 0; - - int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; - while (cap < n_pos) { - if (cap > INT_MAX / 2) - return -1; - cap *= 2; - } - - int head_dim = cfg->head_dim; - int half = head_dim / 2; - float theta = cfg->rope_theta; - - size_t count; - if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) - return -1; - if (realloc_floats(&ws->rope_cos, count) != 0) - return -1; - if (realloc_floats(&ws->rope_sin, count) != 0) - return -1; - - for (int pos = ws->rope_cache_cap; pos < cap; pos++) { - float fp = (float)pos; - float *c = ws->rope_cos + (size_t)pos * head_dim; - float *s = ws->rope_sin + (size_t)pos * head_dim; - for (int d = 0; d < half; d++) { - float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); - float angle = fp * inv; - float cv = cosf(angle), sv = sinf(angle); - c[d] = cv; - c[half + d] = cv; - s[d] = sv; - s[half + d] = sv; - } - } - - ws->rope_cache_cap = cap; - return 0; -} - -static int ensure_offsets(embed_workspace_t *ws, int batch) { - if (!ws || batch <= 0 || batch == INT_MAX) - return -1; - - int needed = batch + 1; - if (needed <= ws->offsets_cap) - return 0; - - if (realloc_ints(&ws->offsets, (size_t)needed) != 0) - return -1; - ws->offsets_cap = needed; - return 0; -} - -/* ======================================================================== - * Model / workspace load and free - * ======================================================================== */ - -/* BERT-family weight loader. The tensor names differ entirely from the Qwen - * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry - * point. Reused embed_layer_t slots carry their BERT meaning: wo is the - * attention output dense, input_norm/post_attn_norm are the two block LayerNorm - * weights, up_proj/down_proj are the two FFN matrices (see embed_internal.h). */ -static int -load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { - const embed_config_t *cfg = &model->config; - embed_weights_t *w = &model->weights; - char name[256]; - - /* sentence-transformers checkpoints store the BERT submodule unprefixed; - * raw HF checkpoints prefix every tensor with "bert.". */ - const char *p; - if (multi_safetensors_find(ms, "embeddings.word_embeddings.weight", NULL)) - p = ""; - else if (multi_safetensors_find(ms, "bert.embeddings.word_embeddings.weight", NULL)) - p = "bert."; - else { - fprintf(stderr, "embed_model_load: BERT word embeddings not found\n"); - return -1; - } - - int hidden = cfg->hidden_size, inter = cfg->intermediate_size; - - if (layer_start == 0) { - const int64_t emb_shape[2] = {cfg->vocab_size, hidden}; - snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p); - w->embed_tokens = load_weight_direct(ms, name, emb_shape, 2); - const int64_t pos_shape[2] = {cfg->max_position_embeddings, hidden}; - snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p); - w->position_embeddings = load_norm_f32(ms, name, pos_shape, 2); - const int64_t type_shape[2] = {2, hidden}; - snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p); - w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2); - const int64_t ln_shape[1] = {hidden}; - snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p); - w->embed_ln_w = load_norm_f32(ms, name, ln_shape, 1); - snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); - w->embed_ln_b = load_norm_f32(ms, name, ln_shape, 1); - if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || - !w->embed_ln_w || !w->embed_ln_b) - return -1; - } - - w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); - if (!w->layers) - return -1; - - for (int i = layer_start; i < layer_end; i++) { - embed_layer_t *l = &w->layers[i]; - -#define BERT_W(field, suffix, d0, d1) \ - do { \ - const int64_t e[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ - l->field = load_weight_direct(ms, name, e, 2); \ - if (!l->field.data) \ - return -1; \ - } while (0) -#define BERT_V(field, suffix, d0) \ - do { \ - const int64_t e[1] = {(d0)}; \ - snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ - l->field = load_norm_f32(ms, name, e, 1); \ - if (!l->field) \ - return -1; \ - } while (0) - - BERT_W(wq, "attention.self.query.weight", hidden, hidden); - BERT_V(q_bias, "attention.self.query.bias", hidden); - BERT_W(wk, "attention.self.key.weight", hidden, hidden); - BERT_V(k_bias, "attention.self.key.bias", hidden); - BERT_W(wv, "attention.self.value.weight", hidden, hidden); - BERT_V(v_bias, "attention.self.value.bias", hidden); - BERT_W(wo, "attention.output.dense.weight", hidden, hidden); - BERT_V(o_bias, "attention.output.dense.bias", hidden); - BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); - BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); - BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); - BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); - BERT_W(down_proj, "output.dense.weight", hidden, inter); - BERT_V(ffn_out_bias, "output.dense.bias", hidden); - BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); - BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); - -#undef BERT_W -#undef BERT_V - } - /* BERT has no final norm: w->norm stays NULL, and pooling skips it. */ - return 0; -} - -embed_model_t * -model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) { - if (!allow_late && model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "embed_model_load: late-interaction models require " - "token-level MaxSim support and are not valid pooled " - "embedding models yet\n"); - return NULL; - } - - embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t)); - if (!model) - return NULL; - - /* Parse config.json */ - if (embed_config_parse(&model->config, model_dir) != 0) { - free(model); - return NULL; - } - - const embed_config_t *cfg = &model->config; - if (layer_end == -1) - layer_end = cfg->n_layers; - if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) { - fprintf(stderr, - "embed_model_load: invalid layer range [%d, %d) " - "for %d-layer model\n", - layer_start, layer_end, cfg->n_layers); - free(model); - return NULL; - } - model->layer_start = layer_start; - model->layer_end = layer_end; - model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL - ? embed_causal_gqa_attention_packed_with_scratch - : embed_bidirectional_gqa_attention_packed_with_scratch; - - /* Open safetensors (handles single file or multi-shard) */ - multi_safetensors_t *ms = multi_safetensors_open(model_dir); - if (!ms) { - fprintf(stderr, "embed_model_load: failed to open safetensors in %s\n", model_dir); - free(model); - return NULL; - } - model->safetensors = ms; - - if (cfg->family == EMBED_FAMILY_BERT) { - if (load_bert_weights(model, ms, layer_start, layer_end) != 0) - goto fail; - if (embed_verbose >= 1) - fprintf(stderr, "embed_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start, - layer_end, cfg->hidden_size); - return model; - } - - const char *weight_prefix = multi_safetensors_weight_prefix(ms, "embed_tokens.weight"); - if (!weight_prefix) { - fprintf(stderr, "embed_model_load: neither embed_tokens.weight nor " - "model.embed_tokens.weight was found\n"); - goto fail; - } - - if (embed_verbose >= 1) - fprintf(stderr, "embed_model_load: loading weights from %s\n", model_dir); - - char name[256]; - - /* Only the first stage needs token embeddings. */ - embed_weights_t *w = &model->weights; - if (layer_start == 0) { - const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size}; - snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix); - w->embed_tokens = load_weight_direct(ms, name, embed_shape, 2); - if (!w->embed_tokens.data) - goto fail; - } - - /* Allocate layer array */ - w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); - if (!w->layers) - goto fail; - - /* Per-layer weights */ - for (int i = layer_start; i < layer_end; i++) { - embed_layer_t *l = &w->layers[i]; - -#define LOAD_NORM(field, fmt, d0) \ - do { \ - const int64_t expect[1] = {(d0)}; \ - snprintf(name, sizeof(name), fmt, weight_prefix, i); \ - l->field = load_norm_f32(ms, name, expect, 1); \ - if (!l->field) \ - goto fail; \ - } while (0) - -#define LOAD2(field, fmt, d0, d1) \ - do { \ - const int64_t expect[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), fmt, weight_prefix, i); \ - l->field = load_weight_direct(ms, name, expect, 2); \ - if (!l->field.data) \ - goto fail; \ - } while (0) - - LOAD2(wq, "%slayers.%d.self_attn.q_proj.weight", cfg->q_dim, cfg->hidden_size); - LOAD2(wk, "%slayers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size); - LOAD2(wv, "%slayers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size); - LOAD2(wo, "%slayers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim); - if (cfg->qkv_bias) { - LOAD_NORM(q_bias, "%slayers.%d.self_attn.q_proj.bias", cfg->q_dim); - LOAD_NORM(k_bias, "%slayers.%d.self_attn.k_proj.bias", cfg->kv_dim); - LOAD_NORM(v_bias, "%slayers.%d.self_attn.v_proj.bias", cfg->kv_dim); - } - if (cfg->qk_norm) { - LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim); - LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim); - } - LOAD_NORM(input_norm, "%slayers.%d.input_layernorm.weight", cfg->hidden_size); - LOAD_NORM(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", cfg->hidden_size); - LOAD2(gate_proj, "%slayers.%d.mlp.gate_proj.weight", cfg->intermediate_size, - cfg->hidden_size); - LOAD2(up_proj, "%slayers.%d.mlp.up_proj.weight", cfg->intermediate_size, cfg->hidden_size); - LOAD2(down_proj, "%slayers.%d.mlp.down_proj.weight", cfg->hidden_size, - cfg->intermediate_size); - -#undef LOAD2 -#undef LOAD_NORM - - if (embed_verbose >= 2) - fprintf(stderr, " layer %d loaded\n", i); - } - - /* Only the final stage applies the output RMSNorm. */ - if (layer_end == cfg->n_layers) { - const int64_t norm_shape[1] = {cfg->hidden_size}; - snprintf(name, sizeof(name), "%snorm.weight", weight_prefix); - w->norm = load_norm_f32(ms, name, norm_shape, 1); - if (!w->norm) - goto fail; - } - - if (embed_verbose >= 1) - fprintf(stderr, - "embed_model_load: layers [%d, %d) loaded " - "(%d-dim embeddings)\n", - layer_start, layer_end, cfg->hidden_size); - - return model; - -fail: - embed_model_free(model); - return NULL; -} - -embed_model_t *embed_model_load(const char *model_dir) { - return model_load_range_ex(model_dir, 0, -1, 0); -} - -void embed_model_free(embed_model_t *model) { - if (!model) - return; - if (model->weights.layers) { - for (int i = 0; i < model->config.n_layers; i++) { - embed_layer_t *l = &model->weights.layers[i]; - free((void *)l->q_bias); - free((void *)l->k_bias); - free((void *)l->v_bias); - free((void *)l->q_norm); - free((void *)l->k_norm); - free((void *)l->input_norm); - free((void *)l->post_attn_norm); - free((void *)l->o_bias); - free((void *)l->attn_ln_bias); - free((void *)l->ffn_inter_bias); - free((void *)l->ffn_out_bias); - free((void *)l->ffn_ln_bias); - } - } - free((void *)model->weights.norm); - free((void *)model->weights.position_embeddings); - free((void *)model->weights.token_type_embeddings); - free((void *)model->weights.embed_ln_w); - free((void *)model->weights.embed_ln_b); - if (model->safetensors) - multi_safetensors_close((multi_safetensors_t *)model->safetensors); - free(model->weights.layers); - free(model); -} - -embed_workspace_t *embed_workspace_new(const embed_model_t *model) { - if (!model) - return NULL; - embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t)); - if (!ws) - return NULL; - ws->model = model; - return ws; -} - -void embed_workspace_free(embed_workspace_t *ws) { - if (!ws) - return; - free(ws->x); - free(ws->x_norm); - free(ws->q); - free(ws->k); - free(ws->v); - free(ws->attn_out); - free(ws->proj_out); - free(ws->ffn_gate); - free(ws->ffn_up); - free(ws->attn_scores); - free(ws->offsets); - free(ws->rope_cos); - free(ws->rope_sin); - free(ws->bf16_widen); - free(ws); -} - -const embed_config_t *embed_model_config(const embed_model_t *model) { - return model ? &model->config : NULL; -} - -size_t embed_workspace_nbytes(const embed_workspace_t *ws) { - if (!ws) - return 0; - - size_t total = sizeof(*ws); - const embed_model_t *model = ws->model; - if (!model) - return total; - - const embed_config_t *c = &model->config; - size_t row_floats = 0; - const int dims[] = { - c->hidden_size, /* x */ - c->hidden_size, /* x_norm */ - c->q_dim, /* q */ - c->kv_dim, /* k */ - c->kv_dim, /* v */ - c->q_dim, /* attn_out */ - c->hidden_size, /* proj_out */ - c->intermediate_size, /* ffn_gate */ - c->intermediate_size, /* ffn_up */ - }; - - for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { - if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0) - return SIZE_MAX; - } - - size_t buf_floats = 0; - if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) - return SIZE_MAX; - - size_t rope_floats = 0; - if (ws->rope_cache_cap > 0) { - size_t rope_rows = 0; - if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 || - mul_size(rope_rows, 2, &rope_floats) != 0) - return SIZE_MAX; - } - - size_t all_floats = 0, bytes = 0; - if (add_size(buf_floats, rope_floats, &all_floats) != 0 || - mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0) - return SIZE_MAX; - - size_t offset_bytes = 0; - if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) - return SIZE_MAX; - if (add_size(total, offset_bytes, &total) != 0) - return SIZE_MAX; - if (add_size(total, ws->attn_scores_bytes, &total) != 0) - return SIZE_MAX; - - size_t widen_bytes = 0; - if (ws->bf16_widen_count > 0 && - mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0) - return SIZE_MAX; - if (add_size(total, widen_bytes, &total) != 0) - return SIZE_MAX; - - return total; -} - -/* ======================================================================== - * Packed forward pass - * ======================================================================== */ - -int build_offsets(const embed_input_t *inputs, - int batch, - int require_ids, - int *offsets, - int *total_out, - int *max_out) { - if (!inputs || batch <= 0 || !offsets || !total_out || !max_out) - return -1; - - int total = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - if ((require_ids && !inputs[b].ids) || inputs[b].n_tokens <= 0) - return -1; - if (total > INT32_MAX - inputs[b].n_tokens) - return -1; - offsets[b] = total; - total += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) - max_seq = inputs[b].n_tokens; - } - offsets[batch] = total; - *total_out = total; - *max_out = max_seq; - return 0; -} - -static int forward_packed_slice_inplace(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, - int batch, - const int *offsets, - int total_seq, - int max_seq, - const float *input_states, - int layer_start, - int layer_end, - int apply_final_norm) { - if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || - total_seq <= 0 || max_seq <= 0) - return -1; - - const embed_config_t *cfg = &model->config; - const embed_weights_t *w = &model->weights; - if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers || - layer_start < model->layer_start || layer_end > model->layer_end || - (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) || - (layer_start == 0 && !w->embed_tokens.data) || - (apply_final_norm && (layer_end != cfg->n_layers || !w->norm))) - return -1; - - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; - int n_kv_heads = cfg->n_kv_heads; - int head_dim = cfg->head_dim; - int q_dim = cfg->q_dim; - int kv_dim = cfg->kv_dim; - int inter = cfg->intermediate_size; - float eps = cfg->rms_norm_eps; - - if (ensure_buffers(ws, cfg, total_seq) != 0) - return -1; - if (ensure_rope_cache(ws, cfg, max_seq) != 0) - return -1; - ensure_attention_scores(ws, offsets, batch); - - const float *rope_cos = ws->rope_cos; - const float *rope_sin = ws->rope_sin; - - float *x = ws->x; - float *x_norm = ws->x_norm; - float *q_buf = ws->q; - float *k_buf = ws->k; - float *v_buf = ws->v; - float *attn_out = ws->attn_out; - float *proj_out = ws->proj_out; - float *ffn_gate = ws->ffn_gate; - float *ffn_up = ws->ffn_up; - - /* A first stage performs embedding lookup. Later stages resume from a - * packed hidden-state tensor received from the preceding stage. */ - if (layer_start == 0) { - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - for (int i = 0; i < inputs[b].n_tokens; i++) { - int id = inputs[b].ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, - i); - return -1; - } - copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id, - hidden); - } - } - } else { - memcpy(x, input_states, (size_t)total_seq * hidden * sizeof(float)); - } - - float scale = 1.0f / sqrtf((float)head_dim); - - /* 2. Transformer layers. Linear/MLP ops run over all packed tokens; - * attention is explicitly block-diagonal by sequence offsets. */ - for (int layer = layer_start; layer < layer_end; layer++) { - const embed_layer_t *l = &w->layers[layer]; - - embed_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - - linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, - hidden, q_dim, kv_dim); - - add_bias_rows(q_buf, l->q_bias, total_seq, q_dim); - add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim); - add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim); - if (cfg->qk_norm) { - embed_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); - embed_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); - } - - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - int len = offsets[b + 1] - start; - embed_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, - head_dim); - embed_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, - n_kv_heads, head_dim); - } - - model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, - head_dim, scale, ws->attn_scores, ws->attn_scores_bytes); - - linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); - - embed_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); - - linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq, - hidden, inter, inter); - embed_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); - linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); - } - - if (apply_final_norm) - embed_rms_norm(x, x, w->norm, total_seq, hidden, eps); - return 0; -} - -int forward_packed_inplace(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, - int batch, - const int *offsets, - int total_seq, - int max_seq, - int apply_final_norm) { - return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL, - 0, model->config.n_layers, apply_final_norm); -} - -/* BERT-family forward: token + learned-position + token-type embeddings and an - * embedding LayerNorm, then post-norm encoder layers (bias on every projection, - * GeLU feed-forward, no RoPE, no per-head norm). Bidirectional attention is - * block-diagonal over the packed batch, so each sequence's positions restart at - * 0. Leaves the final hidden states in ws->x; pooling reads them as-is because - * the last layer's LayerNorm is already the model's final normalization. */ -static int forward_packed_bert(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, - int batch, - const int *offsets, - int total_seq, - int max_seq) { - if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || - total_seq <= 0 || max_seq <= 0) - return -1; - - const embed_config_t *cfg = &model->config; - const embed_weights_t *w = &model->weights; - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; - int head_dim = cfg->head_dim; - int inter = cfg->intermediate_size; - float eps = cfg->layer_norm_eps; - - if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || - !w->embed_ln_w || !w->embed_ln_b) - return -1; - if (max_seq > cfg->max_position_embeddings) { - fprintf(stderr, "embed: sequence length %d exceeds max_position_embeddings %d\n", max_seq, - cfg->max_position_embeddings); - return -1; - } - if (ensure_buffers(ws, cfg, total_seq) != 0) - return -1; - ensure_attention_scores(ws, offsets, batch); - - float *x = ws->x; - float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; - float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up; - - /* Embeddings: token + learned absolute position + token-type[0], then the - * embedding LayerNorm over every packed token. */ - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - for (int i = 0; i < inputs[b].n_tokens; i++) { - int id = inputs[b].ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, i); - return -1; - } - float *row = x + (size_t)(start + i) * hidden; - copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden); - embed_add_inplace(row, w->position_embeddings + (size_t)i * hidden, hidden); - embed_add_inplace(row, w->token_type_embeddings, hidden); - } - } - embed_layer_norm(x, x, w->embed_ln_w, w->embed_ln_b, total_seq, hidden, eps); - - float scale = 1.0f / sqrtf((float)head_dim); - - for (int layer = 0; layer < cfg->n_layers; layer++) { - const embed_layer_t *l = &w->layers[layer]; - - /* Self-attention (post-norm): projections read x directly. */ - linear_qkv_weight(ws, q_buf, k_buf, v_buf, x, &l->wq, &l->wk, &l->wv, total_seq, hidden, - hidden, hidden); - add_bias_rows(q_buf, l->q_bias, total_seq, hidden); - add_bias_rows(k_buf, l->k_bias, total_seq, hidden); - add_bias_rows(v_buf, l->v_bias, total_seq, hidden); - - model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim, - scale, ws->attn_scores, ws->attn_scores_bytes); - - linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, hidden, hidden); - add_bias_rows(proj_out, l->o_bias, total_seq, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); - embed_layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps); - - /* Feed-forward (post-norm): dense -> GeLU -> dense. */ - linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter); - add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter); - if (cfg->ffn_act == EMBED_ACT_GELU_TANH) - embed_gelu_tanh_inplace(ffn_up, total_seq * inter); - else - embed_gelu_inplace(ffn_up, total_seq * inter); - linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden); - add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); - embed_layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps); - } - return 0; -} - -static int pool_embeddings(const embed_model_t *model, - const embed_workspace_t *ws, - const int *offsets, - int batch, - float *out_embeddings) { - const embed_config_t *cfg = &model->config; - int hidden = cfg->hidden_size; - float eps = cfg->rms_norm_eps; - /* Qwen pools pre-final-norm states and applies the final RMSNorm here; BERT - * pools already-normed states (no final norm), signalled by a NULL norm. */ - const float *norm_weight = model->weights.norm; - const float *x = ws->x; - if (hidden <= 0 || !x) - return -1; - - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - int end = offsets[b + 1]; - int len = end - start; - float *emb = out_embeddings + (size_t)b * hidden; - - if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) { - /* Single-token pooling: CLS takes the first token, last-token the - * final one. */ - int idx = cfg->pooling_mode == EMBED_POOL_CLS ? start : end - 1; - const float *row = x + (size_t)idx * hidden; - if (norm_weight) { - float sum_sq = 0.0f; - for (int d = 0; d < hidden; d++) - sum_sq += row[d] * row[d]; - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); - for (int d = 0; d < hidden; d++) - emb[d] = row[d] * rms_inv * norm_weight[d]; - } else { - memcpy(emb, row, (size_t)hidden * sizeof(float)); - } - } else { - memset(emb, 0, (size_t)hidden * sizeof(float)); - for (int i = start; i < end; i++) { - const float *row = x + (size_t)i * hidden; - if (norm_weight) { - float sum_sq = 0.0f; - for (int d = 0; d < hidden; d++) - sum_sq += row[d] * row[d]; - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); - for (int d = 0; d < hidden; d++) - emb[d] += row[d] * rms_inv * norm_weight[d]; - } else { - for (int d = 0; d < hidden; d++) - emb[d] += row[d]; - } - } - float inv_len = 1.0f / (float)len; - for (int d = 0; d < hidden; d++) - emb[d] *= inv_len; - } - - if (cfg->normalize_embeddings && embed_l2_normalize(emb, hidden) != 0) - return -1; - } - return 0; -} - -/* ======================================================================== - * Public forward/embed APIs - * ======================================================================== */ - -int embed_model_forward_into(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_states) { - if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) - return -1; - - size_t count, bytes; - if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &count) != 0 || - mul_size(count, sizeof(float), &bytes) != 0) - return -1; - - embed_input_t input = {token_ids, n_tokens}; - int offsets[2] = {0, n_tokens}; - if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0) - return -1; - - memcpy(out_states, ws->x, bytes); - return 0; -} - -int embed_pool_spans(const embed_config_t *cfg, - const float *states, - int n_tokens, - const embed_span_t *spans, - int n_spans, - float *out_embeddings) { - if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings) - return -1; - - int hidden = cfg->hidden_size; - if (hidden <= 0) - return -1; - - for (int s = 0; s < n_spans; s++) { - int start = spans[s].start; - int len = spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) - return -1; - - float *emb = out_embeddings + (size_t)s * hidden; - memset(emb, 0, (size_t)hidden * sizeof(float)); - - for (int t = 0; t < len; t++) { - const float *row = states + (size_t)(start + t) * hidden; - for (int d = 0; d < hidden; d++) - emb[d] += row[d]; - } - - float inv_len = 1.0f / (float)len; - for (int d = 0; d < hidden; d++) { - emb[d] *= inv_len; - } - } - return 0; -} - -int embed_model_encode_spans_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_context_input_t *inputs, - int batch, - float *out_embeddings) { - if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) - return -1; - - if (ensure_offsets(ws, batch) != 0) - return -1; - if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t)) - return -1; - embed_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); - if (!packed_inputs) - return -1; - int *offsets = ws->offsets; - int total_seq = 0; - int max_seq = 0; - int total_spans = 0; - for (int b = 0; b < batch; b++) { - const embed_context_input_t *input = &inputs[b]; - int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || - total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) { - free(packed_inputs); - return -1; - } - packed_inputs[b] = input->input; - offsets[b] = total_seq; - total_seq += n_tokens; - total_spans += input->n_spans; - if (n_tokens > max_seq) - max_seq = n_tokens; - for (int s = 0; s < input->n_spans; s++) { - int start = input->spans[s].start; - int len = input->spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) { - free(packed_inputs); - return -1; - } - } - } - offsets[batch] = total_seq; - - if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, total_seq, max_seq, 1) != - 0) { - free(packed_inputs); - return -1; - } - free(packed_inputs); - - int span_offset = 0; - int hidden = model->config.hidden_size; - for (int b = 0; b < batch; b++) { - const embed_context_input_t *input = &inputs[b]; - if (embed_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden, - input->input.n_tokens, input->spans, input->n_spans, - out_embeddings + (size_t)span_offset * hidden) != 0) - return -1; - span_offset += input->n_spans; - } - return 0; -} - -int embed_model_encode_spans(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - const embed_span_t *spans, - int n_spans, - float *out_embeddings) { - embed_context_input_t input = { - .input = {token_ids, n_tokens}, - .spans = spans, - .n_spans = n_spans, - }; - return embed_model_encode_spans_batch(model, ws, &input, 1, out_embeddings); -} - -float *embed_model_forward(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens) { - if (!model || !ws || !token_ids || n_tokens <= 0) - return NULL; - - size_t n; - if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0) - return NULL; - float *out = malloc_floats(n); - if (!out) - return NULL; - - if (embed_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) { - free(out); - return NULL; - } - return out; -} - -int embed_model_encode_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, - int batch, - float *out_embeddings) { - if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) - return -1; - - if (ensure_offsets(ws, batch) != 0) - return -1; - int *offsets = ws->offsets; - - int total_seq = 0; - int max_seq = 0; - if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0) - return -1; - - int rc = model->config.family == EMBED_FAMILY_BERT - ? forward_packed_bert(model, ws, inputs, batch, offsets, total_seq, max_seq) - : forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); - if (rc == 0) - rc = pool_embeddings(model, ws, offsets, batch, out_embeddings); - - return rc; -} - -int embed_pool_batch(const embed_config_t *cfg, - const float *states, - const int *seq_lengths, - int batch, - float *out_embeddings) { - if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0) - return -1; - - int offset = 0; - for (int b = 0; b < batch; b++) { - int len = seq_lengths[b]; - if (len <= 0 || offset > INT_MAX - len) - return -1; - - float *emb = out_embeddings + (size_t)b * cfg->hidden_size; - if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) { - /* CLS takes the first token, last-token the final one; these states - * already carry the model's final RMSNorm. */ - int idx = cfg->pooling_mode == EMBED_POOL_CLS ? offset : offset + len - 1; - const float *row = states + (size_t)idx * cfg->hidden_size; - memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float)); - } else { - memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); - for (int t = 0; t < len; t++) { - const float *row = states + (size_t)(offset + t) * cfg->hidden_size; - for (int d = 0; d < cfg->hidden_size; d++) - emb[d] += row[d]; - } - float inv_len = 1.0f / (float)len; - for (int d = 0; d < cfg->hidden_size; d++) - emb[d] *= inv_len; - } - if (cfg->normalize_embeddings && embed_l2_normalize(emb, cfg->hidden_size) != 0) - return -1; - offset += len; - } - return 0; -} - -int embed_model_encode_into(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding) { - embed_input_t input = {token_ids, n_tokens}; - return embed_model_encode_batch(model, ws, &input, 1, out_embedding); -} - -float *embed_model_encode(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens) { - if (!model || !ws || !token_ids || n_tokens <= 0) - return NULL; - - int hidden = model->config.hidden_size; - if (hidden <= 0) - return NULL; - float *emb = malloc_floats((size_t)hidden); - if (!emb) - return NULL; - - if (embed_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) { - free(emb); - return NULL; - } - return emb; -} |
