diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 17:21:54 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 17:49:33 +0200 |
| commit | e54c07a3f54e7f5825bf27d1a9a6dbb5fb0baa8c (patch) | |
| tree | 8418dbc57529a6d8ae822b29f813effe6b6da231 | |
| parent | 3904a33e4416c35a48e8dd7c242a137a2cc17871 (diff) | |
| download | ffwd-e54c07a3f54e7f5825bf27d1a9a6dbb5fb0baa8c.tar.gz ffwd-e54c07a3f54e7f5825bf27d1a9a6dbb5fb0baa8c.zip | |
cpu: split embed.c into model.c and forward.c
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | src/forward.c (renamed from src/embed.c) | 647 | ||||
| -rw-r--r-- | src/late.c | 2 | ||||
| -rw-r--r-- | src/model.c | 647 | ||||
| -rw-r--r-- | src/model_internal.h (renamed from src/engine.h) | 25 |
5 files changed, 675 insertions, 650 deletions
@@ -78,8 +78,8 @@ LDFLAGS = -lm -lpthread $(BACKEND_LDFLAGS) # in one variable stops the per-test source lists from drifting apart. KERNEL_SRCS = src/kernels.c src/kernels_generic.c src/kernels_neon.c src/kernels_avx.c TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c -CORE_SRCS = src/embed.c src/alloc.c src/vec.c src/late.c src/config.c src/safetensors.c \ - $(KERNEL_SRCS) +CORE_SRCS = src/model.c src/forward.c src/late.c src/vec.c src/alloc.c \ + src/config.c src/safetensors.c $(KERNEL_SRCS) SRCS = $(CORE_SRCS) $(TOKENIZER_SRCS) \ deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c diff --git a/src/embed.c b/src/forward.c index baa9d11..d5818a4 100644 --- a/src/embed.c +++ b/src/forward.c @@ -1,11 +1,7 @@ #include "embed.h" -#include "engine.h" -#include "alloc.h" -#include "config.h" -#include "internal.h" +#include "model_internal.h" #include "dtype.h" #include "kernels.h" -#include "safetensors.h" #include <stdio.h> #include <stdlib.h> @@ -13,158 +9,15 @@ #include <math.h> #include <limits.h> -/* ======================================================================== - * Globals - * ======================================================================== */ - -int embed_verbose = 0; - +/* Below these packed-sequence lengths the fused BF16 row kernels beat widening + * the weight to F32 and dispatching through BLAS; above them, widen once. */ #define EMBED_BF16_QKV_FUSE_MAX_SEQ 16 #define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4 /* ======================================================================== - * Model-directory probes - * ======================================================================== */ - -static int model_dir_has_file(const char *model_dir, const char *rel) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) - return 0; - FILE *f = fopen(path, "rb"); - if (!f) - return 0; - fclose(f); - return 1; -} - -int model_dir_has_late_projection(const char *model_dir) { - return model_dir_has_file(model_dir, "1_Dense/config.json") && - model_dir_has_file(model_dir, "1_Dense/model.safetensors"); -} - -/* ======================================================================== - * Weight loading (direct mmap pointers into safetensors) + * Linear projections (weight-dtype dispatch) and workspace scratch growth * ======================================================================== */ -static size_t dtype_size(safetensor_dtype_t dtype) { - switch (dtype) { - case DTYPE_F32: - return sizeof(float); - case DTYPE_BF16: - return sizeof(uint16_t); - default: - return 0; - } -} - -static const char *dtype_name(safetensor_dtype_t dtype) { - switch (dtype) { - case DTYPE_F32: - return "F32"; - case DTYPE_F16: - return "F16"; - case DTYPE_BF16: - return "BF16"; - case DTYPE_I32: - return "I32"; - case DTYPE_I64: - return "I64"; - case DTYPE_BOOL: - return "BOOL"; - default: - return "UNKNOWN"; - } -} - -int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, - const int64_t *shape, - int ndim, - int bf16_ok) { - if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name, - dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (t->shape[i] != shape[i]) { - fprintf(stderr, - "embed: bad shape for %s at dim %d: " - "got %lld, expected %lld\n", - name, i, (long long)t->shape[i], (long long)shape[i]); - return 0; - } - if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "embed: shape too large for %s\n", name); - return 0; - } - numel *= (size_t)shape[i]; - } - - size_t elem_size = dtype_size(t->dtype); - if (elem_size == 0) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype)); - return 0; - } - - size_t bytes; - if (mul_size(numel, elem_size, &bytes) != 0) { - fprintf(stderr, "embed: tensor data too large for %s\n", name); - return 0; - } - if (t->data_size != bytes) { - fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, - bytes); - return 0; - } - - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "embed: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static embed_weight_ref_t -load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { - embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL}; - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "embed: tensor not found: %s\n", name); - return ref; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) - return ref; - - ref.dtype = t->dtype; - ref.data = safetensors_data(sf, t); - return ref; -} - -static float * -load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "embed: tensor not found: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) - return NULL; - return safetensors_get_f32(sf, t); -} - static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { for (int i = 0; i < n; i++) dst[i] = embed_bf16_to_f32(src[i]); @@ -272,495 +125,11 @@ static void linear_pair_weight(embed_workspace_t *ws, } /* ======================================================================== - * Working buffer management - * ======================================================================== */ - -static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) { - if (seq <= ws->buf_seq_cap) - return 0; - - int cap; - if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) - return -1; - -#define R(ptr, n) \ - do { \ - if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ - return -1; \ - } while (0) - - R(ws->x, c->hidden_size); - R(ws->x_norm, c->hidden_size); - R(ws->q, c->q_dim); - R(ws->k, c->kv_dim); - R(ws->v, c->kv_dim); - R(ws->attn_out, c->q_dim); - R(ws->proj_out, c->hidden_size); - R(ws->ffn_gate, c->intermediate_size); - R(ws->ffn_up, c->intermediate_size); - -#undef R - - ws->buf_seq_cap = cap; - return 0; -} - -static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) { - size_t bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (bytes <= ws->attn_scores_bytes) - return; - - void *p = realloc(ws->attn_scores, bytes); - if (!p) - return; - ws->attn_scores = (float *)p; - ws->attn_scores_bytes = bytes; -} - -/* ======================================================================== - * RoPE cache - * ======================================================================== */ - -static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) { - if (n_pos <= ws->rope_cache_cap) - return 0; - - int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; - while (cap < n_pos) { - if (cap > INT_MAX / 2) - return -1; - cap *= 2; - } - - int head_dim = cfg->head_dim; - int half = head_dim / 2; - float theta = cfg->rope_theta; - - size_t count; - if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) - return -1; - if (realloc_floats(&ws->rope_cos, count) != 0) - return -1; - if (realloc_floats(&ws->rope_sin, count) != 0) - return -1; - - for (int pos = ws->rope_cache_cap; pos < cap; pos++) { - float fp = (float)pos; - float *c = ws->rope_cos + (size_t)pos * head_dim; - float *s = ws->rope_sin + (size_t)pos * head_dim; - for (int d = 0; d < half; d++) { - float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); - float angle = fp * inv; - float cv = cosf(angle), sv = sinf(angle); - c[d] = cv; - c[half + d] = cv; - s[d] = sv; - s[half + d] = sv; - } - } - - ws->rope_cache_cap = cap; - return 0; -} - -static int ensure_offsets(embed_workspace_t *ws, int batch) { - if (!ws || batch <= 0 || batch == INT_MAX) - return -1; - - int needed = batch + 1; - if (needed <= ws->offsets_cap) - return 0; - - if (realloc_ints(&ws->offsets, (size_t)needed) != 0) - return -1; - ws->offsets_cap = needed; - return 0; -} - -/* ======================================================================== - * Model / workspace load and free - * ======================================================================== */ - -/* BERT-family weight loader. The tensor names differ entirely from the Qwen - * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry - * point. Reused embed_layer_t slots carry their BERT meaning: wo is the - * attention output dense, input_norm/post_attn_norm are the two block LayerNorm - * weights, up_proj/down_proj are the two FFN matrices (see embed_internal.h). */ -static int -load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { - const embed_config_t *cfg = &model->config; - embed_weights_t *w = &model->weights; - char name[256]; - - /* sentence-transformers checkpoints store the BERT submodule unprefixed; - * raw HF checkpoints prefix every tensor with "bert.". */ - const char *p; - if (multi_safetensors_find(ms, "embeddings.word_embeddings.weight", NULL)) - p = ""; - else if (multi_safetensors_find(ms, "bert.embeddings.word_embeddings.weight", NULL)) - p = "bert."; - else { - fprintf(stderr, "embed_model_load: BERT word embeddings not found\n"); - return -1; - } - - int hidden = cfg->hidden_size, inter = cfg->intermediate_size; - - if (layer_start == 0) { - const int64_t emb_shape[2] = {cfg->vocab_size, hidden}; - snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p); - w->embed_tokens = load_weight_direct(ms, name, emb_shape, 2); - const int64_t pos_shape[2] = {cfg->max_position_embeddings, hidden}; - snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p); - w->position_embeddings = load_norm_f32(ms, name, pos_shape, 2); - const int64_t type_shape[2] = {2, hidden}; - snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p); - w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2); - const int64_t ln_shape[1] = {hidden}; - snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p); - w->embed_ln_w = load_norm_f32(ms, name, ln_shape, 1); - snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); - w->embed_ln_b = load_norm_f32(ms, name, ln_shape, 1); - if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || - !w->embed_ln_w || !w->embed_ln_b) - return -1; - } - - w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); - if (!w->layers) - return -1; - - for (int i = layer_start; i < layer_end; i++) { - embed_layer_t *l = &w->layers[i]; - -#define BERT_W(field, suffix, d0, d1) \ - do { \ - const int64_t e[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ - l->field = load_weight_direct(ms, name, e, 2); \ - if (!l->field.data) \ - return -1; \ - } while (0) -#define BERT_V(field, suffix, d0) \ - do { \ - const int64_t e[1] = {(d0)}; \ - snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ - l->field = load_norm_f32(ms, name, e, 1); \ - if (!l->field) \ - return -1; \ - } while (0) - - BERT_W(wq, "attention.self.query.weight", hidden, hidden); - BERT_V(q_bias, "attention.self.query.bias", hidden); - BERT_W(wk, "attention.self.key.weight", hidden, hidden); - BERT_V(k_bias, "attention.self.key.bias", hidden); - BERT_W(wv, "attention.self.value.weight", hidden, hidden); - BERT_V(v_bias, "attention.self.value.bias", hidden); - BERT_W(wo, "attention.output.dense.weight", hidden, hidden); - BERT_V(o_bias, "attention.output.dense.bias", hidden); - BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); - BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); - BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); - BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); - BERT_W(down_proj, "output.dense.weight", hidden, inter); - BERT_V(ffn_out_bias, "output.dense.bias", hidden); - BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); - BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); - -#undef BERT_W -#undef BERT_V - } - /* BERT has no final norm: w->norm stays NULL, and pooling skips it. */ - return 0; -} - -embed_model_t * -model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) { - if (!allow_late && model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "embed_model_load: late-interaction models require " - "token-level MaxSim support and are not valid pooled " - "embedding models yet\n"); - return NULL; - } - - embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t)); - if (!model) - return NULL; - - /* Parse config.json */ - if (embed_config_parse(&model->config, model_dir) != 0) { - free(model); - return NULL; - } - - const embed_config_t *cfg = &model->config; - if (layer_end == -1) - layer_end = cfg->n_layers; - if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) { - fprintf(stderr, - "embed_model_load: invalid layer range [%d, %d) " - "for %d-layer model\n", - layer_start, layer_end, cfg->n_layers); - free(model); - return NULL; - } - model->layer_start = layer_start; - model->layer_end = layer_end; - model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL - ? embed_causal_gqa_attention_packed_with_scratch - : embed_bidirectional_gqa_attention_packed_with_scratch; - - /* Open safetensors (handles single file or multi-shard) */ - multi_safetensors_t *ms = multi_safetensors_open(model_dir); - if (!ms) { - fprintf(stderr, "embed_model_load: failed to open safetensors in %s\n", model_dir); - free(model); - return NULL; - } - model->safetensors = ms; - - if (cfg->family == EMBED_FAMILY_BERT) { - if (load_bert_weights(model, ms, layer_start, layer_end) != 0) - goto fail; - if (embed_verbose >= 1) - fprintf(stderr, "embed_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start, - layer_end, cfg->hidden_size); - return model; - } - - const char *weight_prefix = multi_safetensors_weight_prefix(ms, "embed_tokens.weight"); - if (!weight_prefix) { - fprintf(stderr, "embed_model_load: neither embed_tokens.weight nor " - "model.embed_tokens.weight was found\n"); - goto fail; - } - - if (embed_verbose >= 1) - fprintf(stderr, "embed_model_load: loading weights from %s\n", model_dir); - - char name[256]; - - /* Only the first stage needs token embeddings. */ - embed_weights_t *w = &model->weights; - if (layer_start == 0) { - const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size}; - snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix); - w->embed_tokens = load_weight_direct(ms, name, embed_shape, 2); - if (!w->embed_tokens.data) - goto fail; - } - - /* Allocate layer array */ - w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); - if (!w->layers) - goto fail; - - /* Per-layer weights */ - for (int i = layer_start; i < layer_end; i++) { - embed_layer_t *l = &w->layers[i]; - -#define LOAD_NORM(field, fmt, d0) \ - do { \ - const int64_t expect[1] = {(d0)}; \ - snprintf(name, sizeof(name), fmt, weight_prefix, i); \ - l->field = load_norm_f32(ms, name, expect, 1); \ - if (!l->field) \ - goto fail; \ - } while (0) - -#define LOAD2(field, fmt, d0, d1) \ - do { \ - const int64_t expect[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), fmt, weight_prefix, i); \ - l->field = load_weight_direct(ms, name, expect, 2); \ - if (!l->field.data) \ - goto fail; \ - } while (0) - - LOAD2(wq, "%slayers.%d.self_attn.q_proj.weight", cfg->q_dim, cfg->hidden_size); - LOAD2(wk, "%slayers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size); - LOAD2(wv, "%slayers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size); - LOAD2(wo, "%slayers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim); - if (cfg->qkv_bias) { - LOAD_NORM(q_bias, "%slayers.%d.self_attn.q_proj.bias", cfg->q_dim); - LOAD_NORM(k_bias, "%slayers.%d.self_attn.k_proj.bias", cfg->kv_dim); - LOAD_NORM(v_bias, "%slayers.%d.self_attn.v_proj.bias", cfg->kv_dim); - } - if (cfg->qk_norm) { - LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim); - LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim); - } - LOAD_NORM(input_norm, "%slayers.%d.input_layernorm.weight", cfg->hidden_size); - LOAD_NORM(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", cfg->hidden_size); - LOAD2(gate_proj, "%slayers.%d.mlp.gate_proj.weight", cfg->intermediate_size, - cfg->hidden_size); - LOAD2(up_proj, "%slayers.%d.mlp.up_proj.weight", cfg->intermediate_size, cfg->hidden_size); - LOAD2(down_proj, "%slayers.%d.mlp.down_proj.weight", cfg->hidden_size, - cfg->intermediate_size); - -#undef LOAD2 -#undef LOAD_NORM - - if (embed_verbose >= 2) - fprintf(stderr, " layer %d loaded\n", i); - } - - /* Only the final stage applies the output RMSNorm. */ - if (layer_end == cfg->n_layers) { - const int64_t norm_shape[1] = {cfg->hidden_size}; - snprintf(name, sizeof(name), "%snorm.weight", weight_prefix); - w->norm = load_norm_f32(ms, name, norm_shape, 1); - if (!w->norm) - goto fail; - } - - if (embed_verbose >= 1) - fprintf(stderr, - "embed_model_load: layers [%d, %d) loaded " - "(%d-dim embeddings)\n", - layer_start, layer_end, cfg->hidden_size); - - return model; - -fail: - embed_model_free(model); - return NULL; -} - -embed_model_t *embed_model_load(const char *model_dir) { - return model_load_range_ex(model_dir, 0, -1, 0); -} - -void embed_model_free(embed_model_t *model) { - if (!model) - return; - if (model->weights.layers) { - for (int i = 0; i < model->config.n_layers; i++) { - embed_layer_t *l = &model->weights.layers[i]; - free((void *)l->q_bias); - free((void *)l->k_bias); - free((void *)l->v_bias); - free((void *)l->q_norm); - free((void *)l->k_norm); - free((void *)l->input_norm); - free((void *)l->post_attn_norm); - free((void *)l->o_bias); - free((void *)l->attn_ln_bias); - free((void *)l->ffn_inter_bias); - free((void *)l->ffn_out_bias); - free((void *)l->ffn_ln_bias); - } - } - free((void *)model->weights.norm); - free((void *)model->weights.position_embeddings); - free((void *)model->weights.token_type_embeddings); - free((void *)model->weights.embed_ln_w); - free((void *)model->weights.embed_ln_b); - if (model->safetensors) - multi_safetensors_close((multi_safetensors_t *)model->safetensors); - free(model->weights.layers); - free(model); -} - -embed_workspace_t *embed_workspace_new(const embed_model_t *model) { - if (!model) - return NULL; - embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t)); - if (!ws) - return NULL; - ws->model = model; - return ws; -} - -void embed_workspace_free(embed_workspace_t *ws) { - if (!ws) - return; - free(ws->x); - free(ws->x_norm); - free(ws->q); - free(ws->k); - free(ws->v); - free(ws->attn_out); - free(ws->proj_out); - free(ws->ffn_gate); - free(ws->ffn_up); - free(ws->attn_scores); - free(ws->offsets); - free(ws->rope_cos); - free(ws->rope_sin); - free(ws->bf16_widen); - free(ws); -} - -const embed_config_t *embed_model_config(const embed_model_t *model) { - return model ? &model->config : NULL; -} - -size_t embed_workspace_nbytes(const embed_workspace_t *ws) { - if (!ws) - return 0; - - size_t total = sizeof(*ws); - const embed_model_t *model = ws->model; - if (!model) - return total; - - const embed_config_t *c = &model->config; - size_t row_floats = 0; - const int dims[] = { - c->hidden_size, /* x */ - c->hidden_size, /* x_norm */ - c->q_dim, /* q */ - c->kv_dim, /* k */ - c->kv_dim, /* v */ - c->q_dim, /* attn_out */ - c->hidden_size, /* proj_out */ - c->intermediate_size, /* ffn_gate */ - c->intermediate_size, /* ffn_up */ - }; - - for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { - if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0) - return SIZE_MAX; - } - - size_t buf_floats = 0; - if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) - return SIZE_MAX; - - size_t rope_floats = 0; - if (ws->rope_cache_cap > 0) { - size_t rope_rows = 0; - if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 || - mul_size(rope_rows, 2, &rope_floats) != 0) - return SIZE_MAX; - } - - size_t all_floats = 0, bytes = 0; - if (add_size(buf_floats, rope_floats, &all_floats) != 0 || - mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0) - return SIZE_MAX; - - size_t offset_bytes = 0; - if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) - return SIZE_MAX; - if (add_size(total, offset_bytes, &total) != 0) - return SIZE_MAX; - if (add_size(total, ws->attn_scores_bytes, &total) != 0) - return SIZE_MAX; - - size_t widen_bytes = 0; - if (ws->bf16_widen_count > 0 && - mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0) - return SIZE_MAX; - if (add_size(total, widen_bytes, &total) != 0) - return SIZE_MAX; - - return total; -} - -/* ======================================================================== * Packed forward pass + * + * The workspace-growth helpers these call (ensure_buffers, ensure_rope_cache, + * ensure_attention_scores, ensure_offsets) live in model.c beside the workspace + * lifecycle and embed_workspace_nbytes, whose size accounting mirrors them. * ======================================================================== */ int build_offsets(const embed_input_t *inputs, @@ -1,5 +1,5 @@ #include "embed.h" -#include "engine.h" +#include "model_internal.h" #include "kernels.h" #include <stdio.h> diff --git a/src/model.c b/src/model.c new file mode 100644 index 0000000..095ce63 --- /dev/null +++ b/src/model.c @@ -0,0 +1,647 @@ +#include "embed.h" +#include "model_internal.h" +#include "config.h" +#include "kernels.h" + +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <math.h> +#include <limits.h> + +/* ======================================================================== + * Globals + * ======================================================================== */ + +int embed_verbose = 0; + +/* ======================================================================== + * Model-directory probes + * ======================================================================== */ + +static int model_dir_has_file(const char *model_dir, const char *rel) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; + FILE *f = fopen(path, "rb"); + if (!f) + return 0; + fclose(f); + return 1; +} + +int model_dir_has_late_projection(const char *model_dir) { + return model_dir_has_file(model_dir, "1_Dense/config.json") && + model_dir_has_file(model_dir, "1_Dense/model.safetensors"); +} + +/* ======================================================================== + * Weight tensor loading and validation (direct mmap pointers into safetensors) + * ======================================================================== */ + +static size_t dtype_size(safetensor_dtype_t dtype) { + switch (dtype) { + case DTYPE_F32: + return sizeof(float); + case DTYPE_BF16: + return sizeof(uint16_t); + default: + return 0; + } +} + +static const char *dtype_name(safetensor_dtype_t dtype) { + switch (dtype) { + case DTYPE_F32: + return "F32"; + case DTYPE_F16: + return "F16"; + case DTYPE_BF16: + return "BF16"; + case DTYPE_I32: + return "I32"; + case DTYPE_I64: + return "I64"; + case DTYPE_BOOL: + return "BOOL"; + default: + return "UNKNOWN"; + } +} + +int tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, + const char *name, + const int64_t *shape, + int ndim, + int bf16_ok) { + if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) { + fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name, + dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); + return 0; + } + if (t->ndim != ndim) { + fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); + return 0; + } + + size_t numel = 1; + for (int i = 0; i < ndim; i++) { + if (t->shape[i] != shape[i]) { + fprintf(stderr, + "embed: bad shape for %s at dim %d: " + "got %lld, expected %lld\n", + name, i, (long long)t->shape[i], (long long)shape[i]); + return 0; + } + if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { + fprintf(stderr, "embed: shape too large for %s\n", name); + return 0; + } + numel *= (size_t)shape[i]; + } + + size_t elem_size = dtype_size(t->dtype); + if (elem_size == 0) { + fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype)); + return 0; + } + + size_t bytes; + if (mul_size(numel, elem_size, &bytes) != 0) { + fprintf(stderr, "embed: tensor data too large for %s\n", name); + return 0; + } + if (t->data_size != bytes) { + fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, + bytes); + return 0; + } + + size_t data_start = 8 + sf->header_size; + if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || + t->data_size > sf->file_size - data_start - t->data_offset) { + fprintf(stderr, "embed: tensor data out of bounds for %s\n", name); + return 0; + } + return 1; +} + +static embed_weight_ref_t +load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { + embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL}; + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "embed: tensor not found: %s\n", name); + return ref; + } + if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) + return ref; + + ref.dtype = t->dtype; + ref.data = safetensors_data(sf, t); + return ref; +} + +static float * +load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "embed: tensor not found: %s\n", name); + return NULL; + } + if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) + return NULL; + return safetensors_get_f32(sf, t); +} + +/* ======================================================================== + * Model / workspace load and free + * ======================================================================== */ + +/* BERT-family weight loader. The tensor names differ entirely from the Qwen + * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry + * point. Reused embed_layer_t slots carry their BERT meaning: wo is the + * attention output dense, input_norm/post_attn_norm are the two block LayerNorm + * weights, up_proj/down_proj are the two FFN matrices (see embed_internal.h). */ +static int +load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { + const embed_config_t *cfg = &model->config; + embed_weights_t *w = &model->weights; + char name[256]; + + /* sentence-transformers checkpoints store the BERT submodule unprefixed; + * raw HF checkpoints prefix every tensor with "bert.". */ + const char *p; + if (multi_safetensors_find(ms, "embeddings.word_embeddings.weight", NULL)) + p = ""; + else if (multi_safetensors_find(ms, "bert.embeddings.word_embeddings.weight", NULL)) + p = "bert."; + else { + fprintf(stderr, "embed_model_load: BERT word embeddings not found\n"); + return -1; + } + + int hidden = cfg->hidden_size, inter = cfg->intermediate_size; + + if (layer_start == 0) { + const int64_t emb_shape[2] = {cfg->vocab_size, hidden}; + snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p); + w->embed_tokens = load_weight_direct(ms, name, emb_shape, 2); + const int64_t pos_shape[2] = {cfg->max_position_embeddings, hidden}; + snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p); + w->position_embeddings = load_norm_f32(ms, name, pos_shape, 2); + const int64_t type_shape[2] = {2, hidden}; + snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p); + w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2); + const int64_t ln_shape[1] = {hidden}; + snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p); + w->embed_ln_w = load_norm_f32(ms, name, ln_shape, 1); + snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); + w->embed_ln_b = load_norm_f32(ms, name, ln_shape, 1); + if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || + !w->embed_ln_w || !w->embed_ln_b) + return -1; + } + + w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); + if (!w->layers) + return -1; + + for (int i = layer_start; i < layer_end; i++) { + embed_layer_t *l = &w->layers[i]; + +#define BERT_W(field, suffix, d0, d1) \ + do { \ + const int64_t e[2] = {(d0), (d1)}; \ + snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ + l->field = load_weight_direct(ms, name, e, 2); \ + if (!l->field.data) \ + return -1; \ + } while (0) +#define BERT_V(field, suffix, d0) \ + do { \ + const int64_t e[1] = {(d0)}; \ + snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ + l->field = load_norm_f32(ms, name, e, 1); \ + if (!l->field) \ + return -1; \ + } while (0) + + BERT_W(wq, "attention.self.query.weight", hidden, hidden); + BERT_V(q_bias, "attention.self.query.bias", hidden); + BERT_W(wk, "attention.self.key.weight", hidden, hidden); + BERT_V(k_bias, "attention.self.key.bias", hidden); + BERT_W(wv, "attention.self.value.weight", hidden, hidden); + BERT_V(v_bias, "attention.self.value.bias", hidden); + BERT_W(wo, "attention.output.dense.weight", hidden, hidden); + BERT_V(o_bias, "attention.output.dense.bias", hidden); + BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); + BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); + BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); + BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); + BERT_W(down_proj, "output.dense.weight", hidden, inter); + BERT_V(ffn_out_bias, "output.dense.bias", hidden); + BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); + BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); + +#undef BERT_W +#undef BERT_V + } + /* BERT has no final norm: w->norm stays NULL, and pooling skips it. */ + return 0; +} + +embed_model_t * +model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) { + if (!allow_late && model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "embed_model_load: late-interaction models require " + "token-level MaxSim support and are not valid pooled " + "embedding models yet\n"); + return NULL; + } + + embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t)); + if (!model) + return NULL; + + /* Parse config.json */ + if (embed_config_parse(&model->config, model_dir) != 0) { + free(model); + return NULL; + } + + const embed_config_t *cfg = &model->config; + if (layer_end == -1) + layer_end = cfg->n_layers; + if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) { + fprintf(stderr, + "embed_model_load: invalid layer range [%d, %d) " + "for %d-layer model\n", + layer_start, layer_end, cfg->n_layers); + free(model); + return NULL; + } + model->layer_start = layer_start; + model->layer_end = layer_end; + model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL + ? embed_causal_gqa_attention_packed_with_scratch + : embed_bidirectional_gqa_attention_packed_with_scratch; + + /* Open safetensors (handles single file or multi-shard) */ + multi_safetensors_t *ms = multi_safetensors_open(model_dir); + if (!ms) { + fprintf(stderr, "embed_model_load: failed to open safetensors in %s\n", model_dir); + free(model); + return NULL; + } + model->safetensors = ms; + + if (cfg->family == EMBED_FAMILY_BERT) { + if (load_bert_weights(model, ms, layer_start, layer_end) != 0) + goto fail; + if (embed_verbose >= 1) + fprintf(stderr, "embed_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start, + layer_end, cfg->hidden_size); + return model; + } + + const char *weight_prefix = multi_safetensors_weight_prefix(ms, "embed_tokens.weight"); + if (!weight_prefix) { + fprintf(stderr, "embed_model_load: neither embed_tokens.weight nor " + "model.embed_tokens.weight was found\n"); + goto fail; + } + + if (embed_verbose >= 1) + fprintf(stderr, "embed_model_load: loading weights from %s\n", model_dir); + + char name[256]; + + /* Only the first stage needs token embeddings. */ + embed_weights_t *w = &model->weights; + if (layer_start == 0) { + const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size}; + snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix); + w->embed_tokens = load_weight_direct(ms, name, embed_shape, 2); + if (!w->embed_tokens.data) + goto fail; + } + + /* Allocate layer array */ + w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); + if (!w->layers) + goto fail; + + /* Per-layer weights */ + for (int i = layer_start; i < layer_end; i++) { + embed_layer_t *l = &w->layers[i]; + +#define LOAD_NORM(field, fmt, d0) \ + do { \ + const int64_t expect[1] = {(d0)}; \ + snprintf(name, sizeof(name), fmt, weight_prefix, i); \ + l->field = load_norm_f32(ms, name, expect, 1); \ + if (!l->field) \ + goto fail; \ + } while (0) + +#define LOAD2(field, fmt, d0, d1) \ + do { \ + const int64_t expect[2] = {(d0), (d1)}; \ + snprintf(name, sizeof(name), fmt, weight_prefix, i); \ + l->field = load_weight_direct(ms, name, expect, 2); \ + if (!l->field.data) \ + goto fail; \ + } while (0) + + LOAD2(wq, "%slayers.%d.self_attn.q_proj.weight", cfg->q_dim, cfg->hidden_size); + LOAD2(wk, "%slayers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size); + LOAD2(wv, "%slayers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size); + LOAD2(wo, "%slayers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim); + if (cfg->qkv_bias) { + LOAD_NORM(q_bias, "%slayers.%d.self_attn.q_proj.bias", cfg->q_dim); + LOAD_NORM(k_bias, "%slayers.%d.self_attn.k_proj.bias", cfg->kv_dim); + LOAD_NORM(v_bias, "%slayers.%d.self_attn.v_proj.bias", cfg->kv_dim); + } + if (cfg->qk_norm) { + LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim); + LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim); + } + LOAD_NORM(input_norm, "%slayers.%d.input_layernorm.weight", cfg->hidden_size); + LOAD_NORM(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", cfg->hidden_size); + LOAD2(gate_proj, "%slayers.%d.mlp.gate_proj.weight", cfg->intermediate_size, + cfg->hidden_size); + LOAD2(up_proj, "%slayers.%d.mlp.up_proj.weight", cfg->intermediate_size, cfg->hidden_size); + LOAD2(down_proj, "%slayers.%d.mlp.down_proj.weight", cfg->hidden_size, + cfg->intermediate_size); + +#undef LOAD2 +#undef LOAD_NORM + + if (embed_verbose >= 2) + fprintf(stderr, " layer %d loaded\n", i); + } + + /* Only the final stage applies the output RMSNorm. */ + if (layer_end == cfg->n_layers) { + const int64_t norm_shape[1] = {cfg->hidden_size}; + snprintf(name, sizeof(name), "%snorm.weight", weight_prefix); + w->norm = load_norm_f32(ms, name, norm_shape, 1); + if (!w->norm) + goto fail; + } + + if (embed_verbose >= 1) + fprintf(stderr, + "embed_model_load: layers [%d, %d) loaded " + "(%d-dim embeddings)\n", + layer_start, layer_end, cfg->hidden_size); + + return model; + +fail: + embed_model_free(model); + return NULL; +} + +embed_model_t *embed_model_load(const char *model_dir) { + return model_load_range_ex(model_dir, 0, -1, 0); +} + +void embed_model_free(embed_model_t *model) { + if (!model) + return; + if (model->weights.layers) { + for (int i = 0; i < model->config.n_layers; i++) { + embed_layer_t *l = &model->weights.layers[i]; + free((void *)l->q_bias); + free((void *)l->k_bias); + free((void *)l->v_bias); + free((void *)l->q_norm); + free((void *)l->k_norm); + free((void *)l->input_norm); + free((void *)l->post_attn_norm); + free((void *)l->o_bias); + free((void *)l->attn_ln_bias); + free((void *)l->ffn_inter_bias); + free((void *)l->ffn_out_bias); + free((void *)l->ffn_ln_bias); + } + } + free((void *)model->weights.norm); + free((void *)model->weights.position_embeddings); + free((void *)model->weights.token_type_embeddings); + free((void *)model->weights.embed_ln_w); + free((void *)model->weights.embed_ln_b); + if (model->safetensors) + multi_safetensors_close((multi_safetensors_t *)model->safetensors); + free(model->weights.layers); + free(model); +} + +embed_workspace_t *embed_workspace_new(const embed_model_t *model) { + if (!model) + return NULL; + embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t)); + if (!ws) + return NULL; + ws->model = model; + return ws; +} + +void embed_workspace_free(embed_workspace_t *ws) { + if (!ws) + return; + free(ws->x); + free(ws->x_norm); + free(ws->q); + free(ws->k); + free(ws->v); + free(ws->attn_out); + free(ws->proj_out); + free(ws->ffn_gate); + free(ws->ffn_up); + free(ws->attn_scores); + free(ws->offsets); + free(ws->rope_cos); + free(ws->rope_sin); + free(ws->bf16_widen); + free(ws); +} + +const embed_config_t *embed_model_config(const embed_model_t *model) { + return model ? &model->config : NULL; +} + +/* ======================================================================== + * Workspace buffer growth + * + * Lazy per-call growth of the forward-pass scratch. Lives here, beside the + * workspace lifecycle and embed_workspace_nbytes below: nbytes reproduces these + * allocation sizes, so the allocator and the accountant stay in one file. + * ======================================================================== */ + +int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) { + if (seq <= ws->buf_seq_cap) + return 0; + + int cap; + if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) + return -1; + +#define R(ptr, n) \ + do { \ + if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ + return -1; \ + } while (0) + + R(ws->x, c->hidden_size); + R(ws->x_norm, c->hidden_size); + R(ws->q, c->q_dim); + R(ws->k, c->kv_dim); + R(ws->v, c->kv_dim); + R(ws->attn_out, c->q_dim); + R(ws->proj_out, c->hidden_size); + R(ws->ffn_gate, c->intermediate_size); + R(ws->ffn_up, c->intermediate_size); + +#undef R + + ws->buf_seq_cap = cap; + return 0; +} + +void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) { + size_t bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (bytes <= ws->attn_scores_bytes) + return; + + void *p = realloc(ws->attn_scores, bytes); + if (!p) + return; + ws->attn_scores = (float *)p; + ws->attn_scores_bytes = bytes; +} + +int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) { + if (n_pos <= ws->rope_cache_cap) + return 0; + + int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; + while (cap < n_pos) { + if (cap > INT_MAX / 2) + return -1; + cap *= 2; + } + + int head_dim = cfg->head_dim; + int half = head_dim / 2; + float theta = cfg->rope_theta; + + size_t count; + if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) + return -1; + if (realloc_floats(&ws->rope_cos, count) != 0) + return -1; + if (realloc_floats(&ws->rope_sin, count) != 0) + return -1; + + for (int pos = ws->rope_cache_cap; pos < cap; pos++) { + float fp = (float)pos; + float *c = ws->rope_cos + (size_t)pos * head_dim; + float *s = ws->rope_sin + (size_t)pos * head_dim; + for (int d = 0; d < half; d++) { + float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = fp * inv; + float cv = cosf(angle), sv = sinf(angle); + c[d] = cv; + c[half + d] = cv; + s[d] = sv; + s[half + d] = sv; + } + } + + ws->rope_cache_cap = cap; + return 0; +} + +int ensure_offsets(embed_workspace_t *ws, int batch) { + if (!ws || batch <= 0 || batch == INT_MAX) + return -1; + + int needed = batch + 1; + if (needed <= ws->offsets_cap) + return 0; + + if (realloc_ints(&ws->offsets, (size_t)needed) != 0) + return -1; + ws->offsets_cap = needed; + return 0; +} + +size_t embed_workspace_nbytes(const embed_workspace_t *ws) { + if (!ws) + return 0; + + size_t total = sizeof(*ws); + const embed_model_t *model = ws->model; + if (!model) + return total; + + const embed_config_t *c = &model->config; + size_t row_floats = 0; + const int dims[] = { + c->hidden_size, /* x */ + c->hidden_size, /* x_norm */ + c->q_dim, /* q */ + c->kv_dim, /* k */ + c->kv_dim, /* v */ + c->q_dim, /* attn_out */ + c->hidden_size, /* proj_out */ + c->intermediate_size, /* ffn_gate */ + c->intermediate_size, /* ffn_up */ + }; + + for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { + if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0) + return SIZE_MAX; + } + + size_t buf_floats = 0; + if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) + return SIZE_MAX; + + size_t rope_floats = 0; + if (ws->rope_cache_cap > 0) { + size_t rope_rows = 0; + if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 || + mul_size(rope_rows, 2, &rope_floats) != 0) + return SIZE_MAX; + } + + size_t all_floats = 0, bytes = 0; + if (add_size(buf_floats, rope_floats, &all_floats) != 0 || + mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0) + return SIZE_MAX; + + size_t offset_bytes = 0; + if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) + return SIZE_MAX; + if (add_size(total, offset_bytes, &total) != 0) + return SIZE_MAX; + if (add_size(total, ws->attn_scores_bytes, &total) != 0) + return SIZE_MAX; + + size_t widen_bytes = 0; + if (ws->bf16_widen_count > 0 && + mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0) + return SIZE_MAX; + if (add_size(total, widen_bytes, &total) != 0) + return SIZE_MAX; + + return total; +} diff --git a/src/engine.h b/src/model_internal.h index abdb0c7..f61b069 100644 --- a/src/engine.h +++ b/src/model_internal.h @@ -1,14 +1,15 @@ -#ifndef EMBED_ENGINE_H -#define EMBED_ENGINE_H +#ifndef EMBED_MODEL_INTERNAL_H +#define EMBED_MODEL_INTERNAL_H #include "internal.h" #include "alloc.h" -/* Cross-file declarations internal to the CPU engine. These functions are not - * part of the public API (embed.h) and carry no EMBED_API visibility; they are - * shared between the engine translation units (model loading, the forward pass, - * and late interaction) that used to live in one file. The frontends and the - * GPU backends do not include this header. */ +/* Cross-file function declarations private to the CPU model code. The companion + * to internal.h: that header holds the shared struct definitions, this one the + * shared functions. None are part of the public API (embed.h) and none carry + * EMBED_API visibility; they are shared between the translation units (model + * loading, the forward pass, and late interaction) that used to live in one + * file. The frontends and the GPU backends do not include this header. */ /* Validate that a safetensor matches an expected dtype/rank/shape and that its * bytes lie within the mapped file. bf16_ok permits BF16 in addition to F32. */ @@ -27,6 +28,14 @@ int model_dir_has_late_projection(const char *model_dir); embed_model_t * model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late); +/* Workspace scratch growth, lazy per forward call. Defined in model.c beside the + * workspace lifecycle (embed_workspace_nbytes mirrors their sizing); called from + * the forward pass in forward.c. */ +int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq); +void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch); +int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos); +int ensure_offsets(embed_workspace_t *ws, int batch); + /* y[seq,out] = x[seq,in] @ w^T, no bias; dispatches F32/BF16 paths. Shared by * the forward pass and the late projection. */ void linear_nobias_weight(embed_workspace_t *ws, @@ -57,4 +66,4 @@ int forward_packed_inplace(const embed_model_t *model, int max_seq, int apply_final_norm); -#endif /* EMBED_ENGINE_H */ +#endif /* EMBED_MODEL_INTERNAL_H */ |
