diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:05:44 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:05:44 +0200 |
| commit | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (patch) | |
| tree | f666589acfd68e76bbde4e74eead44ae7819d0bb /libffwd/forward.c | |
| parent | 91871be6ece9317c5d49353f6474583e7e28168d (diff) | |
| download | langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.tar.gz langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.zip | |
Remove unused model families and inference paths
Diffstat (limited to 'libffwd/forward.c')
| -rw-r--r-- | libffwd/forward.c | 717 |
1 files changed, 46 insertions, 671 deletions
diff --git a/libffwd/forward.c b/libffwd/forward.c index d5d2442..b314642 100644 --- a/libffwd/forward.c +++ b/libffwd/forward.c @@ -1,98 +1,12 @@ -#include "ffwd.h" #include "model_internal.h" -#include "dtype.h" #include "kernels.h" +#include <limits.h> +#include <math.h> #include <stdio.h> -#include <stdlib.h> #include <string.h> -#include <math.h> -#include <limits.h> - -/* For packed sequences below these lengths, fused BF16 row kernels beat - * widening the weight to F32 and using BLAS. For longer packed sequences, widen once. */ -#define FFWD_BF16_QKV_FUSE_MAX_SEQ 16 -#define FFWD_BF16_PAIR_FUSE_MAX_SEQ 4 - -/* Linear projections (weight-dtype dispatch) and workspace scratch growth */ - -static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { - for (int i = 0; i < n; i++) - dst[i] = ffwd_bf16_to_f32(src[i]); -} - -static void copy_weight_row(float *dst, const ffwd_weight_ref_t *w, size_t row, int row_width) { - if (w->dtype == DTYPE_F32) { - const float *src = (const float *)w->data + row * (size_t)row_width; - memcpy(dst, src, (size_t)row_width * sizeof(float)); - } else { - const uint16_t *src = (const uint16_t *)w->data + row * (size_t)row_width; - bf16_row_to_f32(dst, src, row_width); - } -} - -static int ensure_bf16_widen(ffwd_workspace_t *ws, size_t count) { - if (ws->bf16_widen_count >= count) - return 0; - float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float)); - if (!p) - return -1; - ws->bf16_widen = p; - ws->bf16_widen_count = count; - return 0; -} - -void linear_nobias_weight(ffwd_workspace_t *ws, - float *y, - const float *x, - const ffwd_weight_ref_t *w, - int seq_len, - int in_dim, - int out_dim) { - if (w->dtype == DTYPE_F32) { - linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); - return; - } - if (seq_len > 16) { - /* Widen the weight matrix once into workspace scratch and use the - * BLAS-backed F32 path; below that the fused BF16 row kernels win. */ - size_t count = (size_t)out_dim * (size_t)in_dim; - if (ensure_bf16_widen(ws, count) == 0) { - bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count); - linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); - return; - } - } - linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim); -} - -static void linear_qkv_weight(ffwd_workspace_t *ws, - float *q, - float *k, - float *v, - const float *x, - const ffwd_weight_ref_t *wq, - const ffwd_weight_ref_t *wk, - const ffwd_weight_ref_t *wv, - int seq_len, - int in_dim, - int q_dim, - int kv_dim) { - if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 && - seq_len <= FFWD_BF16_QKV_FUSE_MAX_SEQ) { - linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, (const uint16_t *)wk->data, - (const uint16_t *)wv->data, seq_len, in_dim, q_dim, kv_dim); - return; - } - - linear_nobias_weight(ws, q, x, wq, seq_len, in_dim, q_dim); - linear_nobias_weight(ws, k, x, wk, seq_len, in_dim, kv_dim); - linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim); -} static void add_bias_rows(float *x, const float *bias, int rows, int dim) { - if (!bias) - return; for (int row = 0; row < rows; row++) { float *dst = x + (size_t)row * dim; for (int d = 0; d < dim; d++) @@ -100,226 +14,12 @@ static void add_bias_rows(float *x, const float *bias, int rows, int dim) { } } -static void linear_pair_weight(ffwd_workspace_t *ws, - float *a, - float *b, - const float *x, - const ffwd_weight_ref_t *wa, - const ffwd_weight_ref_t *wb, - int seq_len, - int in_dim, - int a_dim, - int b_dim) { - if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && - seq_len <= FFWD_BF16_PAIR_FUSE_MAX_SEQ) { - linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, (const uint16_t *)wb->data, - seq_len, in_dim, a_dim, b_dim); - return; - } - - linear_nobias_weight(ws, a, x, wa, seq_len, in_dim, a_dim); - linear_nobias_weight(ws, b, x, wb, seq_len, in_dim, b_dim); -} - -/* Packed forward pass. - * - * Workspace-growth helpers called here live in model_blas.c beside the - * workspace lifecycle and ffwd_workspace_nbytes: - * ensure_buffers, ensure_rope_cache, ensure_attention_scores, ensure_offsets. - * - * ffwd_workspace_nbytes mirrors their size accounting. - */ - -static ffwd_attention_fn attention_for_config(const ffwd_config_t *cfg) { - return cfg->attention_mode == FFWD_ATTENTION_CAUSAL - ? causal_gqa_attention_packed_with_scratch - : bidirectional_gqa_attention_packed_with_scratch; -} - -int build_offsets(const ffwd_input_t *inputs, - int batch, - int require_ids, - int *offsets, - int *total_out, - int *max_out) { - if (!inputs || batch <= 0 || !offsets || !total_out || !max_out) - return -1; - - int total = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - if ((require_ids && !inputs[b].ids) || inputs[b].n_tokens <= 0) - return -1; - if (total > INT32_MAX - inputs[b].n_tokens) - return -1; - offsets[b] = total; - total += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) - max_seq = inputs[b].n_tokens; - } - offsets[batch] = total; - *total_out = total; - *max_out = max_seq; - return 0; -} - -static int forward_packed_slice_inplace(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_input_t *inputs, - int batch, - const int *offsets, - int total_seq, - int max_seq, - const float *input_states, - int layer_start, - int layer_end, - int apply_final_norm) { - if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || total_seq <= 0 || - max_seq <= 0) - return -1; - - const ffwd_config_t *cfg = &model->config; - const ffwd_weights_t *w = &model->weights; - if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers || - layer_start < model->layer_start || layer_end > model->layer_end || - (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) || - (layer_start == 0 && !w->embed_tokens.data) || - (apply_final_norm && (layer_end != cfg->n_layers || !w->norm))) - return -1; - - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; - int n_kv_heads = cfg->n_kv_heads; - int head_dim = cfg->head_dim; - int q_dim = cfg->q_dim; - int kv_dim = cfg->kv_dim; - int inter = cfg->intermediate_size; - float eps = cfg->rms_norm_eps; - - if (ensure_buffers(ws, cfg, total_seq) != 0) - return -1; - if (ensure_rope_cache(ws, cfg, max_seq) != 0) - return -1; - ensure_attention_scores(ws, offsets, batch); - - const float *rope_cos = ws->rope_cos; - const float *rope_sin = ws->rope_sin; - - float *x = ws->x; - float *x_norm = ws->x_norm; - float *q_buf = ws->q; - float *k_buf = ws->k; - float *v_buf = ws->v; - float *attn_out = ws->attn_out; - float *proj_out = ws->proj_out; - float *ffn_gate = ws->ffn_gate; - float *ffn_up = ws->ffn_up; - - /* A first stage performs embedding lookup. Later stages resume from a - * packed hidden-state tensor received from the preceding stage. */ - if (layer_start == 0) { - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - for (int i = 0; i < inputs[b].n_tokens; i++) { - int id = inputs[b].ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, i); - return -1; - } - copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id, - hidden); - } - } - } else { - memcpy(x, input_states, (size_t)total_seq * hidden * sizeof(float)); - } - - float scale = 1.0f / sqrtf((float)head_dim); - - /* 2. Transformer layers. Linear/MLP ops run over all packed tokens; - * attention is explicitly block-diagonal by sequence offsets. */ - for (int layer = layer_start; layer < layer_end; layer++) { - const ffwd_layer_t *l = &w->layers[layer]; - - rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - - linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, hidden, - q_dim, kv_dim); - - add_bias_rows(q_buf, l->q_bias, total_seq, q_dim); - add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim); - add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim); - if (cfg->qk_norm) { - rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); - rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); - } - - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - int len = offsets[b + 1] - start; - apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, - head_dim); - apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, n_kv_heads, - head_dim); - } - - ffwd_attention_fn attention = attention_for_config(cfg); - attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, head_dim, scale, - ws->attn_scores, ws->attn_scores_bytes); - - linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); - add_inplace(x, proj_out, total_seq * hidden); - - rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); - - linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq, - hidden, inter, inter); - silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); - linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); - add_inplace(x, proj_out, total_seq * hidden); - } - - if (apply_final_norm) - rms_norm(x, x, w->norm, total_seq, hidden, eps); - return 0; -} - -int forward_packed_inplace(const ffwd_model_t *model, +int ffwd_model_encode_into(const ffwd_model_t *model, ffwd_workspace_t *ws, - const ffwd_input_t *inputs, - int batch, - const int *offsets, - int total_seq, - int max_seq, - int apply_final_norm) { - return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL, - 0, model->config.n_layers, apply_final_norm); -} - -/* BERT-family forward pass. - * - * Uses token, learned-position, and token-type embeddings, then embedding - * LayerNorm. - * - * Encoder layers are post-norm, with bias on every projection, GeLU - * feed-forward, no RoPE, and no per-head norm. - * - * Bidirectional attention is block-diagonal over the packed batch. Each - * sequence's positions restart at cfg->position_id_offset: 0 for BERT, or - * pad_id + 1 for RoBERTa/XLM-R. - * - * Leaves final hidden states in ws->x. Pooling reads them as-is because the last - * layer's LayerNorm is already the model's final normalization. - */ -static int forward_packed_bert(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_input_t *inputs, - int batch, - const int *offsets, - int total_seq, - int max_seq) { - if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || total_seq <= 0 || - max_seq <= 0) + const int *token_ids, + int n_tokens, + float *out_embedding) { + if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding) return -1; const ffwd_config_t *cfg = &model->config; @@ -330,392 +30,67 @@ static int forward_packed_bert(const ffwd_model_t *model, int inter = cfg->intermediate_size; float eps = cfg->layer_norm_eps; - if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || - !w->ffwd_ln_w || !w->ffwd_ln_b) - return -1; - if (max_seq > cfg->max_position_embeddings - cfg->position_id_offset) { + if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) { fprintf(stderr, "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings " "%d\n", - max_seq, cfg->position_id_offset, cfg->max_position_embeddings); + n_tokens, cfg->position_id_offset, cfg->max_position_embeddings); return -1; } - if (ensure_buffers(ws, cfg, total_seq) != 0) + if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter || + ensure_buffers(ws, cfg, n_tokens) != 0) return -1; - ensure_attention_scores(ws, offsets, batch); + int offsets[2] = {0, n_tokens}; + ensure_attention_scores(ws, offsets, 1); float *x = ws->x; float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up; - /* Embeddings: token + learned absolute position + token-type[0], then the - * embedding LayerNorm over every packed token. */ - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - for (int i = 0; i < inputs[b].n_tokens; i++) { - int id = inputs[b].ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, i); - return -1; - } - float *row = x + (size_t)(start + i) * hidden; - copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden); - int pos = cfg->position_id_offset + i; - add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); - add_inplace(row, w->token_type_embeddings, hidden); + for (int i = 0; i < n_tokens; i++) { + int id = token_ids[i]; + if (id < 0 || id >= cfg->vocab_size) { + fprintf(stderr, "ffwd: invalid token id %d at position %d\n", id, i); + return -1; } + float *row = x + (size_t)i * hidden; + memcpy(row, w->embed_tokens + (size_t)id * hidden, (size_t)hidden * sizeof(float)); + int pos = cfg->position_id_offset + i; + add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); + add_inplace(row, w->token_type_embeddings, hidden); } - layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, total_seq, hidden, eps); + layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, n_tokens, hidden, eps); float scale = 1.0f / sqrtf((float)head_dim); for (int layer = 0; layer < cfg->n_layers; layer++) { const ffwd_layer_t *l = &w->layers[layer]; - /* Self-attention (post-norm): projections read x directly. */ - linear_qkv_weight(ws, q_buf, k_buf, v_buf, x, &l->wq, &l->wk, &l->wv, total_seq, hidden, - hidden, hidden); - add_bias_rows(q_buf, l->q_bias, total_seq, hidden); - add_bias_rows(k_buf, l->k_bias, total_seq, hidden); - add_bias_rows(v_buf, l->v_bias, total_seq, hidden); - - ffwd_attention_fn attention = attention_for_config(cfg); - attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim, scale, - ws->attn_scores, ws->attn_scores_bytes); - - linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, hidden, hidden); - add_bias_rows(proj_out, l->o_bias, total_seq, hidden); - add_inplace(x, proj_out, total_seq * hidden); - layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps); - - /* Feed-forward (post-norm): dense -> GeLU -> dense. */ - linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter); - add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter); - if (cfg->ffn_act == FFWD_ACT_GELU_TANH) - gelu_tanh_inplace(ffn_up, total_seq * inter); - else - gelu_inplace(ffn_up, total_seq * inter); - linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden); - add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden); - add_inplace(x, proj_out, total_seq * hidden); - layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps); - } - return 0; -} - -static int pool_embeddings(const ffwd_model_t *model, - const ffwd_workspace_t *ws, - const int *offsets, - int batch, - float *out_embeddings) { - const ffwd_config_t *cfg = &model->config; - int hidden = cfg->hidden_size; - float eps = cfg->rms_norm_eps; - /* Qwen pools pre-final-norm states, then applies final RMSNorm here. - * BERT pools already-normalized states, so a NULL norm means no final norm. */ - const float *norm_weight = model->weights.norm; - const float *x = ws->x; - if (hidden <= 0 || !x) - return -1; - - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - int end = offsets[b + 1]; - int len = end - start; - float *emb = out_embeddings + (size_t)b * hidden; - - if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) { - /* Single-token pooling: CLS uses the first token; last-token uses the final token. */ - int idx = cfg->pooling_mode == FFWD_POOL_CLS ? start : end - 1; - const float *row = x + (size_t)idx * hidden; - if (norm_weight) { - float sum_sq = 0.0f; - for (int d = 0; d < hidden; d++) - sum_sq += row[d] * row[d]; - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); - for (int d = 0; d < hidden; d++) - emb[d] = row[d] * rms_inv * norm_weight[d]; - } else { - memcpy(emb, row, (size_t)hidden * sizeof(float)); - } - } else { - memset(emb, 0, (size_t)hidden * sizeof(float)); - for (int i = start; i < end; i++) { - const float *row = x + (size_t)i * hidden; - if (norm_weight) { - float sum_sq = 0.0f; - for (int d = 0; d < hidden; d++) - sum_sq += row[d] * row[d]; - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); - for (int d = 0; d < hidden; d++) - emb[d] += row[d] * rms_inv * norm_weight[d]; - } else { - for (int d = 0; d < hidden; d++) - emb[d] += row[d]; - } - } - float inv_len = 1.0f / (float)len; - for (int d = 0; d < hidden; d++) - emb[d] *= inv_len; - } - - if (cfg->normalize_embeddings && ffwd_l2_normalize(emb, hidden) != 0) - return -1; - } - return 0; -} - -/* Forward/embed APIs */ - -int ffwd_model_forward_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_states) { - if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) - return -1; - - size_t count, bytes; - if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &count) != 0 || - mul_size(count, sizeof(float), &bytes) != 0) - return -1; - - ffwd_input_t input = {token_ids, n_tokens}; - int offsets[2] = {0, n_tokens}; - if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0) - return -1; - - memcpy(out_states, ws->x, bytes); - return 0; -} - -int ffwd_pool_spans(const ffwd_config_t *cfg, - const float *states, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings) { - if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings) - return -1; - - int hidden = cfg->hidden_size; - if (hidden <= 0) - return -1; - - for (int s = 0; s < n_spans; s++) { - int start = spans[s].start; - int len = spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) - return -1; - - float *emb = out_embeddings + (size_t)s * hidden; - memset(emb, 0, (size_t)hidden * sizeof(float)); - - for (int t = 0; t < len; t++) { - const float *row = states + (size_t)(start + t) * hidden; - for (int d = 0; d < hidden; d++) - emb[d] += row[d]; - } - - float inv_len = 1.0f / (float)len; - for (int d = 0; d < hidden; d++) { - emb[d] *= inv_len; - } - } - return 0; -} - -int ffwd_model_encode_spans_batch(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_context_input_t *inputs, - int batch, - float *out_embeddings) { - if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) - return -1; - - if (ensure_offsets(ws, batch) != 0) - return -1; - if ((size_t)batch > SIZE_MAX / sizeof(ffwd_input_t)) - return -1; - ffwd_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); - if (!packed_inputs) - return -1; - int *offsets = ws->offsets; - int total_seq = 0; - int max_seq = 0; - int total_spans = 0; - for (int b = 0; b < batch; b++) { - const ffwd_context_input_t *input = &inputs[b]; - int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || - total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) { - free(packed_inputs); - return -1; - } - packed_inputs[b] = input->input; - offsets[b] = total_seq; - total_seq += n_tokens; - total_spans += input->n_spans; - if (n_tokens > max_seq) - max_seq = n_tokens; - for (int s = 0; s < input->n_spans; s++) { - int start = input->spans[s].start; - int len = input->spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) { - free(packed_inputs); - return -1; - } - } - } - offsets[batch] = total_seq; - - if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, total_seq, max_seq, 1) != - 0) { - free(packed_inputs); - return -1; - } - free(packed_inputs); + linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden); + linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden); + linear_nobias(v_buf, x, l->wv, n_tokens, hidden, hidden); + add_bias_rows(q_buf, l->q_bias, n_tokens, hidden); + add_bias_rows(k_buf, l->k_bias, n_tokens, hidden); + add_bias_rows(v_buf, l->v_bias, n_tokens, hidden); - int span_offset = 0; - int hidden = model->config.hidden_size; - for (int b = 0; b < batch; b++) { - const ffwd_context_input_t *input = &inputs[b]; - if (ffwd_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden, - input->input.n_tokens, input->spans, input->n_spans, - out_embeddings + (size_t)span_offset * hidden) != 0) - return -1; - span_offset += input->n_spans; - } - return 0; -} + bidirectional_gqa_attention_packed_with_scratch( + attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale, + ws->attn_scores, ws->attn_scores_bytes); -int ffwd_model_encode_spans(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings) { - ffwd_context_input_t input = { - .input = {token_ids, n_tokens}, - .spans = spans, - .n_spans = n_spans, - }; - return ffwd_model_encode_spans_batch(model, ws, &input, 1, out_embeddings); -} - -float *ffwd_model_forward(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens) { - if (!model || !ws || !token_ids || n_tokens <= 0) - return NULL; - - size_t n; - if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0) - return NULL; - float *out = malloc_floats(n); - if (!out) - return NULL; + linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden); + add_bias_rows(proj_out, l->o_bias, n_tokens, hidden); + add_inplace(x, proj_out, n_tokens * hidden); + layer_norm(x, x, l->input_norm, l->attn_ln_bias, n_tokens, hidden, eps); - if (ffwd_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) { - free(out); - return NULL; - } - return out; -} - -int ffwd_model_encode_batch(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_input_t *inputs, - int batch, - float *out_embeddings) { - if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) - return -1; - - if (ensure_offsets(ws, batch) != 0) - return -1; - int *offsets = ws->offsets; - - int total_seq = 0; - int max_seq = 0; - if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0) - return -1; - - int rc = model->config.family == FFWD_FAMILY_BERT - ? forward_packed_bert(model, ws, inputs, batch, offsets, total_seq, max_seq) - : forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); - if (rc == 0) - rc = pool_embeddings(model, ws, offsets, batch, out_embeddings); - - return rc; -} - -int ffwd_pool_batch(const ffwd_config_t *cfg, - const float *states, - const int *seq_lengths, - int batch, - float *out_embeddings) { - if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0) - return -1; - - int offset = 0; - for (int b = 0; b < batch; b++) { - int len = seq_lengths[b]; - if (len <= 0 || offset > INT_MAX - len) - return -1; - - float *emb = out_embeddings + (size_t)b * cfg->hidden_size; - if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) { - /* CLS takes the first token, last-token the final one; these states - * already carry the model's final RMSNorm. */ - int idx = cfg->pooling_mode == FFWD_POOL_CLS ? offset : offset + len - 1; - const float *row = states + (size_t)idx * cfg->hidden_size; - memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float)); - } else { - memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); - for (int t = 0; t < len; t++) { - const float *row = states + (size_t)(offset + t) * cfg->hidden_size; - for (int d = 0; d < cfg->hidden_size; d++) - emb[d] += row[d]; - } - float inv_len = 1.0f / (float)len; - for (int d = 0; d < cfg->hidden_size; d++) - emb[d] *= inv_len; - } - if (cfg->normalize_embeddings && ffwd_l2_normalize(emb, cfg->hidden_size) != 0) - return -1; - offset += len; + linear_nobias(ffn_up, x, l->up_proj, n_tokens, hidden, inter); + add_bias_rows(ffn_up, l->ffn_inter_bias, n_tokens, inter); + gelu_inplace(ffn_up, n_tokens * inter); + linear_nobias(proj_out, ffn_up, l->down_proj, n_tokens, inter, hidden); + add_bias_rows(proj_out, l->ffn_out_bias, n_tokens, hidden); + add_inplace(x, proj_out, n_tokens * hidden); + layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, n_tokens, hidden, eps); } + /* The classifier consumes <s> after the encoder's final LayerNorm, without L2 normalization. */ + memcpy(out_embedding, x, (size_t)hidden * sizeof(float)); return 0; } - -int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding) { - ffwd_input_t input = {token_ids, n_tokens}; - return ffwd_model_encode_batch(model, ws, &input, 1, out_embedding); -} - -float *ffwd_model_encode(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens) { - if (!model || !ws || !token_ids || n_tokens <= 0) - return NULL; - - int hidden = model->config.hidden_size; - if (hidden <= 0) - return NULL; - float *emb = malloc_floats((size_t)hidden); - if (!emb) - return NULL; - - if (ffwd_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) { - free(emb); - return NULL; - } - return emb; -} |
