aboutsummaryrefslogtreecommitdiff
path: root/embed_internal.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 15:40:22 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 15:40:22 +0200
commit898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4 (patch)
treefe8cbd924a4d6da59c0df2b21d928d4a50a27b8c /embed_internal.h
parentceecbea701e3ac882ce5f1693204152cc75c0d79 (diff)
downloadffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.tar.gz
ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.zip
cpu: add the BERT encoder family forward and loader
Diffstat (limited to 'embed_internal.h')
-rw-r--r--embed_internal.h18
1 files changed, 17 insertions, 1 deletions
diff --git a/embed_internal.h b/embed_internal.h
index 0536692..39efbc4 100644
--- a/embed_internal.h
+++ b/embed_internal.h
@@ -24,12 +24,28 @@ typedef struct {
embed_weight_ref_t gate_proj; /* [intermediate, hidden] */
embed_weight_ref_t up_proj; /* [intermediate, hidden] */
embed_weight_ref_t down_proj; /* [hidden, intermediate] */
+ /* BERT family (all NULL for Qwen3). The attention projections reuse
+ * wq/wk/wv/wo and q/k/v_bias; the two FFN matrices reuse up_proj
+ * (intermediate.dense) and down_proj (output.dense); the two block
+ * LayerNorm weights reuse input_norm (post-attention) and post_attn_norm
+ * (post-FFN). These fields add the biases those reused slots lack. */
+ const float *o_bias; /* attention output dense bias [hidden] */
+ const float *attn_ln_bias; /* post-attention LayerNorm bias [hidden] */
+ const float *ffn_inter_bias; /* intermediate.dense bias [intermediate] */
+ const float *ffn_out_bias; /* output.dense bias [hidden] */
+ const float *ffn_ln_bias; /* post-FFN LayerNorm bias [hidden] */
} embed_layer_t;
typedef struct {
embed_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
embed_layer_t *layers; /* [n_layers] heap-allocated */
- const float *norm; /* [hidden] */
+ const float *norm; /* [hidden]; NULL for BERT (no final norm) */
+ /* BERT family (NULL for Qwen3): learned absolute position embeddings,
+ * token-type (segment) embeddings, and the embedding LayerNorm. */
+ const float *position_embeddings; /* [max_position_embeddings, hidden] */
+ const float *token_type_embeddings; /* [2, hidden] */
+ const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */
+ const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */
} embed_weights_t;
typedef void (*embed_attention_fn)(float *out,