aboutsummaryrefslogtreecommitdiff
path: root/embed.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 15:40:22 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 15:40:22 +0200
commit898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4 (patch)
treefe8cbd924a4d6da59c0df2b21d928d4a50a27b8c /embed.h
parentceecbea701e3ac882ce5f1693204152cc75c0d79 (diff)
downloadffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.tar.gz
ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.zip
cpu: add the BERT encoder family forward and loader
Diffstat (limited to 'embed.h')
-rw-r--r--embed.h8
1 files changed, 8 insertions, 0 deletions
diff --git a/embed.h b/embed.h
index 197ee7f..860576e 100644
--- a/embed.h
+++ b/embed.h
@@ -43,7 +43,13 @@ typedef enum {
EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */
} embed_pooling_mode_t;
+typedef enum {
+ EMBED_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (pplx, Qwen2/3) */
+ EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */
+} embed_family_t;
+
typedef struct {
+ embed_family_t family;
int hidden_size;
int n_layers;
int n_heads;
@@ -53,7 +59,9 @@ typedef struct {
int kv_dim; /* n_kv_heads * head_dim */
int intermediate_size;
int vocab_size;
+ int max_position_embeddings; /* BERT learned position table size */
float rms_norm_eps;
+ float layer_norm_eps; /* BERT LayerNorm epsilon */
float rope_theta;
int qk_norm;
int qkv_bias;