diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:40:22 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:40:22 +0200 |
| commit | 898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4 (patch) | |
| tree | fe8cbd924a4d6da59c0df2b21d928d4a50a27b8c /embed.h | |
| parent | ceecbea701e3ac882ce5f1693204152cc75c0d79 (diff) | |
| download | ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.tar.gz ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.zip | |
cpu: add the BERT encoder family forward and loader
Diffstat (limited to 'embed.h')
| -rw-r--r-- | embed.h | 8 |
1 files changed, 8 insertions, 0 deletions
@@ -43,7 +43,13 @@ typedef enum { EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */ } embed_pooling_mode_t; +typedef enum { + EMBED_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (pplx, Qwen2/3) */ + EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */ +} embed_family_t; + typedef struct { + embed_family_t family; int hidden_size; int n_layers; int n_heads; @@ -53,7 +59,9 @@ typedef struct { int kv_dim; /* n_kv_heads * head_dim */ int intermediate_size; int vocab_size; + int max_position_embeddings; /* BERT learned position table size */ float rms_norm_eps; + float layer_norm_eps; /* BERT LayerNorm epsilon */ float rope_theta; int qk_norm; int qkv_bias; |
