diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:40:22 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:40:22 +0200 |
| commit | 898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4 (patch) | |
| tree | fe8cbd924a4d6da59c0df2b21d928d4a50a27b8c /embed_config.c | |
| parent | ceecbea701e3ac882ce5f1693204152cc75c0d79 (diff) | |
| download | ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.tar.gz ffwd-898466e9d0c19c6ac5bfa4f3f02e13f70a0e73c4.zip | |
cpu: add the BERT encoder family forward and loader
Diffstat (limited to 'embed_config.c')
| -rw-r--r-- | embed_config.c | 22 |
1 files changed, 22 insertions, 0 deletions
diff --git a/embed_config.c b/embed_config.c index fc07334..03838dd 100644 --- a/embed_config.c +++ b/embed_config.c @@ -206,7 +206,10 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0); cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE); cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); + cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12); cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); + cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0); + cfg->family = EMBED_FAMILY_QWEN3; cfg->qk_norm = 1; cfg->qkv_bias = 0; cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; @@ -223,6 +226,25 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->qk_norm = 0; cfg->qkv_bias = 1; cfg->attention_mode = EMBED_ATTENTION_CAUSAL; + } else if (json_string_equals(buf, "model_type", "bert")) { + cfg->family = EMBED_FAMILY_BERT; + cfg->qk_norm = 0; + cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */ + cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; + cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */ + cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; + if (json_string_equals(buf, "hidden_act", "gelu_new") || + json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) { + fprintf(stderr, "embed_config: BERT tanh-GeLU is not supported yet; need exact gelu\n"); + free(buf); + return -1; + } + if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) || + cfg->layer_norm_eps <= 0.0f) { + fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n"); + free(buf); + return -1; + } } json_bool_t is_causal = json_get_bool(buf, "is_causal"); |
