aboutsummaryrefslogtreecommitdiff
path: root/embed_config.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /embed_config.c
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'embed_config.c')
-rw-r--r--embed_config.c7
1 files changed, 4 insertions, 3 deletions
diff --git a/embed_config.c b/embed_config.c
index 03838dd..8ab81a5 100644
--- a/embed_config.c
+++ b/embed_config.c
@@ -233,11 +233,12 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */
cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
+ /* hidden_act selects the feed-forward GeLU curve. gelu_new and
+ * gelu_pytorch_tanh are the same tanh approximation; everything else
+ * (gelu, absent) is the exact erf GeLU the released encoders use. */
if (json_string_equals(buf, "hidden_act", "gelu_new") ||
json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) {
- fprintf(stderr, "embed_config: BERT tanh-GeLU is not supported yet; need exact gelu\n");
- free(buf);
- return -1;
+ cfg->ffn_act = EMBED_ACT_GELU_TANH;
}
if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) ||
cfg->layer_norm_eps <= 0.0f) {