From c20279050f88d9c24187c3e402912596e1f54587 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 16 Jun 2026 09:26:20 +0200 Subject: all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh) --- embed.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'embed.c') diff --git a/embed.c b/embed.c index d427d01..d1e1ce0 100644 --- a/embed.c +++ b/embed.c @@ -1351,7 +1351,10 @@ static int forward_packed_bert(const embed_model_t *model, /* Feed-forward (post-norm): dense -> GeLU -> dense. */ linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter); add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter); - qwen_gelu_inplace(ffn_up, total_seq * inter); + if (cfg->ffn_act == EMBED_ACT_GELU_TANH) + qwen_gelu_tanh_inplace(ffn_up, total_seq * inter); + else + qwen_gelu_inplace(ffn_up, total_seq * inter); linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden); add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); -- cgit v1.3.1