aboutsummaryrefslogtreecommitdiff
path: root/embed.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /embed.c
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'embed.c')
-rw-r--r--embed.c5
1 files changed, 4 insertions, 1 deletions
diff --git a/embed.c b/embed.c
index d427d01..d1e1ce0 100644
--- a/embed.c
+++ b/embed.c
@@ -1351,7 +1351,10 @@ static int forward_packed_bert(const embed_model_t *model,
/* Feed-forward (post-norm): dense -> GeLU -> dense. */
linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter);
add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter);
- qwen_gelu_inplace(ffn_up, total_seq * inter);
+ if (cfg->ffn_act == EMBED_ACT_GELU_TANH)
+ qwen_gelu_tanh_inplace(ffn_up, total_seq * inter);
+ else
+ qwen_gelu_inplace(ffn_up, total_seq * inter);
linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden);
add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);