aboutsummaryrefslogtreecommitdiff
path: root/qwen_kernels.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /qwen_kernels.c
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'qwen_kernels.c')
-rw-r--r--qwen_kernels.c9
1 files changed, 9 insertions, 0 deletions
diff --git a/qwen_kernels.c b/qwen_kernels.c
index e1da7ed..9368529 100644
--- a/qwen_kernels.c
+++ b/qwen_kernels.c
@@ -837,6 +837,15 @@ void qwen_gelu_inplace(float *x, int n) {
x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f));
}
+void qwen_gelu_tanh_inplace(float *x, int n) {
+ /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */
+ for (int i = 0; i < n; i++) {
+ float v = x[i];
+ float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
+ x[i] = 0.5f * v * (1.0f + tanhf(inner));
+ }
+}
+
void qwen_softmax(float *x, int rows, int cols) {
for (int r = 0; r < rows; r++) {
float *row = x + r * cols;