aboutsummaryrefslogtreecommitdiff
path: root/embed_mlx.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /embed_mlx.c
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'embed_mlx.c')
-rw-r--r--embed_mlx.c43
1 files changed, 42 insertions, 1 deletions
diff --git a/embed_mlx.c b/embed_mlx.c
index 8d9de39..7cd9357 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -840,6 +840,46 @@ static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) {
return g;
}
+/* Tanh-approximation GeLU (gelu_new / gelu_pytorch_tanh):
+ * 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). Frees nothing given. */
+static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) {
+ mlx_array x2 = mlx_array_new();
+ mlx_multiply(&x2, x, x, S);
+ mlx_array x3 = mlx_array_new();
+ mlx_multiply(&x3, x2, x, S);
+ mlx_array_free(x2);
+ mlx_array c = mlx_array_new_float32(0.044715f);
+ mlx_array cx3 = mlx_array_new();
+ mlx_multiply(&cx3, x3, c, S);
+ mlx_array_free(x3);
+ mlx_array_free(c);
+ mlx_array sum = mlx_array_new();
+ mlx_add(&sum, x, cx3, S);
+ mlx_array_free(cx3);
+ mlx_array s2pi = mlx_array_new_float32(0.79788456080286536f);
+ mlx_array inner = mlx_array_new();
+ mlx_multiply(&inner, sum, s2pi, S);
+ mlx_array_free(sum);
+ mlx_array_free(s2pi);
+ mlx_array t = mlx_array_new();
+ mlx_tanh(&t, inner, S);
+ mlx_array_free(inner);
+ mlx_array one = mlx_array_new_float32(1.0f);
+ mlx_array t1 = mlx_array_new();
+ mlx_add(&t1, t, one, S);
+ mlx_array_free(t);
+ mlx_array_free(one);
+ mlx_array half = mlx_array_new_float32(0.5f);
+ mlx_array xh = mlx_array_new();
+ mlx_multiply(&xh, x, half, S);
+ mlx_array_free(half);
+ mlx_array g = mlx_array_new();
+ mlx_multiply(&g, xh, t1, S);
+ mlx_array_free(xh);
+ mlx_array_free(t1);
+ return g;
+}
+
/* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU
* feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the
* x it is given and returns the new hidden states. */
@@ -929,7 +969,8 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
mlx_array mid_b = mlx_array_new();
mlx_add(&mid_b, mid, l->ffn_inter_bias, S);
mlx_array_free(mid);
- mlx_array g = mlx_gelu_erf(mid_b, S);
+ mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S)
+ : mlx_gelu_erf(mid_b, S);
mlx_array_free(mid_b);
mlx_array out = linear(g, &l->down_proj, S);
mlx_array_free(g);