aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
-rw-r--r--embed.c5
-rw-r--r--embed.h9
-rw-r--r--embed_config.c7
-rw-r--r--embed_cuda.cu22
-rw-r--r--embed_mlx.c43
-rw-r--r--qwen_kernels.c9
-rw-r--r--qwen_kernels.h4
-rw-r--r--tests/test_kernels.c19
8 files changed, 109 insertions, 9 deletions
diff --git a/embed.c b/embed.c
index d427d01..d1e1ce0 100644
--- a/embed.c
+++ b/embed.c
@@ -1351,7 +1351,10 @@ static int forward_packed_bert(const embed_model_t *model,
/* Feed-forward (post-norm): dense -> GeLU -> dense. */
linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter);
add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter);
- qwen_gelu_inplace(ffn_up, total_seq * inter);
+ if (cfg->ffn_act == EMBED_ACT_GELU_TANH)
+ qwen_gelu_tanh_inplace(ffn_up, total_seq * inter);
+ else
+ qwen_gelu_inplace(ffn_up, total_seq * inter);
linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden);
add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
diff --git a/embed.h b/embed.h
index 860576e..c7381bd 100644
--- a/embed.h
+++ b/embed.h
@@ -48,8 +48,17 @@ typedef enum {
EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */
} embed_family_t;
+/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf
+ * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or
+ * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */
+typedef enum {
+ EMBED_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */
+ EMBED_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */
+} embed_act_t;
+
typedef struct {
embed_family_t family;
+ embed_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */
int hidden_size;
int n_layers;
int n_heads;
diff --git a/embed_config.c b/embed_config.c
index 03838dd..8ab81a5 100644
--- a/embed_config.c
+++ b/embed_config.c
@@ -233,11 +233,12 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */
cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
+ /* hidden_act selects the feed-forward GeLU curve. gelu_new and
+ * gelu_pytorch_tanh are the same tanh approximation; everything else
+ * (gelu, absent) is the exact erf GeLU the released encoders use. */
if (json_string_equals(buf, "hidden_act", "gelu_new") ||
json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) {
- fprintf(stderr, "embed_config: BERT tanh-GeLU is not supported yet; need exact gelu\n");
- free(buf);
- return -1;
+ cfg->ffn_act = EMBED_ACT_GELU_TANH;
}
if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) ||
cfg->layer_norm_eps <= 0.0f) {
diff --git a/embed_cuda.cu b/embed_cuda.cu
index 2cdb864..94887fb 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -685,8 +685,8 @@ __global__ static void layer_norm_kernel(float *out,
o[d] = (xr[d] - mean) * inv * gamma[d] + beta[d];
}
-// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). Not the tanh
-// approximation - BERT-family checkpoints use the exact form.
+// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). The released encoders
+// use this; the tanh approximation below is for gelu_new / gelu_pytorch_tanh.
__global__ static void gelu_kernel(float *x, size_t n) {
size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) {
@@ -695,6 +695,16 @@ __global__ static void gelu_kernel(float *x, size_t n) {
}
}
+// Tanh-approximation GeLU in place: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))).
+__global__ static void gelu_tanh_kernel(float *x, size_t n) {
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) {
+ float v = x[i];
+ float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
+ x[i] = 0.5f * v * (1.0f + tanhf(inner));
+ }
+}
+
// Add the learned absolute position embedding (indexed by the within-sequence
// position) and the token-type[0] embedding to the token embeddings already in
// x. token_type holds row 0 only; every input token has segment id 0.
@@ -1502,8 +1512,12 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i
ctx->ffn_gate_up, l->ffn_inter_bias, total, inter);
if (launch_check() != 0)
return -1;
- gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
- ctx->ffn_gate_up, (size_t)inter_count);
+ if (c->ffn_act == EMBED_ACT_GELU_TANH)
+ gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, (size_t)inter_count);
+ else
+ gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, (size_t)inter_count);
if (launch_check() != 0)
return -1;
if (linear_accum(ctx, &l->down_proj, ctx->ffn_gate_up, ctx->x, total, inter, hidden) != 0)
diff --git a/embed_mlx.c b/embed_mlx.c
index 8d9de39..7cd9357 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -840,6 +840,46 @@ static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) {
return g;
}
+/* Tanh-approximation GeLU (gelu_new / gelu_pytorch_tanh):
+ * 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). Frees nothing given. */
+static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) {
+ mlx_array x2 = mlx_array_new();
+ mlx_multiply(&x2, x, x, S);
+ mlx_array x3 = mlx_array_new();
+ mlx_multiply(&x3, x2, x, S);
+ mlx_array_free(x2);
+ mlx_array c = mlx_array_new_float32(0.044715f);
+ mlx_array cx3 = mlx_array_new();
+ mlx_multiply(&cx3, x3, c, S);
+ mlx_array_free(x3);
+ mlx_array_free(c);
+ mlx_array sum = mlx_array_new();
+ mlx_add(&sum, x, cx3, S);
+ mlx_array_free(cx3);
+ mlx_array s2pi = mlx_array_new_float32(0.79788456080286536f);
+ mlx_array inner = mlx_array_new();
+ mlx_multiply(&inner, sum, s2pi, S);
+ mlx_array_free(sum);
+ mlx_array_free(s2pi);
+ mlx_array t = mlx_array_new();
+ mlx_tanh(&t, inner, S);
+ mlx_array_free(inner);
+ mlx_array one = mlx_array_new_float32(1.0f);
+ mlx_array t1 = mlx_array_new();
+ mlx_add(&t1, t, one, S);
+ mlx_array_free(t);
+ mlx_array_free(one);
+ mlx_array half = mlx_array_new_float32(0.5f);
+ mlx_array xh = mlx_array_new();
+ mlx_multiply(&xh, x, half, S);
+ mlx_array_free(half);
+ mlx_array g = mlx_array_new();
+ mlx_multiply(&g, xh, t1, S);
+ mlx_array_free(xh);
+ mlx_array_free(t1);
+ return g;
+}
+
/* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU
* feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the
* x it is given and returns the new hidden states. */
@@ -929,7 +969,8 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
mlx_array mid_b = mlx_array_new();
mlx_add(&mid_b, mid, l->ffn_inter_bias, S);
mlx_array_free(mid);
- mlx_array g = mlx_gelu_erf(mid_b, S);
+ mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S)
+ : mlx_gelu_erf(mid_b, S);
mlx_array_free(mid_b);
mlx_array out = linear(g, &l->down_proj, S);
mlx_array_free(g);
diff --git a/qwen_kernels.c b/qwen_kernels.c
index e1da7ed..9368529 100644
--- a/qwen_kernels.c
+++ b/qwen_kernels.c
@@ -837,6 +837,15 @@ void qwen_gelu_inplace(float *x, int n) {
x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f));
}
+void qwen_gelu_tanh_inplace(float *x, int n) {
+ /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */
+ for (int i = 0; i < n; i++) {
+ float v = x[i];
+ float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
+ x[i] = 0.5f * v * (1.0f + tanhf(inner));
+ }
+}
+
void qwen_softmax(float *x, int rows, int cols) {
for (int r = 0; r < rows; r++) {
float *row = x + r * cols;
diff --git a/qwen_kernels.h b/qwen_kernels.h
index 36d2913..9abb489 100644
--- a/qwen_kernels.h
+++ b/qwen_kernels.h
@@ -111,6 +111,10 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n);
* "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */
void qwen_gelu_inplace(float *x, int n);
+/* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh):
+ * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */
+void qwen_gelu_tanh_inplace(float *x, int n);
+
/* ========================================================================
* Attention Operations
* ======================================================================== */
diff --git a/tests/test_kernels.c b/tests/test_kernels.c
index 0f92df6..4a9e9db 100644
--- a/tests/test_kernels.c
+++ b/tests/test_kernels.c
@@ -108,6 +108,24 @@ static void test_gelu(void) {
expect_close("gelu_neg1", out[2], -0.158655254f, 1e-5f);
}
+static void test_gelu_tanh(void) {
+ float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f};
+ float out[5];
+ memcpy(out, x, sizeof(out));
+ qwen_gelu_tanh_inplace(out, 5);
+ for (int i = 0; i < 5; i++) {
+ float v = x[i];
+ float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
+ float want = 0.5f * v * (1.0f + tanhf(inner));
+ expect_close("gelu_tanh", out[i], want, 1e-6f);
+ }
+ /* Reference values for the tanh approximation; distinct from the erf GeLU
+ * (erf gelu(1) = 0.841344746) by ~1.5e-4, well outside this tolerance. */
+ expect_close("gelu_tanh0", out[0], 0.0f, 1e-7f);
+ expect_close("gelu_tanh1", out[1], 0.841191991f, 1e-5f);
+ expect_close("gelu_tanh_neg1", out[2], -0.158808009f, 1e-5f);
+}
+
static void test_rope_neox(void) {
const int seq = 2, heads = 1, head_dim = 4;
const int positions[2] = {0, 3};
@@ -579,6 +597,7 @@ int main(void) {
test_rms_norm();
test_layer_norm();
test_gelu();
+ test_gelu_tanh();
test_rope_neox();
test_packed_gqa_attention();
test_packed_gqa_attention_long();