diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 09:26:20 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 09:26:20 +0200 |
| commit | c20279050f88d9c24187c3e402912596e1f54587 (patch) | |
| tree | 2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 | |
| parent | 849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff) | |
| download | ffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip | |
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
| -rw-r--r-- | embed.c | 5 | ||||
| -rw-r--r-- | embed.h | 9 | ||||
| -rw-r--r-- | embed_config.c | 7 | ||||
| -rw-r--r-- | embed_cuda.cu | 22 | ||||
| -rw-r--r-- | embed_mlx.c | 43 | ||||
| -rw-r--r-- | qwen_kernels.c | 9 | ||||
| -rw-r--r-- | qwen_kernels.h | 4 | ||||
| -rw-r--r-- | tests/test_kernels.c | 19 |
8 files changed, 109 insertions, 9 deletions
@@ -1351,7 +1351,10 @@ static int forward_packed_bert(const embed_model_t *model, /* Feed-forward (post-norm): dense -> GeLU -> dense. */ linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter); add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter); - qwen_gelu_inplace(ffn_up, total_seq * inter); + if (cfg->ffn_act == EMBED_ACT_GELU_TANH) + qwen_gelu_tanh_inplace(ffn_up, total_seq * inter); + else + qwen_gelu_inplace(ffn_up, total_seq * inter); linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden); add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); @@ -48,8 +48,17 @@ typedef enum { EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */ } embed_family_t; +/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf + * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or + * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */ +typedef enum { + EMBED_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */ + EMBED_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */ +} embed_act_t; + typedef struct { embed_family_t family; + embed_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */ int hidden_size; int n_layers; int n_heads; diff --git a/embed_config.c b/embed_config.c index 03838dd..8ab81a5 100644 --- a/embed_config.c +++ b/embed_config.c @@ -233,11 +233,12 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */ cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; + /* hidden_act selects the feed-forward GeLU curve. gelu_new and + * gelu_pytorch_tanh are the same tanh approximation; everything else + * (gelu, absent) is the exact erf GeLU the released encoders use. */ if (json_string_equals(buf, "hidden_act", "gelu_new") || json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) { - fprintf(stderr, "embed_config: BERT tanh-GeLU is not supported yet; need exact gelu\n"); - free(buf); - return -1; + cfg->ffn_act = EMBED_ACT_GELU_TANH; } if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { diff --git a/embed_cuda.cu b/embed_cuda.cu index 2cdb864..94887fb 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -685,8 +685,8 @@ __global__ static void layer_norm_kernel(float *out, o[d] = (xr[d] - mean) * inv * gamma[d] + beta[d]; } -// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). Not the tanh -// approximation - BERT-family checkpoints use the exact form. +// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). The released encoders +// use this; the tanh approximation below is for gelu_new / gelu_pytorch_tanh. __global__ static void gelu_kernel(float *x, size_t n) { size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; if (i < n) { @@ -695,6 +695,16 @@ __global__ static void gelu_kernel(float *x, size_t n) { } } +// Tanh-approximation GeLU in place: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))). +__global__ static void gelu_tanh_kernel(float *x, size_t n) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) { + float v = x[i]; + float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); + x[i] = 0.5f * v * (1.0f + tanhf(inner)); + } +} + // Add the learned absolute position embedding (indexed by the within-sequence // position) and the token-type[0] embedding to the token embeddings already in // x. token_type holds row 0 only; every input token has segment id 0. @@ -1502,8 +1512,12 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i ctx->ffn_gate_up, l->ffn_inter_bias, total, inter); if (launch_check() != 0) return -1; - gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>( - ctx->ffn_gate_up, (size_t)inter_count); + if (c->ffn_act == EMBED_ACT_GELU_TANH) + gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->ffn_gate_up, (size_t)inter_count); + else + gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->ffn_gate_up, (size_t)inter_count); if (launch_check() != 0) return -1; if (linear_accum(ctx, &l->down_proj, ctx->ffn_gate_up, ctx->x, total, inter, hidden) != 0) diff --git a/embed_mlx.c b/embed_mlx.c index 8d9de39..7cd9357 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -840,6 +840,46 @@ static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) { return g; } +/* Tanh-approximation GeLU (gelu_new / gelu_pytorch_tanh): + * 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). Frees nothing given. */ +static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) { + mlx_array x2 = mlx_array_new(); + mlx_multiply(&x2, x, x, S); + mlx_array x3 = mlx_array_new(); + mlx_multiply(&x3, x2, x, S); + mlx_array_free(x2); + mlx_array c = mlx_array_new_float32(0.044715f); + mlx_array cx3 = mlx_array_new(); + mlx_multiply(&cx3, x3, c, S); + mlx_array_free(x3); + mlx_array_free(c); + mlx_array sum = mlx_array_new(); + mlx_add(&sum, x, cx3, S); + mlx_array_free(cx3); + mlx_array s2pi = mlx_array_new_float32(0.79788456080286536f); + mlx_array inner = mlx_array_new(); + mlx_multiply(&inner, sum, s2pi, S); + mlx_array_free(sum); + mlx_array_free(s2pi); + mlx_array t = mlx_array_new(); + mlx_tanh(&t, inner, S); + mlx_array_free(inner); + mlx_array one = mlx_array_new_float32(1.0f); + mlx_array t1 = mlx_array_new(); + mlx_add(&t1, t, one, S); + mlx_array_free(t); + mlx_array_free(one); + mlx_array half = mlx_array_new_float32(0.5f); + mlx_array xh = mlx_array_new(); + mlx_multiply(&xh, x, half, S); + mlx_array_free(half); + mlx_array g = mlx_array_new(); + mlx_multiply(&g, xh, t1, S); + mlx_array_free(xh); + mlx_array_free(t1); + return g; +} + /* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU * feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the * x it is given and returns the new hidden states. */ @@ -929,7 +969,8 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx, mlx_array mid_b = mlx_array_new(); mlx_add(&mid_b, mid, l->ffn_inter_bias, S); mlx_array_free(mid); - mlx_array g = mlx_gelu_erf(mid_b, S); + mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S) + : mlx_gelu_erf(mid_b, S); mlx_array_free(mid_b); mlx_array out = linear(g, &l->down_proj, S); mlx_array_free(g); diff --git a/qwen_kernels.c b/qwen_kernels.c index e1da7ed..9368529 100644 --- a/qwen_kernels.c +++ b/qwen_kernels.c @@ -837,6 +837,15 @@ void qwen_gelu_inplace(float *x, int n) { x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); } +void qwen_gelu_tanh_inplace(float *x, int n) { + /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */ + for (int i = 0; i < n; i++) { + float v = x[i]; + float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); + x[i] = 0.5f * v * (1.0f + tanhf(inner)); + } +} + void qwen_softmax(float *x, int rows, int cols) { for (int r = 0; r < rows; r++) { float *row = x + r * cols; diff --git a/qwen_kernels.h b/qwen_kernels.h index 36d2913..9abb489 100644 --- a/qwen_kernels.h +++ b/qwen_kernels.h @@ -111,6 +111,10 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n); * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ void qwen_gelu_inplace(float *x, int n); +/* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh): + * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */ +void qwen_gelu_tanh_inplace(float *x, int n); + /* ======================================================================== * Attention Operations * ======================================================================== */ diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 0f92df6..4a9e9db 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -108,6 +108,24 @@ static void test_gelu(void) { expect_close("gelu_neg1", out[2], -0.158655254f, 1e-5f); } +static void test_gelu_tanh(void) { + float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f}; + float out[5]; + memcpy(out, x, sizeof(out)); + qwen_gelu_tanh_inplace(out, 5); + for (int i = 0; i < 5; i++) { + float v = x[i]; + float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); + float want = 0.5f * v * (1.0f + tanhf(inner)); + expect_close("gelu_tanh", out[i], want, 1e-6f); + } + /* Reference values for the tanh approximation; distinct from the erf GeLU + * (erf gelu(1) = 0.841344746) by ~1.5e-4, well outside this tolerance. */ + expect_close("gelu_tanh0", out[0], 0.0f, 1e-7f); + expect_close("gelu_tanh1", out[1], 0.841191991f, 1e-5f); + expect_close("gelu_tanh_neg1", out[2], -0.158808009f, 1e-5f); +} + static void test_rope_neox(void) { const int seq = 2, heads = 1, head_dim = 4; const int positions[2] = {0, 3}; @@ -579,6 +597,7 @@ int main(void) { test_rms_norm(); test_layer_norm(); test_gelu(); + test_gelu_tanh(); test_rope_neox(); test_packed_gqa_attention(); test_packed_gqa_attention_long(); |
