aboutsummaryrefslogtreecommitdiff
path: root/embed_cuda.cu
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /embed_cuda.cu
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'embed_cuda.cu')
-rw-r--r--embed_cuda.cu22
1 files changed, 18 insertions, 4 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu
index 2cdb864..94887fb 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -685,8 +685,8 @@ __global__ static void layer_norm_kernel(float *out,
o[d] = (xr[d] - mean) * inv * gamma[d] + beta[d];
}
-// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). Not the tanh
-// approximation - BERT-family checkpoints use the exact form.
+// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). The released encoders
+// use this; the tanh approximation below is for gelu_new / gelu_pytorch_tanh.
__global__ static void gelu_kernel(float *x, size_t n) {
size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
if (i < n) {
@@ -695,6 +695,16 @@ __global__ static void gelu_kernel(float *x, size_t n) {
}
}
+// Tanh-approximation GeLU in place: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))).
+__global__ static void gelu_tanh_kernel(float *x, size_t n) {
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) {
+ float v = x[i];
+ float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
+ x[i] = 0.5f * v * (1.0f + tanhf(inner));
+ }
+}
+
// Add the learned absolute position embedding (indexed by the within-sequence
// position) and the token-type[0] embedding to the token embeddings already in
// x. token_type holds row 0 only; every input token has segment id 0.
@@ -1502,8 +1512,12 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i
ctx->ffn_gate_up, l->ffn_inter_bias, total, inter);
if (launch_check() != 0)
return -1;
- gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
- ctx->ffn_gate_up, (size_t)inter_count);
+ if (c->ffn_act == EMBED_ACT_GELU_TANH)
+ gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, (size_t)inter_count);
+ else
+ gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, (size_t)inter_count);
if (launch_check() != 0)
return -1;
if (linear_accum(ctx, &l->down_proj, ctx->ffn_gate_up, ctx->x, total, inter, hidden) != 0)