aboutsummaryrefslogtreecommitdiff
path: root/tests/test_kernels.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 09:26:20 +0200
commitc20279050f88d9c24187c3e402912596e1f54587 (patch)
tree2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /tests/test_kernels.c
parent849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff)
downloadffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz
ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'tests/test_kernels.c')
-rw-r--r--tests/test_kernels.c19
1 files changed, 19 insertions, 0 deletions
diff --git a/tests/test_kernels.c b/tests/test_kernels.c
index 0f92df6..4a9e9db 100644
--- a/tests/test_kernels.c
+++ b/tests/test_kernels.c
@@ -108,6 +108,24 @@ static void test_gelu(void) {
expect_close("gelu_neg1", out[2], -0.158655254f, 1e-5f);
}
+static void test_gelu_tanh(void) {
+ float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f};
+ float out[5];
+ memcpy(out, x, sizeof(out));
+ qwen_gelu_tanh_inplace(out, 5);
+ for (int i = 0; i < 5; i++) {
+ float v = x[i];
+ float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
+ float want = 0.5f * v * (1.0f + tanhf(inner));
+ expect_close("gelu_tanh", out[i], want, 1e-6f);
+ }
+ /* Reference values for the tanh approximation; distinct from the erf GeLU
+ * (erf gelu(1) = 0.841344746) by ~1.5e-4, well outside this tolerance. */
+ expect_close("gelu_tanh0", out[0], 0.0f, 1e-7f);
+ expect_close("gelu_tanh1", out[1], 0.841191991f, 1e-5f);
+ expect_close("gelu_tanh_neg1", out[2], -0.158808009f, 1e-5f);
+}
+
static void test_rope_neox(void) {
const int seq = 2, heads = 1, head_dim = 4;
const int positions[2] = {0, 3};
@@ -579,6 +597,7 @@ int main(void) {
test_rms_norm();
test_layer_norm();
test_gelu();
+ test_gelu_tanh();
test_rope_neox();
test_packed_gqa_attention();
test_packed_gqa_attention_long();