diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 09:26:20 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 09:26:20 +0200 |
| commit | c20279050f88d9c24187c3e402912596e1f54587 (patch) | |
| tree | 2750fc95cca51258c1ceb31cfcce6c2a46cb2bc7 /tests/test_kernels.c | |
| parent | 849c571a8e79d92c1fd3dabecf0fb3621aa4f59b (diff) | |
| download | ffwd-c20279050f88d9c24187c3e402912596e1f54587.tar.gz ffwd-c20279050f88d9c24187c3e402912596e1f54587.zip | |
all: support BERT tanh-GeLU (gelu_new / gelu_pytorch_tanh)
Diffstat (limited to 'tests/test_kernels.c')
| -rw-r--r-- | tests/test_kernels.c | 19 |
1 files changed, 19 insertions, 0 deletions
diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 0f92df6..4a9e9db 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -108,6 +108,24 @@ static void test_gelu(void) { expect_close("gelu_neg1", out[2], -0.158655254f, 1e-5f); } +static void test_gelu_tanh(void) { + float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f}; + float out[5]; + memcpy(out, x, sizeof(out)); + qwen_gelu_tanh_inplace(out, 5); + for (int i = 0; i < 5; i++) { + float v = x[i]; + float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); + float want = 0.5f * v * (1.0f + tanhf(inner)); + expect_close("gelu_tanh", out[i], want, 1e-6f); + } + /* Reference values for the tanh approximation; distinct from the erf GeLU + * (erf gelu(1) = 0.841344746) by ~1.5e-4, well outside this tolerance. */ + expect_close("gelu_tanh0", out[0], 0.0f, 1e-7f); + expect_close("gelu_tanh1", out[1], 0.841191991f, 1e-5f); + expect_close("gelu_tanh_neg1", out[2], -0.158808009f, 1e-5f); +} + static void test_rope_neox(void) { const int seq = 2, heads = 1, head_dim = 4; const int positions[2] = {0, 3}; @@ -579,6 +597,7 @@ int main(void) { test_rms_norm(); test_layer_norm(); test_gelu(); + test_gelu_tanh(); test_rope_neox(); test_packed_gqa_attention(); test_packed_gqa_attention_long(); |
