diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:19:53 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:19:53 +0200 |
| commit | ceecbea701e3ac882ce5f1693204152cc75c0d79 (patch) | |
| tree | 397fc18d66aa68235f008d8d54a85683adb1ff5b | |
| parent | 8e8836796e7dd22820c820c0916e83ecdbf38368 (diff) | |
| download | ffwd-ceecbea701e3ac882ce5f1693204152cc75c0d79.tar.gz ffwd-ceecbea701e3ac882ce5f1693204152cc75c0d79.zip | |
kernels: add LayerNorm and exact-GeLU for the BERT family
| -rw-r--r-- | qwen_kernels.c | 38 | ||||
| -rw-r--r-- | qwen_kernels.h | 15 | ||||
| -rw-r--r-- | tests/test_kernels.c | 52 |
3 files changed, 105 insertions, 0 deletions
diff --git a/qwen_kernels.c b/qwen_kernels.c index c7f5762..e1da7ed 100644 --- a/qwen_kernels.c +++ b/qwen_kernels.c @@ -768,6 +768,38 @@ void qwen_rms_norm_per_head( } } +/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each + * row's mean and variance are reduced before the row is rewritten. Scalar for + * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */ +void qwen_layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps) { + if (seq_len <= 0 || hidden <= 0) + return; + float inv_h = 1.0f / (float)hidden; + for (int s = 0; s < seq_len; s++) { + const float *row = x + (size_t)s * hidden; + float *dst = out + (size_t)s * hidden; + float mean = 0.0f; + for (int d = 0; d < hidden; d++) + mean += row[d]; + mean *= inv_h; + float var = 0.0f; + for (int d = 0; d < hidden; d++) { + float diff = row[d] - mean; + var += diff * diff; + } + var *= inv_h; + float inv_std = 1.0f / sqrtf(var + eps); + for (int d = 0; d < hidden; d++) + dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; + } +} + /* ======================================================================== * Activation Functions * ======================================================================== */ @@ -799,6 +831,12 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n) { #endif } +void qwen_gelu_inplace(float *x, int n) { + /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ + for (int i = 0; i < n; i++) + x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); +} + void qwen_softmax(float *x, int rows, int cols) { for (int r = 0; r < rows; r++) { float *row = x + r * cols; diff --git a/qwen_kernels.h b/qwen_kernels.h index 3a6a4d6..36d2913 100644 --- a/qwen_kernels.h +++ b/qwen_kernels.h @@ -88,6 +88,17 @@ void qwen_rms_norm( void qwen_rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); +/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps) + * + beta, with mean and biased (population) variance over the hidden axis per + * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ +void qwen_layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps); + /* ======================================================================== * Activation Functions * ======================================================================== */ @@ -96,6 +107,10 @@ void qwen_softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ void qwen_silu_mul_inplace(float *gate, const float *up, int n); +/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the + * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ +void qwen_gelu_inplace(float *x, int n); + /* ======================================================================== * Attention Operations * ======================================================================== */ diff --git a/tests/test_kernels.c b/tests/test_kernels.c index fb3d2df..0f92df6 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -58,6 +58,56 @@ static void test_rms_norm(void) { } } +static void test_layer_norm(void) { + const int seq = 2, hidden = 4; + const float x[8] = {1.0f, 2.0f, -3.0f, 4.0f, 0.5f, -1.0f, 2.0f, -0.5f}; + const float g[4] = {1.0f, 0.5f, -1.0f, 2.0f}; + const float b[4] = {0.1f, -0.2f, 0.3f, -0.4f}; + float out[8]; + + qwen_layer_norm(out, x, g, b, seq, hidden, 1e-12f); + + for (int s = 0; s < seq; s++) { + float mean = 0.0f; + for (int d = 0; d < hidden; d++) + mean += x[s * hidden + d]; + mean /= hidden; + float var = 0.0f; + for (int d = 0; d < hidden; d++) { + float diff = x[s * hidden + d] - mean; + var += diff * diff; + } + var /= hidden; + float inv = 1.0f / sqrtf(var + 1e-12f); + for (int d = 0; d < hidden; d++) { + float want = g[d] * (x[s * hidden + d] - mean) * inv + b[d]; + expect_close("layer_norm", out[s * hidden + d], want, 1e-5f); + } + } + + /* In-place (out aliases x) must match the out-of-place result. */ + float z[8]; + memcpy(z, x, sizeof(z)); + qwen_layer_norm(z, z, g, b, seq, hidden, 1e-12f); + for (int i = 0; i < 8; i++) + expect_close("layer_norm_inplace", z[i], out[i], 1e-6f); +} + +static void test_gelu(void) { + float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f}; + float out[5]; + memcpy(out, x, sizeof(out)); + qwen_gelu_inplace(out, 5); + for (int i = 0; i < 5; i++) { + float want = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); + expect_close("gelu", out[i], want, 1e-6f); + } + /* Known reference values for the exact erf GeLU. */ + expect_close("gelu0", out[0], 0.0f, 1e-7f); + expect_close("gelu1", out[1], 0.841344746f, 1e-5f); + expect_close("gelu_neg1", out[2], -0.158655254f, 1e-5f); +} + static void test_rope_neox(void) { const int seq = 2, heads = 1, head_dim = 4; const int positions[2] = {0, 3}; @@ -527,6 +577,8 @@ static void test_generic_vs_impl(void) { int main(void) { qwen_set_threads(1); test_rms_norm(); + test_layer_norm(); + test_gelu(); test_rope_neox(); test_packed_gqa_attention(); test_packed_gqa_attention_long(); |
