diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:19:53 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 15:19:53 +0200 |
| commit | ceecbea701e3ac882ce5f1693204152cc75c0d79 (patch) | |
| tree | 397fc18d66aa68235f008d8d54a85683adb1ff5b /qwen_kernels.h | |
| parent | 8e8836796e7dd22820c820c0916e83ecdbf38368 (diff) | |
| download | ffwd-ceecbea701e3ac882ce5f1693204152cc75c0d79.tar.gz ffwd-ceecbea701e3ac882ce5f1693204152cc75c0d79.zip | |
kernels: add LayerNorm and exact-GeLU for the BERT family
Diffstat (limited to 'qwen_kernels.h')
| -rw-r--r-- | qwen_kernels.h | 15 |
1 files changed, 15 insertions, 0 deletions
diff --git a/qwen_kernels.h b/qwen_kernels.h index 3a6a4d6..36d2913 100644 --- a/qwen_kernels.h +++ b/qwen_kernels.h @@ -88,6 +88,17 @@ void qwen_rms_norm( void qwen_rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); +/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps) + * + beta, with mean and biased (population) variance over the hidden axis per + * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ +void qwen_layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps); + /* ======================================================================== * Activation Functions * ======================================================================== */ @@ -96,6 +107,10 @@ void qwen_softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ void qwen_silu_mul_inplace(float *gate, const float *up, int n); +/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the + * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ +void qwen_gelu_inplace(float *x, int n); + /* ======================================================================== * Attention Operations * ======================================================================== */ |
