aboutsummaryrefslogtreecommitdiff
path: root/qwen_kernels.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 15:19:53 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 15:19:53 +0200
commitceecbea701e3ac882ce5f1693204152cc75c0d79 (patch)
tree397fc18d66aa68235f008d8d54a85683adb1ff5b /qwen_kernels.h
parent8e8836796e7dd22820c820c0916e83ecdbf38368 (diff)
downloadffwd-ceecbea701e3ac882ce5f1693204152cc75c0d79.tar.gz
ffwd-ceecbea701e3ac882ce5f1693204152cc75c0d79.zip
kernels: add LayerNorm and exact-GeLU for the BERT family
Diffstat (limited to 'qwen_kernels.h')
-rw-r--r--qwen_kernels.h15
1 files changed, 15 insertions, 0 deletions
diff --git a/qwen_kernels.h b/qwen_kernels.h
index 3a6a4d6..36d2913 100644
--- a/qwen_kernels.h
+++ b/qwen_kernels.h
@@ -88,6 +88,17 @@ void qwen_rms_norm(
void qwen_rms_norm_per_head(
float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps);
+/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps)
+ * + beta, with mean and biased (population) variance over the hidden axis per
+ * row. Unlike RMSNorm this subtracts the mean and adds a bias. */
+void qwen_layer_norm(float *out,
+ const float *x,
+ const float *gamma,
+ const float *beta,
+ int seq_len,
+ int hidden,
+ float eps);
+
/* ========================================================================
* Activation Functions
* ======================================================================== */
@@ -96,6 +107,10 @@ void qwen_softmax(float *x, int rows, int cols);
/* gate = SiLU(gate) * up */
void qwen_silu_mul_inplace(float *gate, const float *up, int n);
+/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the
+ * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */
+void qwen_gelu_inplace(float *x, int n);
+
/* ========================================================================
* Attention Operations
* ======================================================================== */