diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-27 00:58:05 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-27 01:03:41 +0200 |
| commit | af55803d0dff9eb692c760c8d486e736c945061d (patch) | |
| tree | 0875a1112bf98304e445e0544c4309670d6f4769 /libffwd/kernels.h | |
| parent | 0f861b7f3098750ddee4f6570f3be228d1da5320 (diff) | |
| download | ffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip | |
chore: clean up comment formatting and wording
Diffstat (limited to 'libffwd/kernels.h')
| -rw-r--r-- | libffwd/kernels.h | 37 |
1 files changed, 13 insertions, 24 deletions
diff --git a/libffwd/kernels.h b/libffwd/kernels.h index 2a917e0..11312db 100644 --- a/libffwd/kernels.h +++ b/libffwd/kernels.h @@ -1,5 +1,5 @@ /* - * kernels.h - tkern public API: transformer inference kernels (CPU, float32) + * Transformer inference kernels (CPU, float32). * * Low-level math operations. All operate on float32 tensors in row-major order. */ @@ -10,15 +10,11 @@ #include <stddef.h> #include <stdint.h> -/* ======================================================================== - * Basic Operations - * ======================================================================== */ +/* Basic operations */ void add_inplace(float *a, const float *b, int n); -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ +/* Matrix operations */ /* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ void matmul_t(float *C, const float *A, const float *B, int M, int K, int N); @@ -62,9 +58,7 @@ void linear_nobias_bf16_qkv(float *q, /* Dot product using the best available local SIMD implementation. */ float dot_f32(const float *a, const float *b, int n); -/* ======================================================================== - * Normalization - * ======================================================================== */ +/* Normalization */ /* RMS Normalization: out = x / rms(x) * weight */ void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); @@ -75,9 +69,10 @@ void rms_norm(float *out, const float *x, const float *weight, int seq_len, int void rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); -/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps) - * + beta, with mean and biased (population) variance over the hidden axis per - * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ +/* Layer normalization for BERT-family models: + * out = gamma * (x - mean) / sqrt(var + eps) + beta. + * Mean and biased/population variance are computed over the hidden axis per row. + * Unlike RMSNorm, this subtracts the mean and adds a bias. */ void layer_norm(float *out, const float *x, const float *gamma, @@ -86,25 +81,21 @@ void layer_norm(float *out, int hidden, float eps); -/* ======================================================================== - * Activation Functions - * ======================================================================== */ +/* Activation functions */ void softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ void silu_mul_inplace(float *gate, const float *up, int n); -/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the - * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ +/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). + * This is the "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ void gelu_inplace(float *x, int n); /* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh): * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */ void gelu_tanh_inplace(float *x, int n); -/* ======================================================================== - * Attention Operations - * ======================================================================== */ +/* Attention operations */ /* * Bidirectional GQA attention over a packed/ragged batch. @@ -169,9 +160,7 @@ void causal_gqa_attention_packed_with_scratch(float *out, float *scratch, size_t scratch_bytes); -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ +/* Position embeddings */ /* * NeoX-style RoPE: compute cos/sin for positions. |
