aboutsummaryrefslogtreecommitdiff
path: root/libffwd/norm.c
blob: 57c93dbd2daa2eb5a1e6a0ffb767ead72a74bfb8 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
/*
 * Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the
 * per-head RMSNorm used for QK-norm.
 */

#include "kernels.h"
#include "threadpool.h"

#include <math.h>
#include <stdlib.h>
#include <string.h>

#ifdef USE_BLAS
#    ifdef __APPLE__
#        include <Accelerate/Accelerate.h>
#    else
#        include <cblas.h>
#    endif
#endif

#ifndef M_PI
#    define M_PI 3.14159265358979323846
#endif

/* Normalization */

/* Rows split across the pool only when the tensor is big enough to pay
 * for the dispatch. */
#define RMS_NORM_PARALLEL_ELEMS 262144

/* The sum of squares and the scaled write-back are plain loops: the compiler
 * auto-vectorizes both in place, which is faster than reducing through a
 * two-input dot kernel that reloads the row and is reduction-latency bound. */
static void rms_norm_range(
    float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) {
    for (int s = start; s < end; s++) {
        const float *x_row = x + (size_t)s * hidden;
        float *out_row = out + (size_t)s * hidden;

        float sum_sq = 0.0f;
        for (int i = 0; i < hidden; i++)
            sum_sq += x_row[i] * x_row[i];
        float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps);
        for (int i = 0; i < hidden; i++)
            out_row[i] = x_row[i] * rms_inv * weight[i];
    }
}

typedef struct {
    float *out;
    const float *x;
    const float *weight;
    int seq_len;
    int hidden;
    float eps;
} rms_norm_task_t;

static void rms_norm_worker(int tid, int n_threads, void *arg) {
    rms_norm_task_t *t = (rms_norm_task_t *)arg;
    int chunk = (t->seq_len + n_threads - 1) / n_threads;
    int start = tid * chunk;
    int end = start + chunk;
    if (end > t->seq_len)
        end = t->seq_len;
    if (start >= end)
        return;
    rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps);
}

void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) {
    if (seq_len <= 0 || hidden <= 0)
        return;

    long long elems = (long long)seq_len * hidden;
    if (tp_num_threads() > 1 && elems >= RMS_NORM_PARALLEL_ELEMS) {
        rms_norm_task_t task = {
            .out = out,
            .x = x,
            .weight = weight,
            .seq_len = seq_len,
            .hidden = hidden,
            .eps = eps,
        };
        parallel_for(rms_norm_worker, &task);
    } else {
        rms_norm_range(out, x, weight, 0, seq_len, hidden, eps);
    }
}

void rms_norm_per_head(
    float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) {
    /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */
    int hidden = n_heads * head_dim;
    for (int s = 0; s < seq_len; s++) {
        for (int h = 0; h < n_heads; h++) {
            float *vec = x + s * hidden + h * head_dim;

            float sum_sq = 0.0f;
            for (int d = 0; d < head_dim; d++)
                sum_sq += vec[d] * vec[d];
            float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps);
            for (int d = 0; d < head_dim; d++)
                vec[d] = vec[d] * rms_inv * weight[d];
        }
    }
}

/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each
 * row's mean and variance are reduced before the row is rewritten. Scalar for
 * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */
void layer_norm(float *out,
                const float *x,
                const float *gamma,
                const float *beta,
                int seq_len,
                int hidden,
                float eps) {
    if (seq_len <= 0 || hidden <= 0)
        return;
    float inv_h = 1.0f / (float)hidden;
    for (int s = 0; s < seq_len; s++) {
        const float *row = x + (size_t)s * hidden;
        float *dst = out + (size_t)s * hidden;
        float mean = 0.0f;
        for (int d = 0; d < hidden; d++)
            mean += row[d];
        mean *= inv_h;
        float var = 0.0f;
        for (int d = 0; d < hidden; d++) {
            float diff = row[d] - mean;
            var += diff * diff;
        }
        var *= inv_h;
        float inv_std = 1.0f / sqrtf(var + eps);
        for (int d = 0; d < hidden; d++)
            dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d];
    }
}