blob: 59e8bdbd05306ecd7542111ec4cf419b6289948d (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
#include "kernels.h"
#include <math.h>
/* Safe in place: each row's mean and variance are reduced before it is rewritten. */
void layer_norm(float *out,
const float *x,
const float *gamma,
const float *beta,
int seq_len,
int hidden,
float eps) {
if (seq_len <= 0 || hidden <= 0)
return;
float inv_h = 1.0f / (float)hidden;
for (int s = 0; s < seq_len; s++) {
const float *row = x + (size_t)s * hidden;
float *dst = out + (size_t)s * hidden;
float mean = 0.0f;
for (int d = 0; d < hidden; d++)
mean += row[d];
mean *= inv_h;
float var = 0.0f;
for (int d = 0; d < hidden; d++) {
float diff = row[d] - mean;
var += diff * diff;
}
var *= inv_h;
float inv_std = 1.0f / sqrtf(var + eps);
for (int d = 0; d < hidden; d++)
dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d];
}
}
|