1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
|
/* Dense kernels behind the forward pass.
*
* The whole model reduces to one primitive: several output time steps sharing
* one weight matrix. The convolutions reach it because the activation buffers
* hold time steps as consecutive rows with a zero guard row at each end, so
* the three taps a kernel-3 convolution needs are already contiguous and each
* output step is a plain dot product over a window that starts `x_stride`
* floats later; conv weights are packed to match (tools/export_weights.py).
*
* Taking all the steps in one call is what makes the kernel fast. A matvec
* touches each weight exactly once and cannot do better than memory, and this
* model streams roughly a megabyte of weights per 32 ms chunk, so it runs at
* the machine's read bandwidth. Calling the kernel once per time step would
* re-read conv1's weights four times and conv2's twice, which is 1.6 MB per
* chunk rather than 0.95 MB; folding the steps into one pass over the weights
* removes that traffic outright, and the extra activation loads it costs come
* from a few kilobytes that stay in L1.
*
* Both the scalar and the SIMD implementation are always compiled and selected
* through this table, so a single binary can compare them. */
#ifndef SVAD_KERNELS_H
#define SVAD_KERNELS_H
#define SVAD_MAX_STEPS 4
typedef struct {
const char *name;
/* For s in 0..steps-1 and i in 0..m-1:
* y[s*y_stride + i] = (bias ? bias[i] : 0) + dot(W + i*n, x + s*x_stride, n)
* n is a multiple of 4; steps is at most SVAD_MAX_STEPS. */
void (*matvec)(float *y,
int y_stride,
const float *W,
const float *x,
int x_stride,
const float *bias,
int m,
int n,
int steps);
float (*dot)(const float *a, const float *b, int n);
} svad_kernels;
const svad_kernels *svad_kernels_scalar(void);
/* The best set this build supports; equals the scalar set on architectures
* without an implementation here. */
const svad_kernels *svad_kernels_simd(void);
#endif /* SVAD_KERNELS_H */
|