aboutsummaryrefslogtreecommitdiff
path: root/kernels.h
blob: cd719a7443186487a2054af203a1b21b4c465947 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
/* Dense kernels behind the forward pass.
 *
 * The whole model reduces to one primitive: several output time steps sharing
 * one weight matrix. The convolutions reach it because the activation buffers
 * hold time steps as consecutive rows with a zero guard row at each end, so
 * the three taps a kernel-3 convolution needs are already contiguous and each
 * output step is a plain dot product over a window that starts `x_stride`
 * floats later; conv weights are packed to match (tools/export_weights.py).
 *
 * Taking all the steps in one call is what makes the kernel fast. A matvec
 * touches each weight exactly once and cannot do better than memory, and this
 * model streams roughly a megabyte of weights per 32 ms chunk, so it runs at
 * the machine's read bandwidth. Calling the kernel once per time step would
 * re-read conv1's weights four times and conv2's twice, which is 1.6 MB per
 * chunk rather than 0.95 MB; folding the steps into one pass over the weights
 * removes that traffic outright, and the extra activation loads it costs come
 * from a few kilobytes that stay in L1.
 *
 * Both the scalar and the SIMD implementation are always compiled and selected
 * through this table, so a single binary can compare them. */

#ifndef SVAD_KERNELS_H
#define SVAD_KERNELS_H

#define SVAD_MAX_STEPS 4

typedef struct {
    const char *name;

    /* For s in 0..steps-1 and i in 0..m-1:
     *   y[s*y_stride + i] = (bias ? bias[i] : 0) + dot(W + i*n, x + s*x_stride, n)
     * n is a multiple of 4; steps is at most SVAD_MAX_STEPS. */
    void (*matvec)(float *y,
                   int y_stride,
                   const float *W,
                   const float *x,
                   int x_stride,
                   const float *bias,
                   int m,
                   int n,
                   int steps);

    float (*dot)(const float *a, const float *b, int n);
} svad_kernels;

const svad_kernels *svad_kernels_scalar(void);

/* The best set this build supports; equals the scalar set on architectures
 * without an implementation here. */
const svad_kernels *svad_kernels_simd(void);

#endif /* SVAD_KERNELS_H */