aboutsummaryrefslogtreecommitdiff
path: root/src/kernels.h
blob: 678b844d956b2614d308b0bdb234afc7109aaf29 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
/* The single dense primitive behind the whole forward pass.
 *
 * Every layer of this model computes, for a run of output positions, the dot
 * products of a window of activations against a fixed set of weight rows. The
 * convolutions reach that form because activations are stored time-major: the
 * taps a kernel-5 convolution needs for one output step are five consecutive
 * frames, which is one contiguous span, and the next output step starts one
 * frame later. The sinc filterbank is the same thing with a span of 251
 * samples advancing ten samples at a time. So the only parameter that changes
 * between layers is how far apart consecutive windows start, which is what
 * lda is; it is unrelated to the window length k.
 *
 * Taking every output position in one call is what makes this fast. A single
 * position would re-read the whole weight matrix; taking four at a time reuses
 * each loaded weight vector four times, and the outer panel loop keeps the
 * activations that those four rows share resident in L1 while the weights
 * stream past once.
 *
 * Both implementations are always compiled and chosen through this table, so
 * one binary can compare them.
 */

#ifndef PYSEG_KERNELS_H
#define PYSEG_KERNELS_H

/* Output rows processed against the full weight matrix before moving on. Sized
 * so the activation window and the output panel together stay inside L1. */
#define PYSEG_MPANEL 32

typedef struct {
    const char *name;

    /* For s in 0..m-1 and i in 0..n-1:
     *   y[s*ldy + i] = (bias ? bias[i] : 0) + dot(a + s*lda, w + i*k, k)
     * k must be a multiple of four. */
    void (*gemm_nt)(float *y,
                    int ldy,
                    const float *a,
                    int lda,
                    const float *w,
                    const float *bias,
                    int m,
                    int n,
                    int k);

    /* One LSTM cell update in place. The preactivations are the sum of rec,
     * the hidden-to-hidden product, and in, the projected input for this
     * time step; both hold the four gate blocks in PyTorch's order, input then
     * forget then candidate then output, each hid long:
     *   c = sigmoid(f) * c + sigmoid(i) * tanh(g)
     *   h = sigmoid(o) * tanh(c)
     *
     * This is here rather than in the forward pass because it is not a
     * negligible epilogue. A window drives 4712 cell updates, which is three
     * million logistic and hyperbolic evaluations; called one at a time
     * through libm that costs more than either convolution stage. hid is a
     * multiple of four. */
    void (*lstm_cell)(float *h, float *c, const float *rec, const float *in, int hid);
} pyseg_kernels;

const pyseg_kernels *pyseg_kernels_scalar(void);

/* The best set this build supports; equals the scalar set on architectures
 * without an implementation here. */
const pyseg_kernels *pyseg_kernels_simd(void);

#endif /* PYSEG_KERNELS_H */