1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
/* The single dense primitive behind the whole forward pass.
*
* Every layer of this model computes, for a run of output positions, the dot
* products of a window of activations against a fixed set of weight rows. The
* convolutions reach that form because activations are stored time-major: the
* taps a kernel-5 convolution needs for one output step are five consecutive
* frames, which is one contiguous span, and the next output step starts one
* frame later. The sinc filterbank is the same thing with a span of 251
* samples advancing ten samples at a time. So the only parameter that changes
* between layers is how far apart consecutive windows start, which is what
* lda is; it is unrelated to the window length k.
*
* Taking every output position in one call is what makes this fast. A single
* position would re-read the whole weight matrix; taking four at a time reuses
* each loaded weight vector four times, and the outer panel loop keeps the
* activations that those four rows share resident in L1 while the weights
* stream past once.
*
* Both implementations are always compiled and chosen through this table, so
* one binary can compare them.
*/
#ifndef PYSEG_KERNELS_H
#define PYSEG_KERNELS_H
/* Output rows processed against the full weight matrix before moving on. Sized
* so the activation window and the output panel together stay inside L1. */
#define PYSEG_MPANEL 32
typedef struct {
const char *name;
/* For s in 0..m-1 and i in 0..n-1:
* y[s*ldy + i] = (bias ? bias[i] : 0) + dot(a + s*lda, w + i*k, k)
* k must be a multiple of four. */
void (*gemm_nt)(float *y,
int ldy,
const float *a,
int lda,
const float *w,
const float *bias,
int m,
int n,
int k);
/* One LSTM cell update in place. The preactivations are the sum of rec,
* the hidden-to-hidden product, and in, the projected input for this
* time step; both hold the four gate blocks in PyTorch's order, input then
* forget then candidate then output, each hid long:
* c = sigmoid(f) * c + sigmoid(i) * tanh(g)
* h = sigmoid(o) * tanh(c)
*
* This is here rather than in the forward pass because it is not a
* negligible epilogue. A window drives 4712 cell updates, which is three
* million logistic and hyperbolic evaluations; called one at a time
* through libm that costs more than either convolution stage. hid is a
* multiple of four. */
void (*lstm_cell)(float *h, float *c, const float *rec, const float *in, int hid);
} pyseg_kernels;
const pyseg_kernels *pyseg_kernels_scalar(void);
/* The best set this build supports; equals the scalar set on architectures
* without an implementation here. */
const pyseg_kernels *pyseg_kernels_simd(void);
#endif /* PYSEG_KERNELS_H */
|