/* The single dense primitive behind the whole forward pass. * * Every layer of this model computes, for a run of output positions, the dot * products of a window of activations against a fixed set of weight rows. The * convolutions reach that form because activations are stored time-major: the * taps a kernel-5 convolution needs for one output step are five consecutive * frames, which is one contiguous span, and the next output step starts one * frame later. The sinc filterbank is the same thing with a span of 251 * samples advancing ten samples at a time. So the only parameter that changes * between layers is how far apart consecutive windows start, which is what * lda is; it is unrelated to the window length k. * * Taking every output position in one call is what makes this fast. A single * position would re-read the whole weight matrix; taking four at a time reuses * each loaded weight vector four times, and the outer panel loop keeps the * activations that those four rows share resident in L1 while the weights * stream past once. * * Both implementations are always compiled and chosen through this table, so * one binary can compare them. */ #ifndef PYSEG_KERNELS_H #define PYSEG_KERNELS_H /* Output rows processed against the full weight matrix before moving on. Sized * so the activation window and the output panel together stay inside L1. */ #define PYSEG_MPANEL 32 typedef struct { const char *name; /* For s in 0..m-1 and i in 0..n-1: * y[s*ldy + i] = (bias ? bias[i] : 0) + dot(a + s*lda, w + i*k, k) * k must be a multiple of four. */ void (*gemm_nt)(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int m, int n, int k); /* One LSTM cell update in place. The preactivations are the sum of rec, * the hidden-to-hidden product, and in, the projected input for this * time step; both hold the four gate blocks in PyTorch's order, input then * forget then candidate then output, each hid long: * c = sigmoid(f) * c + sigmoid(i) * tanh(g) * h = sigmoid(o) * tanh(c) * * This is here rather than in the forward pass because it is not a * negligible epilogue. A window drives 4712 cell updates, which is three * million logistic and hyperbolic evaluations; called one at a time * through libm that costs more than either convolution stage. hid is a * multiple of four. */ void (*lstm_cell)(float *h, float *c, const float *rec, const float *in, int hid); } pyseg_kernels; const pyseg_kernels *pyseg_kernels_scalar(void); /* The best set this build supports; equals the scalar set on architectures * without an implementation here. */ const pyseg_kernels *pyseg_kernels_simd(void); #endif /* PYSEG_KERNELS_H */