aboutsummaryrefslogtreecommitdiff
path: root/src/kernels.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/kernels.c')
-rw-r--r--src/kernels.c555
1 files changed, 555 insertions, 0 deletions
diff --git a/src/kernels.c b/src/kernels.c
new file mode 100644
index 0000000..01c1d19
--- /dev/null
+++ b/src/kernels.c
@@ -0,0 +1,555 @@
+/* Scalar and SIMD implementations of the dense primitive.
+ *
+ * Each variant blocks a rectangle of output positions against weight rows and
+ * accumulates along the window, and the rectangle is chosen by the register
+ * file rather than by the algorithm. AArch64 has 32 vector registers, so four
+ * positions by four rows fits: sixteen accumulators plus eight live operands,
+ * eight loads for sixteen multiply-adds. x86-64 has 16, so the same rectangle
+ * spills its accumulators to the stack every iteration and four by two is
+ * faster despite doing half as much per pass; on one server core that
+ * difference was a factor of 1.8 over a whole recording.
+ *
+ * Matrix-vector shapes, meaning the LSTM recurrence where there is only one
+ * output position, widen instead to eight weight rows. With a single
+ * activation window there is nothing to reuse but that window, so the weight
+ * loads are the entire cost and the only thing worth amortising is the
+ * activation.
+ *
+ * Accumulation order differs between the variants, so their results differ in
+ * the last bits; tests/test_kernels.c bounds that.
+ */
+
+#include "kernels.h"
+
+#include <math.h>
+#include <stddef.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+# define PYSEG_INLINE static inline __attribute__((always_inline))
+#else
+# define PYSEG_INLINE static inline
+#endif
+
+static inline int min_int(int a, int b) { return a < b ? a : b; }
+
+/* ---- portable C ------------------------------------------------------ */
+
+PYSEG_INLINE void
+dot4x4_scalar(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) {
+ float acc[4][4] = {{0.0f}};
+ for (int j = 0; j < k; ++j) {
+ const float a0 = a[j], a1 = a[lda + j], a2 = a[2 * lda + j], a3 = a[3 * lda + j];
+ for (int i = 0; i < 4; ++i) {
+ const float wv = w[(size_t)i * k + j];
+ acc[0][i] += a0 * wv;
+ acc[1][i] += a1 * wv;
+ acc[2][i] += a2 * wv;
+ acc[3][i] += a3 * wv;
+ }
+ }
+ for (int s = 0; s < 4; ++s)
+ for (int i = 0; i < 4; ++i)
+ y[(size_t)s * ldy + i] = bias ? bias[i] + acc[s][i] : acc[s][i];
+}
+
+/* One output position against however many weight rows are left. */
+PYSEG_INLINE void
+row_scalar(float *y, const float *a, const float *w, const float *bias, int n, int k) {
+ for (int i = 0; i < n; ++i) {
+ const float *wr = w + (size_t)i * k;
+ float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f;
+ int j = 0;
+ for (; j + 4 <= k; j += 4) {
+ s0 += a[j + 0] * wr[j + 0];
+ s1 += a[j + 1] * wr[j + 1];
+ s2 += a[j + 2] * wr[j + 2];
+ s3 += a[j + 3] * wr[j + 3];
+ }
+ const float s = (s0 + s1) + (s2 + s3);
+ y[i] = bias ? bias[i] + s : s;
+ }
+}
+
+static void gemm_nt_scalar(float *y,
+ int ldy,
+ const float *a,
+ int lda,
+ const float *w,
+ const float *bias,
+ int m,
+ int n,
+ int k) {
+ for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) {
+ const int mend = min_int(m0 + PYSEG_MPANEL, m);
+ const int m4 = m0 + (mend - m0) / 4 * 4;
+ int i = 0;
+ for (; i + 4 <= n; i += 4)
+ for (int s = m0; s < m4; s += 4)
+ dot4x4_scalar(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda,
+ w + (size_t)i * k, bias ? bias + i : NULL, k);
+ for (int s = m0; s < m4; ++s)
+ if (i < n)
+ row_scalar(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k,
+ bias ? bias + i : NULL, n - i, k);
+ for (int s = m4; s < mend; ++s)
+ row_scalar(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k);
+ }
+}
+
+static void lstm_cell_scalar(float *h, float *c, const float *rec, const float *in, int hid) {
+ for (int i = 0; i < hid; ++i) {
+ const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i])));
+ const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i])));
+ const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]);
+ const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i])));
+ const float cn = fg * c[i] + ig * gg;
+ c[i] = cn;
+ h[i] = og * tanhf(cn);
+ }
+}
+
+static const pyseg_kernels kernels_scalar = {"scalar", gemm_nt_scalar, lstm_cell_scalar};
+
+const pyseg_kernels *pyseg_kernels_scalar(void) { return &kernels_scalar; }
+
+/* ---- AArch64 NEON ---------------------------------------------------- */
+
+#if defined(__aarch64__) && defined(__ARM_NEON)
+# define PYSEG_HAVE_SIMD 1
+# include <arm_neon.h>
+
+/* Four lane-wise accumulators reduced to one vector holding their four totals.
+ * Pairwise adds do this in three instructions instead of four cross-lane
+ * reductions plus the moves back into a vector. */
+PYSEG_INLINE float32x4_t hsum4(float32x4_t a, float32x4_t b, float32x4_t c, float32x4_t d) {
+ return vpaddq_f32(vpaddq_f32(a, b), vpaddq_f32(c, d));
+}
+
+PYSEG_INLINE void
+dot4x4_neon(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) {
+ const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda;
+ const float *w0 = w, *w1 = w + k, *w2 = w + 2 * k, *w3 = w + 3 * k;
+
+ float32x4_t c00 = vdupq_n_f32(0.0f), c01 = c00, c02 = c00, c03 = c00;
+ float32x4_t c10 = c00, c11 = c00, c12 = c00, c13 = c00;
+ float32x4_t c20 = c00, c21 = c00, c22 = c00, c23 = c00;
+ float32x4_t c30 = c00, c31 = c00, c32 = c00, c33 = c00;
+
+ for (int j = 0; j < k; j += 4) {
+ const float32x4_t x0 = vld1q_f32(a0 + j);
+ const float32x4_t x1 = vld1q_f32(a1 + j);
+ const float32x4_t x2 = vld1q_f32(a2 + j);
+ const float32x4_t x3 = vld1q_f32(a3 + j);
+ const float32x4_t v0 = vld1q_f32(w0 + j);
+ const float32x4_t v1 = vld1q_f32(w1 + j);
+ const float32x4_t v2 = vld1q_f32(w2 + j);
+ const float32x4_t v3 = vld1q_f32(w3 + j);
+ c00 = vfmaq_f32(c00, x0, v0);
+ c01 = vfmaq_f32(c01, x0, v1);
+ c02 = vfmaq_f32(c02, x0, v2);
+ c03 = vfmaq_f32(c03, x0, v3);
+ c10 = vfmaq_f32(c10, x1, v0);
+ c11 = vfmaq_f32(c11, x1, v1);
+ c12 = vfmaq_f32(c12, x1, v2);
+ c13 = vfmaq_f32(c13, x1, v3);
+ c20 = vfmaq_f32(c20, x2, v0);
+ c21 = vfmaq_f32(c21, x2, v1);
+ c22 = vfmaq_f32(c22, x2, v2);
+ c23 = vfmaq_f32(c23, x2, v3);
+ c30 = vfmaq_f32(c30, x3, v0);
+ c31 = vfmaq_f32(c31, x3, v1);
+ c32 = vfmaq_f32(c32, x3, v2);
+ c33 = vfmaq_f32(c33, x3, v3);
+ }
+
+ const float32x4_t b = bias ? vld1q_f32(bias) : vdupq_n_f32(0.0f);
+ vst1q_f32(y, vaddq_f32(b, hsum4(c00, c01, c02, c03)));
+ vst1q_f32(y + ldy, vaddq_f32(b, hsum4(c10, c11, c12, c13)));
+ vst1q_f32(y + 2 * ldy, vaddq_f32(b, hsum4(c20, c21, c22, c23)));
+ vst1q_f32(y + 3 * ldy, vaddq_f32(b, hsum4(c30, c31, c32, c33)));
+}
+
+/* Single output position, eight weight rows at a time.
+ *
+ * With one activation window there is nothing to reuse but the window itself,
+ * so this shape costs one weight load per multiply-add and runs at whatever
+ * rate the memory system delivers weights, not at the multiply pipes' rate.
+ * Eight rows measured fastest here: four leaves the load pipeline too shallow,
+ * and sixteen has enough concurrent streams to start working against the
+ * hardware prefetcher. */
+PYSEG_INLINE void
+row_neon(float *y, const float *a, const float *w, const float *bias, int n, int k) {
+ int i = 0;
+ for (; i + 8 <= n; i += 8) {
+ const float *w0 = w + (size_t)i * k;
+ float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0;
+ float32x4_t c4 = c0, c5 = c0, c6 = c0, c7 = c0;
+ for (int j = 0; j < k; j += 4) {
+ const float32x4_t x = vld1q_f32(a + j);
+ c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j));
+ c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j));
+ c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j));
+ c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j));
+ c4 = vfmaq_f32(c4, x, vld1q_f32(w0 + 4 * k + j));
+ c5 = vfmaq_f32(c5, x, vld1q_f32(w0 + 5 * k + j));
+ c6 = vfmaq_f32(c6, x, vld1q_f32(w0 + 6 * k + j));
+ c7 = vfmaq_f32(c7, x, vld1q_f32(w0 + 7 * k + j));
+ }
+ const float32x4_t lo = hsum4(c0, c1, c2, c3);
+ const float32x4_t hi = hsum4(c4, c5, c6, c7);
+ if (bias) {
+ vst1q_f32(y + i, vaddq_f32(vld1q_f32(bias + i), lo));
+ vst1q_f32(y + i + 4, vaddq_f32(vld1q_f32(bias + i + 4), hi));
+ } else {
+ vst1q_f32(y + i, lo);
+ vst1q_f32(y + i + 4, hi);
+ }
+ }
+ for (; i + 4 <= n; i += 4) {
+ const float *w0 = w + (size_t)i * k;
+ float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0;
+ for (int j = 0; j < k; j += 4) {
+ const float32x4_t x = vld1q_f32(a + j);
+ c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j));
+ c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j));
+ c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j));
+ c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j));
+ }
+ const float32x4_t v = hsum4(c0, c1, c2, c3);
+ vst1q_f32(y + i, bias ? vaddq_f32(vld1q_f32(bias + i), v) : v);
+ }
+ for (; i < n; ++i) {
+ const float *wr = w + (size_t)i * k;
+ float32x4_t c0 = vdupq_n_f32(0.0f);
+ for (int j = 0; j < k; j += 4)
+ c0 = vfmaq_f32(c0, vld1q_f32(a + j), vld1q_f32(wr + j));
+ const float s = vaddvq_f32(c0);
+ y[i] = bias ? bias[i] + s : s;
+ }
+}
+
+static void gemm_nt_neon(float *y,
+ int ldy,
+ const float *a,
+ int lda,
+ const float *w,
+ const float *bias,
+ int m,
+ int n,
+ int k) {
+ if (m == 1) {
+ row_neon(y, a, w, bias, n, k);
+ return;
+ }
+ for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) {
+ const int mend = min_int(m0 + PYSEG_MPANEL, m);
+ const int m4 = m0 + (mend - m0) / 4 * 4;
+ int i = 0;
+ for (; i + 4 <= n; i += 4)
+ for (int s = m0; s < m4; s += 4)
+ dot4x4_neon(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k,
+ bias ? bias + i : NULL, k);
+ for (int s = m0; s < m4; ++s)
+ if (i < n)
+ row_neon(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k,
+ bias ? bias + i : NULL, n - i, k);
+ for (int s = m4; s < mend; ++s)
+ row_neon(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k);
+ }
+}
+
+/* Vector exponential, the Cephes single-precision scheme: split the argument
+ * into an integer power of two and a remainder inside half a ln 2, evaluate a
+ * degree-six polynomial on the remainder, then scale by assembling the power
+ * of two directly in the exponent field. Accurate to about one unit in the
+ * last place, which is the same claim libm makes.
+ *
+ * The clamp is what keeps the exponent assembly in range; beyond it the
+ * logistic and hyperbolic functions built on this are saturated anyway. */
+PYSEG_INLINE float32x4_t vexpq(float32x4_t x) {
+ x = vminq_f32(vmaxq_f32(x, vdupq_n_f32(-88.0f)), vdupq_n_f32(88.0f));
+
+ const float32x4_t n = vrndnq_f32(vmulq_f32(x, vdupq_n_f32(1.44269504088896341f)));
+ /* ln 2 split in two so the product with n stays exact in the high part. */
+ float32x4_t r = vfmsq_f32(x, n, vdupq_n_f32(0.693359375f));
+ r = vfmsq_f32(r, n, vdupq_n_f32(-2.12194440e-4f));
+
+ float32x4_t p = vdupq_n_f32(1.9875691500e-4f);
+ p = vfmaq_f32(vdupq_n_f32(1.3981999507e-3f), p, r);
+ p = vfmaq_f32(vdupq_n_f32(8.3334519073e-3f), p, r);
+ p = vfmaq_f32(vdupq_n_f32(4.1665795894e-2f), p, r);
+ p = vfmaq_f32(vdupq_n_f32(1.6666665459e-1f), p, r);
+ p = vfmaq_f32(vdupq_n_f32(5.0000001201e-1f), p, r);
+ p = vfmaq_f32(vaddq_f32(r, vdupq_n_f32(1.0f)), p, vmulq_f32(r, r));
+
+ const int32x4_t bits = vshlq_n_s32(vaddq_s32(vcvtq_s32_f32(n), vdupq_n_s32(127)), 23);
+ return vmulq_f32(p, vreinterpretq_f32_s32(bits));
+}
+
+PYSEG_INLINE float32x4_t vsigmoidq(float32x4_t x) {
+ return vdivq_f32(vdupq_n_f32(1.0f), vaddq_f32(vdupq_n_f32(1.0f), vexpq(vnegq_f32(x))));
+}
+
+/* tanh x = 2 sigmoid(2x) - 1. The subtraction loses relative accuracy near
+ * zero, where sigmoid is close to a half, but the absolute error stays at the
+ * rounding level of the result and that is what propagates into the cell
+ * state. */
+PYSEG_INLINE float32x4_t vtanhq(float32x4_t x) {
+ return vfmsq_f32(vdupq_n_f32(-1.0f), vsigmoidq(vaddq_f32(x, x)), vdupq_n_f32(-2.0f));
+}
+
+static void lstm_cell_neon(float *h, float *c, const float *rec, const float *in, int hid) {
+ for (int i = 0; i < hid; i += 4) {
+ const float32x4_t ig = vsigmoidq(vaddq_f32(vld1q_f32(rec + i), vld1q_f32(in + i)));
+ const float32x4_t fg =
+ vsigmoidq(vaddq_f32(vld1q_f32(rec + hid + i), vld1q_f32(in + hid + i)));
+ const float32x4_t gg =
+ vtanhq(vaddq_f32(vld1q_f32(rec + 2 * hid + i), vld1q_f32(in + 2 * hid + i)));
+ const float32x4_t og =
+ vsigmoidq(vaddq_f32(vld1q_f32(rec + 3 * hid + i), vld1q_f32(in + 3 * hid + i)));
+ const float32x4_t cn = vfmaq_f32(vmulq_f32(ig, gg), fg, vld1q_f32(c + i));
+ vst1q_f32(c + i, cn);
+ vst1q_f32(h + i, vmulq_f32(og, vtanhq(cn)));
+ }
+}
+
+static const pyseg_kernels kernels_simd = {"neon", gemm_nt_neon, lstm_cell_neon};
+#endif /* aarch64 NEON */
+
+/* ---- x86-64 AVX2 -----------------------------------------------------
+ *
+ * Compiled into every x86 build whatever baseline the rest of the file uses,
+ * by carrying the instruction set on the functions rather than on the command
+ * line, and selected at startup against the running processor. One binary is
+ * then both safe on a host without AVX2 and fast on one with it.
+ *
+ * Every function reachable from an AVX2 entry point needs the same attribute,
+ * including the always-inlined ones, or the compiler refuses to inline across
+ * the mismatch. */
+
+#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__))
+# define PYSEG_HAVE_SIMD 1
+# define PYSEG_X86_DISPATCH 1
+# include <immintrin.h>
+
+# define PYSEG_AVX2 __attribute__((target("avx2,fma")))
+# define PYSEG_AVX2_INLINE static inline __attribute__((always_inline, target("avx2,fma")))
+
+PYSEG_AVX2_INLINE float hsum256(__m256 v) {
+ __m128 lo = _mm256_castps256_ps128(v);
+ __m128 hi = _mm256_extractf128_ps(v, 1);
+ lo = _mm_add_ps(lo, hi);
+ lo = _mm_add_ps(lo, _mm_movehl_ps(lo, lo));
+ lo = _mm_add_ss(lo, _mm_shuffle_ps(lo, lo, 0x55));
+ return _mm_cvtss_f32(lo);
+}
+
+/* Four output positions against two weight rows: eight accumulators, which is
+ * half the register file. That is the shape this instruction set allows. The
+ * NEON kernel above blocks four by four because it has thirty-two vector
+ * registers to hold sixteen accumulators plus the operands; asking for the
+ * same here spills the accumulators to the stack every iteration and costs
+ * more than the blocking saves.
+ *
+ * The window length is a multiple of four but not always of eight, so the
+ * eight-wide body is followed by at most one four-wide step, done in scalar
+ * code since it is at most four elements of at least sixty. */
+PYSEG_AVX2_INLINE void
+dot4x2_avx2(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) {
+ const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda;
+ const float *w0 = w, *w1 = w + k;
+
+ __m256 c00 = _mm256_setzero_ps(), c01 = c00, c10 = c00, c11 = c00;
+ __m256 c20 = c00, c21 = c00, c30 = c00, c31 = c00;
+
+ int j = 0;
+ for (; j + 8 <= k; j += 8) {
+ const __m256 v0 = _mm256_loadu_ps(w0 + j);
+ const __m256 v1 = _mm256_loadu_ps(w1 + j);
+ const __m256 x0 = _mm256_loadu_ps(a0 + j);
+ const __m256 x1 = _mm256_loadu_ps(a1 + j);
+ const __m256 x2 = _mm256_loadu_ps(a2 + j);
+ const __m256 x3 = _mm256_loadu_ps(a3 + j);
+ c00 = _mm256_fmadd_ps(x0, v0, c00);
+ c01 = _mm256_fmadd_ps(x0, v1, c01);
+ c10 = _mm256_fmadd_ps(x1, v0, c10);
+ c11 = _mm256_fmadd_ps(x1, v1, c11);
+ c20 = _mm256_fmadd_ps(x2, v0, c20);
+ c21 = _mm256_fmadd_ps(x2, v1, c21);
+ c30 = _mm256_fmadd_ps(x3, v0, c30);
+ c31 = _mm256_fmadd_ps(x3, v1, c31);
+ }
+
+ const float *ar[4] = {a0, a1, a2, a3};
+ __m256 acc[4][2] = {{c00, c01}, {c10, c11}, {c20, c21}, {c30, c31}};
+ for (int s = 0; s < 4; ++s)
+ for (int i = 0; i < 2; ++i) {
+ float v = hsum256(acc[s][i]);
+ for (int t = j; t < k; ++t)
+ v += ar[s][t] * w[(size_t)i * k + t];
+ y[(size_t)s * ldy + i] = bias ? bias[i] + v : v;
+ }
+}
+
+/* Single output position, eight weight rows at a time; the matrix-vector shape
+ * of the LSTM recurrence. Eight accumulators fit, and one activation load then
+ * serves eight weight loads. */
+PYSEG_AVX2_INLINE void
+row_avx2(float *y, const float *a, const float *w, const float *bias, int n, int k) {
+ int i = 0;
+ for (; i + 8 <= n; i += 8) {
+ const float *w0 = w + (size_t)i * k;
+ __m256 c0 = _mm256_setzero_ps(), c1 = c0, c2 = c0, c3 = c0;
+ __m256 c4 = c0, c5 = c0, c6 = c0, c7 = c0;
+ int j = 0;
+ for (; j + 8 <= k; j += 8) {
+ const __m256 x = _mm256_loadu_ps(a + j);
+ const float *p = w0 + j;
+ c0 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p), c0);
+ c1 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + k), c1);
+ c2 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 2 * k), c2);
+ c3 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 3 * k), c3);
+ c4 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 4 * k), c4);
+ c5 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 5 * k), c5);
+ c6 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 6 * k), c6);
+ c7 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 7 * k), c7);
+ }
+ const __m256 acc[8] = {c0, c1, c2, c3, c4, c5, c6, c7};
+ for (int q = 0; q < 8; ++q) {
+ float v = hsum256(acc[q]);
+ for (int t = j; t < k; ++t)
+ v += a[t] * w0[(size_t)q * k + t];
+ y[i + q] = bias ? bias[i + q] + v : v;
+ }
+ }
+ for (; i < n; ++i) {
+ const float *wr = w + (size_t)i * k;
+ __m256 acc = _mm256_setzero_ps();
+ int j = 0;
+ for (; j + 8 <= k; j += 8)
+ acc = _mm256_fmadd_ps(_mm256_loadu_ps(a + j), _mm256_loadu_ps(wr + j), acc);
+ float v = hsum256(acc);
+ for (; j < k; ++j)
+ v += a[j] * wr[j];
+ y[i] = bias ? bias[i] + v : v;
+ }
+}
+
+PYSEG_AVX2 static void gemm_nt_avx2(float *y,
+ int ldy,
+ const float *a,
+ int lda,
+ const float *w,
+ const float *bias,
+ int m,
+ int n,
+ int k) {
+ if (m == 1) {
+ row_avx2(y, a, w, bias, n, k);
+ return;
+ }
+ for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) {
+ const int mend = min_int(m0 + PYSEG_MPANEL, m);
+ const int m4 = m0 + (mend - m0) / 4 * 4;
+ int i = 0;
+ for (; i + 2 <= n; i += 2)
+ for (int s = m0; s < m4; s += 4)
+ dot4x2_avx2(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k,
+ bias ? bias + i : NULL, k);
+ for (int s = m0; s < m4; ++s)
+ if (i < n)
+ row_avx2(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k,
+ bias ? bias + i : NULL, n - i, k);
+ for (int s = m4; s < mend; ++s)
+ row_avx2(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k);
+ }
+}
+
+/* The same Cephes scheme as the NEON path; see the comment there. */
+PYSEG_AVX2_INLINE __m256 vexp256(__m256 x) {
+ x = _mm256_min_ps(_mm256_max_ps(x, _mm256_set1_ps(-88.0f)), _mm256_set1_ps(88.0f));
+
+ const __m256 n = _mm256_round_ps(_mm256_mul_ps(x, _mm256_set1_ps(1.44269504088896341f)),
+ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ __m256 r = _mm256_fnmadd_ps(n, _mm256_set1_ps(0.693359375f), x);
+ r = _mm256_fnmadd_ps(n, _mm256_set1_ps(-2.12194440e-4f), r);
+
+ __m256 p = _mm256_set1_ps(1.9875691500e-4f);
+ p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.3981999507e-3f));
+ p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(8.3334519073e-3f));
+ p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(4.1665795894e-2f));
+ p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.6666665459e-1f));
+ p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(5.0000001201e-1f));
+ p = _mm256_fmadd_ps(p, _mm256_mul_ps(r, r), _mm256_add_ps(r, _mm256_set1_ps(1.0f)));
+
+ const __m256i bits =
+ _mm256_slli_epi32(_mm256_add_epi32(_mm256_cvtps_epi32(n), _mm256_set1_epi32(127)), 23);
+ return _mm256_mul_ps(p, _mm256_castsi256_ps(bits));
+}
+
+PYSEG_AVX2_INLINE __m256 vsigmoid256(__m256 x) {
+ const __m256 one = _mm256_set1_ps(1.0f);
+ return _mm256_div_ps(one, _mm256_add_ps(one, vexp256(_mm256_sub_ps(_mm256_setzero_ps(), x))));
+}
+
+PYSEG_AVX2_INLINE __m256 vtanh256(__m256 x) {
+ return _mm256_fmsub_ps(vsigmoid256(_mm256_add_ps(x, x)), _mm256_set1_ps(2.0f),
+ _mm256_set1_ps(1.0f));
+}
+
+PYSEG_AVX2_INLINE __m256 gate256(const float *rec, const float *in, int offset) {
+ return _mm256_add_ps(_mm256_loadu_ps(rec + offset), _mm256_loadu_ps(in + offset));
+}
+
+PYSEG_AVX2 static void
+lstm_cell_avx2(float *h, float *c, const float *rec, const float *in, int hid) {
+ int i = 0;
+ for (; i + 8 <= hid; i += 8) {
+ const __m256 ig = vsigmoid256(gate256(rec, in, i));
+ const __m256 fg = vsigmoid256(gate256(rec, in, hid + i));
+ const __m256 gg = vtanh256(gate256(rec, in, 2 * hid + i));
+ const __m256 og = vsigmoid256(gate256(rec, in, 3 * hid + i));
+ const __m256 cn = _mm256_fmadd_ps(fg, _mm256_loadu_ps(c + i), _mm256_mul_ps(ig, gg));
+ _mm256_storeu_ps(c + i, cn);
+ _mm256_storeu_ps(h + i, _mm256_mul_ps(og, vtanh256(cn)));
+ }
+ for (; i < hid; ++i) {
+ const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i])));
+ const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i])));
+ const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]);
+ const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i])));
+ const float cn = fg * c[i] + ig * gg;
+ c[i] = cn;
+ h[i] = og * tanhf(cn);
+ }
+}
+
+static const pyseg_kernels kernels_avx2 = {"avx2", gemm_nt_avx2, lstm_cell_avx2};
+
+/* AVX2 also needs the operating system to have enabled the wider register
+ * state, which __builtin_cpu_supports checks along with the feature bits. */
+static int cpu_has_avx2(void) {
+# if defined(__has_builtin)
+# if __has_builtin(__builtin_cpu_init)
+ __builtin_cpu_init();
+# endif
+# elif defined(__GNUC__)
+ __builtin_cpu_init();
+# endif
+ return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
+}
+#endif /* x86 */
+
+const pyseg_kernels *pyseg_kernels_simd(void) {
+#if defined(PYSEG_X86_DISPATCH)
+ /* Resolved once; the answer cannot change while the process runs. */
+ static const pyseg_kernels *chosen;
+ if (!chosen)
+ chosen = cpu_has_avx2() ? &kernels_avx2 : &kernels_scalar;
+ return chosen;
+#elif defined(PYSEG_HAVE_SIMD)
+ return &kernels_simd;
+#else
+ return &kernels_scalar;
+#endif
+}