From 64c3a1b528a88b4ae21bf3cbcff2537200477806 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sun, 13 Sep 2026 20:03:39 +0200 Subject: Init --- src/kernels.c | 555 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 555 insertions(+) create mode 100644 src/kernels.c (limited to 'src/kernels.c') diff --git a/src/kernels.c b/src/kernels.c new file mode 100644 index 0000000..01c1d19 --- /dev/null +++ b/src/kernels.c @@ -0,0 +1,555 @@ +/* Scalar and SIMD implementations of the dense primitive. + * + * Each variant blocks a rectangle of output positions against weight rows and + * accumulates along the window, and the rectangle is chosen by the register + * file rather than by the algorithm. AArch64 has 32 vector registers, so four + * positions by four rows fits: sixteen accumulators plus eight live operands, + * eight loads for sixteen multiply-adds. x86-64 has 16, so the same rectangle + * spills its accumulators to the stack every iteration and four by two is + * faster despite doing half as much per pass; on one server core that + * difference was a factor of 1.8 over a whole recording. + * + * Matrix-vector shapes, meaning the LSTM recurrence where there is only one + * output position, widen instead to eight weight rows. With a single + * activation window there is nothing to reuse but that window, so the weight + * loads are the entire cost and the only thing worth amortising is the + * activation. + * + * Accumulation order differs between the variants, so their results differ in + * the last bits; tests/test_kernels.c bounds that. + */ + +#include "kernels.h" + +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +# define PYSEG_INLINE static inline __attribute__((always_inline)) +#else +# define PYSEG_INLINE static inline +#endif + +static inline int min_int(int a, int b) { return a < b ? a : b; } + +/* ---- portable C ------------------------------------------------------ */ + +PYSEG_INLINE void +dot4x4_scalar(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) { + float acc[4][4] = {{0.0f}}; + for (int j = 0; j < k; ++j) { + const float a0 = a[j], a1 = a[lda + j], a2 = a[2 * lda + j], a3 = a[3 * lda + j]; + for (int i = 0; i < 4; ++i) { + const float wv = w[(size_t)i * k + j]; + acc[0][i] += a0 * wv; + acc[1][i] += a1 * wv; + acc[2][i] += a2 * wv; + acc[3][i] += a3 * wv; + } + } + for (int s = 0; s < 4; ++s) + for (int i = 0; i < 4; ++i) + y[(size_t)s * ldy + i] = bias ? bias[i] + acc[s][i] : acc[s][i]; +} + +/* One output position against however many weight rows are left. */ +PYSEG_INLINE void +row_scalar(float *y, const float *a, const float *w, const float *bias, int n, int k) { + for (int i = 0; i < n; ++i) { + const float *wr = w + (size_t)i * k; + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + int j = 0; + for (; j + 4 <= k; j += 4) { + s0 += a[j + 0] * wr[j + 0]; + s1 += a[j + 1] * wr[j + 1]; + s2 += a[j + 2] * wr[j + 2]; + s3 += a[j + 3] * wr[j + 3]; + } + const float s = (s0 + s1) + (s2 + s3); + y[i] = bias ? bias[i] + s : s; + } +} + +static void gemm_nt_scalar(float *y, + int ldy, + const float *a, + int lda, + const float *w, + const float *bias, + int m, + int n, + int k) { + for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) { + const int mend = min_int(m0 + PYSEG_MPANEL, m); + const int m4 = m0 + (mend - m0) / 4 * 4; + int i = 0; + for (; i + 4 <= n; i += 4) + for (int s = m0; s < m4; s += 4) + dot4x4_scalar(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, + w + (size_t)i * k, bias ? bias + i : NULL, k); + for (int s = m0; s < m4; ++s) + if (i < n) + row_scalar(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k, + bias ? bias + i : NULL, n - i, k); + for (int s = m4; s < mend; ++s) + row_scalar(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k); + } +} + +static void lstm_cell_scalar(float *h, float *c, const float *rec, const float *in, int hid) { + for (int i = 0; i < hid; ++i) { + const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i]))); + const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i]))); + const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]); + const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i]))); + const float cn = fg * c[i] + ig * gg; + c[i] = cn; + h[i] = og * tanhf(cn); + } +} + +static const pyseg_kernels kernels_scalar = {"scalar", gemm_nt_scalar, lstm_cell_scalar}; + +const pyseg_kernels *pyseg_kernels_scalar(void) { return &kernels_scalar; } + +/* ---- AArch64 NEON ---------------------------------------------------- */ + +#if defined(__aarch64__) && defined(__ARM_NEON) +# define PYSEG_HAVE_SIMD 1 +# include + +/* Four lane-wise accumulators reduced to one vector holding their four totals. + * Pairwise adds do this in three instructions instead of four cross-lane + * reductions plus the moves back into a vector. */ +PYSEG_INLINE float32x4_t hsum4(float32x4_t a, float32x4_t b, float32x4_t c, float32x4_t d) { + return vpaddq_f32(vpaddq_f32(a, b), vpaddq_f32(c, d)); +} + +PYSEG_INLINE void +dot4x4_neon(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) { + const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda; + const float *w0 = w, *w1 = w + k, *w2 = w + 2 * k, *w3 = w + 3 * k; + + float32x4_t c00 = vdupq_n_f32(0.0f), c01 = c00, c02 = c00, c03 = c00; + float32x4_t c10 = c00, c11 = c00, c12 = c00, c13 = c00; + float32x4_t c20 = c00, c21 = c00, c22 = c00, c23 = c00; + float32x4_t c30 = c00, c31 = c00, c32 = c00, c33 = c00; + + for (int j = 0; j < k; j += 4) { + const float32x4_t x0 = vld1q_f32(a0 + j); + const float32x4_t x1 = vld1q_f32(a1 + j); + const float32x4_t x2 = vld1q_f32(a2 + j); + const float32x4_t x3 = vld1q_f32(a3 + j); + const float32x4_t v0 = vld1q_f32(w0 + j); + const float32x4_t v1 = vld1q_f32(w1 + j); + const float32x4_t v2 = vld1q_f32(w2 + j); + const float32x4_t v3 = vld1q_f32(w3 + j); + c00 = vfmaq_f32(c00, x0, v0); + c01 = vfmaq_f32(c01, x0, v1); + c02 = vfmaq_f32(c02, x0, v2); + c03 = vfmaq_f32(c03, x0, v3); + c10 = vfmaq_f32(c10, x1, v0); + c11 = vfmaq_f32(c11, x1, v1); + c12 = vfmaq_f32(c12, x1, v2); + c13 = vfmaq_f32(c13, x1, v3); + c20 = vfmaq_f32(c20, x2, v0); + c21 = vfmaq_f32(c21, x2, v1); + c22 = vfmaq_f32(c22, x2, v2); + c23 = vfmaq_f32(c23, x2, v3); + c30 = vfmaq_f32(c30, x3, v0); + c31 = vfmaq_f32(c31, x3, v1); + c32 = vfmaq_f32(c32, x3, v2); + c33 = vfmaq_f32(c33, x3, v3); + } + + const float32x4_t b = bias ? vld1q_f32(bias) : vdupq_n_f32(0.0f); + vst1q_f32(y, vaddq_f32(b, hsum4(c00, c01, c02, c03))); + vst1q_f32(y + ldy, vaddq_f32(b, hsum4(c10, c11, c12, c13))); + vst1q_f32(y + 2 * ldy, vaddq_f32(b, hsum4(c20, c21, c22, c23))); + vst1q_f32(y + 3 * ldy, vaddq_f32(b, hsum4(c30, c31, c32, c33))); +} + +/* Single output position, eight weight rows at a time. + * + * With one activation window there is nothing to reuse but the window itself, + * so this shape costs one weight load per multiply-add and runs at whatever + * rate the memory system delivers weights, not at the multiply pipes' rate. + * Eight rows measured fastest here: four leaves the load pipeline too shallow, + * and sixteen has enough concurrent streams to start working against the + * hardware prefetcher. */ +PYSEG_INLINE void +row_neon(float *y, const float *a, const float *w, const float *bias, int n, int k) { + int i = 0; + for (; i + 8 <= n; i += 8) { + const float *w0 = w + (size_t)i * k; + float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0; + float32x4_t c4 = c0, c5 = c0, c6 = c0, c7 = c0; + for (int j = 0; j < k; j += 4) { + const float32x4_t x = vld1q_f32(a + j); + c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j)); + c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j)); + c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j)); + c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j)); + c4 = vfmaq_f32(c4, x, vld1q_f32(w0 + 4 * k + j)); + c5 = vfmaq_f32(c5, x, vld1q_f32(w0 + 5 * k + j)); + c6 = vfmaq_f32(c6, x, vld1q_f32(w0 + 6 * k + j)); + c7 = vfmaq_f32(c7, x, vld1q_f32(w0 + 7 * k + j)); + } + const float32x4_t lo = hsum4(c0, c1, c2, c3); + const float32x4_t hi = hsum4(c4, c5, c6, c7); + if (bias) { + vst1q_f32(y + i, vaddq_f32(vld1q_f32(bias + i), lo)); + vst1q_f32(y + i + 4, vaddq_f32(vld1q_f32(bias + i + 4), hi)); + } else { + vst1q_f32(y + i, lo); + vst1q_f32(y + i + 4, hi); + } + } + for (; i + 4 <= n; i += 4) { + const float *w0 = w + (size_t)i * k; + float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0; + for (int j = 0; j < k; j += 4) { + const float32x4_t x = vld1q_f32(a + j); + c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j)); + c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j)); + c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j)); + c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j)); + } + const float32x4_t v = hsum4(c0, c1, c2, c3); + vst1q_f32(y + i, bias ? vaddq_f32(vld1q_f32(bias + i), v) : v); + } + for (; i < n; ++i) { + const float *wr = w + (size_t)i * k; + float32x4_t c0 = vdupq_n_f32(0.0f); + for (int j = 0; j < k; j += 4) + c0 = vfmaq_f32(c0, vld1q_f32(a + j), vld1q_f32(wr + j)); + const float s = vaddvq_f32(c0); + y[i] = bias ? bias[i] + s : s; + } +} + +static void gemm_nt_neon(float *y, + int ldy, + const float *a, + int lda, + const float *w, + const float *bias, + int m, + int n, + int k) { + if (m == 1) { + row_neon(y, a, w, bias, n, k); + return; + } + for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) { + const int mend = min_int(m0 + PYSEG_MPANEL, m); + const int m4 = m0 + (mend - m0) / 4 * 4; + int i = 0; + for (; i + 4 <= n; i += 4) + for (int s = m0; s < m4; s += 4) + dot4x4_neon(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k, + bias ? bias + i : NULL, k); + for (int s = m0; s < m4; ++s) + if (i < n) + row_neon(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k, + bias ? bias + i : NULL, n - i, k); + for (int s = m4; s < mend; ++s) + row_neon(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k); + } +} + +/* Vector exponential, the Cephes single-precision scheme: split the argument + * into an integer power of two and a remainder inside half a ln 2, evaluate a + * degree-six polynomial on the remainder, then scale by assembling the power + * of two directly in the exponent field. Accurate to about one unit in the + * last place, which is the same claim libm makes. + * + * The clamp is what keeps the exponent assembly in range; beyond it the + * logistic and hyperbolic functions built on this are saturated anyway. */ +PYSEG_INLINE float32x4_t vexpq(float32x4_t x) { + x = vminq_f32(vmaxq_f32(x, vdupq_n_f32(-88.0f)), vdupq_n_f32(88.0f)); + + const float32x4_t n = vrndnq_f32(vmulq_f32(x, vdupq_n_f32(1.44269504088896341f))); + /* ln 2 split in two so the product with n stays exact in the high part. */ + float32x4_t r = vfmsq_f32(x, n, vdupq_n_f32(0.693359375f)); + r = vfmsq_f32(r, n, vdupq_n_f32(-2.12194440e-4f)); + + float32x4_t p = vdupq_n_f32(1.9875691500e-4f); + p = vfmaq_f32(vdupq_n_f32(1.3981999507e-3f), p, r); + p = vfmaq_f32(vdupq_n_f32(8.3334519073e-3f), p, r); + p = vfmaq_f32(vdupq_n_f32(4.1665795894e-2f), p, r); + p = vfmaq_f32(vdupq_n_f32(1.6666665459e-1f), p, r); + p = vfmaq_f32(vdupq_n_f32(5.0000001201e-1f), p, r); + p = vfmaq_f32(vaddq_f32(r, vdupq_n_f32(1.0f)), p, vmulq_f32(r, r)); + + const int32x4_t bits = vshlq_n_s32(vaddq_s32(vcvtq_s32_f32(n), vdupq_n_s32(127)), 23); + return vmulq_f32(p, vreinterpretq_f32_s32(bits)); +} + +PYSEG_INLINE float32x4_t vsigmoidq(float32x4_t x) { + return vdivq_f32(vdupq_n_f32(1.0f), vaddq_f32(vdupq_n_f32(1.0f), vexpq(vnegq_f32(x)))); +} + +/* tanh x = 2 sigmoid(2x) - 1. The subtraction loses relative accuracy near + * zero, where sigmoid is close to a half, but the absolute error stays at the + * rounding level of the result and that is what propagates into the cell + * state. */ +PYSEG_INLINE float32x4_t vtanhq(float32x4_t x) { + return vfmsq_f32(vdupq_n_f32(-1.0f), vsigmoidq(vaddq_f32(x, x)), vdupq_n_f32(-2.0f)); +} + +static void lstm_cell_neon(float *h, float *c, const float *rec, const float *in, int hid) { + for (int i = 0; i < hid; i += 4) { + const float32x4_t ig = vsigmoidq(vaddq_f32(vld1q_f32(rec + i), vld1q_f32(in + i))); + const float32x4_t fg = + vsigmoidq(vaddq_f32(vld1q_f32(rec + hid + i), vld1q_f32(in + hid + i))); + const float32x4_t gg = + vtanhq(vaddq_f32(vld1q_f32(rec + 2 * hid + i), vld1q_f32(in + 2 * hid + i))); + const float32x4_t og = + vsigmoidq(vaddq_f32(vld1q_f32(rec + 3 * hid + i), vld1q_f32(in + 3 * hid + i))); + const float32x4_t cn = vfmaq_f32(vmulq_f32(ig, gg), fg, vld1q_f32(c + i)); + vst1q_f32(c + i, cn); + vst1q_f32(h + i, vmulq_f32(og, vtanhq(cn))); + } +} + +static const pyseg_kernels kernels_simd = {"neon", gemm_nt_neon, lstm_cell_neon}; +#endif /* aarch64 NEON */ + +/* ---- x86-64 AVX2 ----------------------------------------------------- + * + * Compiled into every x86 build whatever baseline the rest of the file uses, + * by carrying the instruction set on the functions rather than on the command + * line, and selected at startup against the running processor. One binary is + * then both safe on a host without AVX2 and fast on one with it. + * + * Every function reachable from an AVX2 entry point needs the same attribute, + * including the always-inlined ones, or the compiler refuses to inline across + * the mismatch. */ + +#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) +# define PYSEG_HAVE_SIMD 1 +# define PYSEG_X86_DISPATCH 1 +# include + +# define PYSEG_AVX2 __attribute__((target("avx2,fma"))) +# define PYSEG_AVX2_INLINE static inline __attribute__((always_inline, target("avx2,fma"))) + +PYSEG_AVX2_INLINE float hsum256(__m256 v) { + __m128 lo = _mm256_castps256_ps128(v); + __m128 hi = _mm256_extractf128_ps(v, 1); + lo = _mm_add_ps(lo, hi); + lo = _mm_add_ps(lo, _mm_movehl_ps(lo, lo)); + lo = _mm_add_ss(lo, _mm_shuffle_ps(lo, lo, 0x55)); + return _mm_cvtss_f32(lo); +} + +/* Four output positions against two weight rows: eight accumulators, which is + * half the register file. That is the shape this instruction set allows. The + * NEON kernel above blocks four by four because it has thirty-two vector + * registers to hold sixteen accumulators plus the operands; asking for the + * same here spills the accumulators to the stack every iteration and costs + * more than the blocking saves. + * + * The window length is a multiple of four but not always of eight, so the + * eight-wide body is followed by at most one four-wide step, done in scalar + * code since it is at most four elements of at least sixty. */ +PYSEG_AVX2_INLINE void +dot4x2_avx2(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) { + const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda; + const float *w0 = w, *w1 = w + k; + + __m256 c00 = _mm256_setzero_ps(), c01 = c00, c10 = c00, c11 = c00; + __m256 c20 = c00, c21 = c00, c30 = c00, c31 = c00; + + int j = 0; + for (; j + 8 <= k; j += 8) { + const __m256 v0 = _mm256_loadu_ps(w0 + j); + const __m256 v1 = _mm256_loadu_ps(w1 + j); + const __m256 x0 = _mm256_loadu_ps(a0 + j); + const __m256 x1 = _mm256_loadu_ps(a1 + j); + const __m256 x2 = _mm256_loadu_ps(a2 + j); + const __m256 x3 = _mm256_loadu_ps(a3 + j); + c00 = _mm256_fmadd_ps(x0, v0, c00); + c01 = _mm256_fmadd_ps(x0, v1, c01); + c10 = _mm256_fmadd_ps(x1, v0, c10); + c11 = _mm256_fmadd_ps(x1, v1, c11); + c20 = _mm256_fmadd_ps(x2, v0, c20); + c21 = _mm256_fmadd_ps(x2, v1, c21); + c30 = _mm256_fmadd_ps(x3, v0, c30); + c31 = _mm256_fmadd_ps(x3, v1, c31); + } + + const float *ar[4] = {a0, a1, a2, a3}; + __m256 acc[4][2] = {{c00, c01}, {c10, c11}, {c20, c21}, {c30, c31}}; + for (int s = 0; s < 4; ++s) + for (int i = 0; i < 2; ++i) { + float v = hsum256(acc[s][i]); + for (int t = j; t < k; ++t) + v += ar[s][t] * w[(size_t)i * k + t]; + y[(size_t)s * ldy + i] = bias ? bias[i] + v : v; + } +} + +/* Single output position, eight weight rows at a time; the matrix-vector shape + * of the LSTM recurrence. Eight accumulators fit, and one activation load then + * serves eight weight loads. */ +PYSEG_AVX2_INLINE void +row_avx2(float *y, const float *a, const float *w, const float *bias, int n, int k) { + int i = 0; + for (; i + 8 <= n; i += 8) { + const float *w0 = w + (size_t)i * k; + __m256 c0 = _mm256_setzero_ps(), c1 = c0, c2 = c0, c3 = c0; + __m256 c4 = c0, c5 = c0, c6 = c0, c7 = c0; + int j = 0; + for (; j + 8 <= k; j += 8) { + const __m256 x = _mm256_loadu_ps(a + j); + const float *p = w0 + j; + c0 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p), c0); + c1 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + k), c1); + c2 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 2 * k), c2); + c3 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 3 * k), c3); + c4 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 4 * k), c4); + c5 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 5 * k), c5); + c6 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 6 * k), c6); + c7 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 7 * k), c7); + } + const __m256 acc[8] = {c0, c1, c2, c3, c4, c5, c6, c7}; + for (int q = 0; q < 8; ++q) { + float v = hsum256(acc[q]); + for (int t = j; t < k; ++t) + v += a[t] * w0[(size_t)q * k + t]; + y[i + q] = bias ? bias[i + q] + v : v; + } + } + for (; i < n; ++i) { + const float *wr = w + (size_t)i * k; + __m256 acc = _mm256_setzero_ps(); + int j = 0; + for (; j + 8 <= k; j += 8) + acc = _mm256_fmadd_ps(_mm256_loadu_ps(a + j), _mm256_loadu_ps(wr + j), acc); + float v = hsum256(acc); + for (; j < k; ++j) + v += a[j] * wr[j]; + y[i] = bias ? bias[i] + v : v; + } +} + +PYSEG_AVX2 static void gemm_nt_avx2(float *y, + int ldy, + const float *a, + int lda, + const float *w, + const float *bias, + int m, + int n, + int k) { + if (m == 1) { + row_avx2(y, a, w, bias, n, k); + return; + } + for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) { + const int mend = min_int(m0 + PYSEG_MPANEL, m); + const int m4 = m0 + (mend - m0) / 4 * 4; + int i = 0; + for (; i + 2 <= n; i += 2) + for (int s = m0; s < m4; s += 4) + dot4x2_avx2(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k, + bias ? bias + i : NULL, k); + for (int s = m0; s < m4; ++s) + if (i < n) + row_avx2(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k, + bias ? bias + i : NULL, n - i, k); + for (int s = m4; s < mend; ++s) + row_avx2(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k); + } +} + +/* The same Cephes scheme as the NEON path; see the comment there. */ +PYSEG_AVX2_INLINE __m256 vexp256(__m256 x) { + x = _mm256_min_ps(_mm256_max_ps(x, _mm256_set1_ps(-88.0f)), _mm256_set1_ps(88.0f)); + + const __m256 n = _mm256_round_ps(_mm256_mul_ps(x, _mm256_set1_ps(1.44269504088896341f)), + _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + __m256 r = _mm256_fnmadd_ps(n, _mm256_set1_ps(0.693359375f), x); + r = _mm256_fnmadd_ps(n, _mm256_set1_ps(-2.12194440e-4f), r); + + __m256 p = _mm256_set1_ps(1.9875691500e-4f); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.3981999507e-3f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(8.3334519073e-3f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(4.1665795894e-2f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.6666665459e-1f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(5.0000001201e-1f)); + p = _mm256_fmadd_ps(p, _mm256_mul_ps(r, r), _mm256_add_ps(r, _mm256_set1_ps(1.0f))); + + const __m256i bits = + _mm256_slli_epi32(_mm256_add_epi32(_mm256_cvtps_epi32(n), _mm256_set1_epi32(127)), 23); + return _mm256_mul_ps(p, _mm256_castsi256_ps(bits)); +} + +PYSEG_AVX2_INLINE __m256 vsigmoid256(__m256 x) { + const __m256 one = _mm256_set1_ps(1.0f); + return _mm256_div_ps(one, _mm256_add_ps(one, vexp256(_mm256_sub_ps(_mm256_setzero_ps(), x)))); +} + +PYSEG_AVX2_INLINE __m256 vtanh256(__m256 x) { + return _mm256_fmsub_ps(vsigmoid256(_mm256_add_ps(x, x)), _mm256_set1_ps(2.0f), + _mm256_set1_ps(1.0f)); +} + +PYSEG_AVX2_INLINE __m256 gate256(const float *rec, const float *in, int offset) { + return _mm256_add_ps(_mm256_loadu_ps(rec + offset), _mm256_loadu_ps(in + offset)); +} + +PYSEG_AVX2 static void +lstm_cell_avx2(float *h, float *c, const float *rec, const float *in, int hid) { + int i = 0; + for (; i + 8 <= hid; i += 8) { + const __m256 ig = vsigmoid256(gate256(rec, in, i)); + const __m256 fg = vsigmoid256(gate256(rec, in, hid + i)); + const __m256 gg = vtanh256(gate256(rec, in, 2 * hid + i)); + const __m256 og = vsigmoid256(gate256(rec, in, 3 * hid + i)); + const __m256 cn = _mm256_fmadd_ps(fg, _mm256_loadu_ps(c + i), _mm256_mul_ps(ig, gg)); + _mm256_storeu_ps(c + i, cn); + _mm256_storeu_ps(h + i, _mm256_mul_ps(og, vtanh256(cn))); + } + for (; i < hid; ++i) { + const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i]))); + const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i]))); + const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]); + const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i]))); + const float cn = fg * c[i] + ig * gg; + c[i] = cn; + h[i] = og * tanhf(cn); + } +} + +static const pyseg_kernels kernels_avx2 = {"avx2", gemm_nt_avx2, lstm_cell_avx2}; + +/* AVX2 also needs the operating system to have enabled the wider register + * state, which __builtin_cpu_supports checks along with the feature bits. */ +static int cpu_has_avx2(void) { +# if defined(__has_builtin) +# if __has_builtin(__builtin_cpu_init) + __builtin_cpu_init(); +# endif +# elif defined(__GNUC__) + __builtin_cpu_init(); +# endif + return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); +} +#endif /* x86 */ + +const pyseg_kernels *pyseg_kernels_simd(void) { +#if defined(PYSEG_X86_DISPATCH) + /* Resolved once; the answer cannot change while the process runs. */ + static const pyseg_kernels *chosen; + if (!chosen) + chosen = cpu_has_avx2() ? &kernels_avx2 : &kernels_scalar; + return chosen; +#elif defined(PYSEG_HAVE_SIMD) + return &kernels_simd; +#else + return &kernels_scalar; +#endif +} -- cgit v1.3.1