aboutsummaryrefslogtreecommitdiff
path: root/src/kernels.c
blob: 01c1d19ce9e8ff4b8622aed87784589ca2ee599c (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
/* Scalar and SIMD implementations of the dense primitive.
 *
 * Each variant blocks a rectangle of output positions against weight rows and
 * accumulates along the window, and the rectangle is chosen by the register
 * file rather than by the algorithm. AArch64 has 32 vector registers, so four
 * positions by four rows fits: sixteen accumulators plus eight live operands,
 * eight loads for sixteen multiply-adds. x86-64 has 16, so the same rectangle
 * spills its accumulators to the stack every iteration and four by two is
 * faster despite doing half as much per pass; on one server core that
 * difference was a factor of 1.8 over a whole recording.
 *
 * Matrix-vector shapes, meaning the LSTM recurrence where there is only one
 * output position, widen instead to eight weight rows. With a single
 * activation window there is nothing to reuse but that window, so the weight
 * loads are the entire cost and the only thing worth amortising is the
 * activation.
 *
 * Accumulation order differs between the variants, so their results differ in
 * the last bits; tests/test_kernels.c bounds that.
 */

#include "kernels.h"

#include <math.h>
#include <stddef.h>

#if defined(__GNUC__) || defined(__clang__)
#    define PYSEG_INLINE static inline __attribute__((always_inline))
#else
#    define PYSEG_INLINE static inline
#endif

static inline int min_int(int a, int b) { return a < b ? a : b; }

/* ---- portable C ------------------------------------------------------ */

PYSEG_INLINE void
dot4x4_scalar(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) {
    float acc[4][4] = {{0.0f}};
    for (int j = 0; j < k; ++j) {
        const float a0 = a[j], a1 = a[lda + j], a2 = a[2 * lda + j], a3 = a[3 * lda + j];
        for (int i = 0; i < 4; ++i) {
            const float wv = w[(size_t)i * k + j];
            acc[0][i] += a0 * wv;
            acc[1][i] += a1 * wv;
            acc[2][i] += a2 * wv;
            acc[3][i] += a3 * wv;
        }
    }
    for (int s = 0; s < 4; ++s)
        for (int i = 0; i < 4; ++i)
            y[(size_t)s * ldy + i] = bias ? bias[i] + acc[s][i] : acc[s][i];
}

/* One output position against however many weight rows are left. */
PYSEG_INLINE void
row_scalar(float *y, const float *a, const float *w, const float *bias, int n, int k) {
    for (int i = 0; i < n; ++i) {
        const float *wr = w + (size_t)i * k;
        float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f;
        int j = 0;
        for (; j + 4 <= k; j += 4) {
            s0 += a[j + 0] * wr[j + 0];
            s1 += a[j + 1] * wr[j + 1];
            s2 += a[j + 2] * wr[j + 2];
            s3 += a[j + 3] * wr[j + 3];
        }
        const float s = (s0 + s1) + (s2 + s3);
        y[i] = bias ? bias[i] + s : s;
    }
}

static void gemm_nt_scalar(float *y,
                           int ldy,
                           const float *a,
                           int lda,
                           const float *w,
                           const float *bias,
                           int m,
                           int n,
                           int k) {
    for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) {
        const int mend = min_int(m0 + PYSEG_MPANEL, m);
        const int m4 = m0 + (mend - m0) / 4 * 4;
        int i = 0;
        for (; i + 4 <= n; i += 4)
            for (int s = m0; s < m4; s += 4)
                dot4x4_scalar(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda,
                              w + (size_t)i * k, bias ? bias + i : NULL, k);
        for (int s = m0; s < m4; ++s)
            if (i < n)
                row_scalar(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k,
                           bias ? bias + i : NULL, n - i, k);
        for (int s = m4; s < mend; ++s)
            row_scalar(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k);
    }
}

static void lstm_cell_scalar(float *h, float *c, const float *rec, const float *in, int hid) {
    for (int i = 0; i < hid; ++i) {
        const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i])));
        const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i])));
        const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]);
        const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i])));
        const float cn = fg * c[i] + ig * gg;
        c[i] = cn;
        h[i] = og * tanhf(cn);
    }
}

static const pyseg_kernels kernels_scalar = {"scalar", gemm_nt_scalar, lstm_cell_scalar};

const pyseg_kernels *pyseg_kernels_scalar(void) { return &kernels_scalar; }

/* ---- AArch64 NEON ---------------------------------------------------- */

#if defined(__aarch64__) && defined(__ARM_NEON)
#    define PYSEG_HAVE_SIMD 1
#    include <arm_neon.h>

/* Four lane-wise accumulators reduced to one vector holding their four totals.
 * Pairwise adds do this in three instructions instead of four cross-lane
 * reductions plus the moves back into a vector. */
PYSEG_INLINE float32x4_t hsum4(float32x4_t a, float32x4_t b, float32x4_t c, float32x4_t d) {
    return vpaddq_f32(vpaddq_f32(a, b), vpaddq_f32(c, d));
}

PYSEG_INLINE void
dot4x4_neon(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) {
    const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda;
    const float *w0 = w, *w1 = w + k, *w2 = w + 2 * k, *w3 = w + 3 * k;

    float32x4_t c00 = vdupq_n_f32(0.0f), c01 = c00, c02 = c00, c03 = c00;
    float32x4_t c10 = c00, c11 = c00, c12 = c00, c13 = c00;
    float32x4_t c20 = c00, c21 = c00, c22 = c00, c23 = c00;
    float32x4_t c30 = c00, c31 = c00, c32 = c00, c33 = c00;

    for (int j = 0; j < k; j += 4) {
        const float32x4_t x0 = vld1q_f32(a0 + j);
        const float32x4_t x1 = vld1q_f32(a1 + j);
        const float32x4_t x2 = vld1q_f32(a2 + j);
        const float32x4_t x3 = vld1q_f32(a3 + j);
        const float32x4_t v0 = vld1q_f32(w0 + j);
        const float32x4_t v1 = vld1q_f32(w1 + j);
        const float32x4_t v2 = vld1q_f32(w2 + j);
        const float32x4_t v3 = vld1q_f32(w3 + j);
        c00 = vfmaq_f32(c00, x0, v0);
        c01 = vfmaq_f32(c01, x0, v1);
        c02 = vfmaq_f32(c02, x0, v2);
        c03 = vfmaq_f32(c03, x0, v3);
        c10 = vfmaq_f32(c10, x1, v0);
        c11 = vfmaq_f32(c11, x1, v1);
        c12 = vfmaq_f32(c12, x1, v2);
        c13 = vfmaq_f32(c13, x1, v3);
        c20 = vfmaq_f32(c20, x2, v0);
        c21 = vfmaq_f32(c21, x2, v1);
        c22 = vfmaq_f32(c22, x2, v2);
        c23 = vfmaq_f32(c23, x2, v3);
        c30 = vfmaq_f32(c30, x3, v0);
        c31 = vfmaq_f32(c31, x3, v1);
        c32 = vfmaq_f32(c32, x3, v2);
        c33 = vfmaq_f32(c33, x3, v3);
    }

    const float32x4_t b = bias ? vld1q_f32(bias) : vdupq_n_f32(0.0f);
    vst1q_f32(y, vaddq_f32(b, hsum4(c00, c01, c02, c03)));
    vst1q_f32(y + ldy, vaddq_f32(b, hsum4(c10, c11, c12, c13)));
    vst1q_f32(y + 2 * ldy, vaddq_f32(b, hsum4(c20, c21, c22, c23)));
    vst1q_f32(y + 3 * ldy, vaddq_f32(b, hsum4(c30, c31, c32, c33)));
}

/* Single output position, eight weight rows at a time.
 *
 * With one activation window there is nothing to reuse but the window itself,
 * so this shape costs one weight load per multiply-add and runs at whatever
 * rate the memory system delivers weights, not at the multiply pipes' rate.
 * Eight rows measured fastest here: four leaves the load pipeline too shallow,
 * and sixteen has enough concurrent streams to start working against the
 * hardware prefetcher. */
PYSEG_INLINE void
row_neon(float *y, const float *a, const float *w, const float *bias, int n, int k) {
    int i = 0;
    for (; i + 8 <= n; i += 8) {
        const float *w0 = w + (size_t)i * k;
        float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0;
        float32x4_t c4 = c0, c5 = c0, c6 = c0, c7 = c0;
        for (int j = 0; j < k; j += 4) {
            const float32x4_t x = vld1q_f32(a + j);
            c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j));
            c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j));
            c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j));
            c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j));
            c4 = vfmaq_f32(c4, x, vld1q_f32(w0 + 4 * k + j));
            c5 = vfmaq_f32(c5, x, vld1q_f32(w0 + 5 * k + j));
            c6 = vfmaq_f32(c6, x, vld1q_f32(w0 + 6 * k + j));
            c7 = vfmaq_f32(c7, x, vld1q_f32(w0 + 7 * k + j));
        }
        const float32x4_t lo = hsum4(c0, c1, c2, c3);
        const float32x4_t hi = hsum4(c4, c5, c6, c7);
        if (bias) {
            vst1q_f32(y + i, vaddq_f32(vld1q_f32(bias + i), lo));
            vst1q_f32(y + i + 4, vaddq_f32(vld1q_f32(bias + i + 4), hi));
        } else {
            vst1q_f32(y + i, lo);
            vst1q_f32(y + i + 4, hi);
        }
    }
    for (; i + 4 <= n; i += 4) {
        const float *w0 = w + (size_t)i * k;
        float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0;
        for (int j = 0; j < k; j += 4) {
            const float32x4_t x = vld1q_f32(a + j);
            c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j));
            c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j));
            c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j));
            c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j));
        }
        const float32x4_t v = hsum4(c0, c1, c2, c3);
        vst1q_f32(y + i, bias ? vaddq_f32(vld1q_f32(bias + i), v) : v);
    }
    for (; i < n; ++i) {
        const float *wr = w + (size_t)i * k;
        float32x4_t c0 = vdupq_n_f32(0.0f);
        for (int j = 0; j < k; j += 4)
            c0 = vfmaq_f32(c0, vld1q_f32(a + j), vld1q_f32(wr + j));
        const float s = vaddvq_f32(c0);
        y[i] = bias ? bias[i] + s : s;
    }
}

static void gemm_nt_neon(float *y,
                         int ldy,
                         const float *a,
                         int lda,
                         const float *w,
                         const float *bias,
                         int m,
                         int n,
                         int k) {
    if (m == 1) {
        row_neon(y, a, w, bias, n, k);
        return;
    }
    for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) {
        const int mend = min_int(m0 + PYSEG_MPANEL, m);
        const int m4 = m0 + (mend - m0) / 4 * 4;
        int i = 0;
        for (; i + 4 <= n; i += 4)
            for (int s = m0; s < m4; s += 4)
                dot4x4_neon(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k,
                            bias ? bias + i : NULL, k);
        for (int s = m0; s < m4; ++s)
            if (i < n)
                row_neon(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k,
                         bias ? bias + i : NULL, n - i, k);
        for (int s = m4; s < mend; ++s)
            row_neon(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k);
    }
}

/* Vector exponential, the Cephes single-precision scheme: split the argument
 * into an integer power of two and a remainder inside half a ln 2, evaluate a
 * degree-six polynomial on the remainder, then scale by assembling the power
 * of two directly in the exponent field. Accurate to about one unit in the
 * last place, which is the same claim libm makes.
 *
 * The clamp is what keeps the exponent assembly in range; beyond it the
 * logistic and hyperbolic functions built on this are saturated anyway. */
PYSEG_INLINE float32x4_t vexpq(float32x4_t x) {
    x = vminq_f32(vmaxq_f32(x, vdupq_n_f32(-88.0f)), vdupq_n_f32(88.0f));

    const float32x4_t n = vrndnq_f32(vmulq_f32(x, vdupq_n_f32(1.44269504088896341f)));
    /* ln 2 split in two so the product with n stays exact in the high part. */
    float32x4_t r = vfmsq_f32(x, n, vdupq_n_f32(0.693359375f));
    r = vfmsq_f32(r, n, vdupq_n_f32(-2.12194440e-4f));

    float32x4_t p = vdupq_n_f32(1.9875691500e-4f);
    p = vfmaq_f32(vdupq_n_f32(1.3981999507e-3f), p, r);
    p = vfmaq_f32(vdupq_n_f32(8.3334519073e-3f), p, r);
    p = vfmaq_f32(vdupq_n_f32(4.1665795894e-2f), p, r);
    p = vfmaq_f32(vdupq_n_f32(1.6666665459e-1f), p, r);
    p = vfmaq_f32(vdupq_n_f32(5.0000001201e-1f), p, r);
    p = vfmaq_f32(vaddq_f32(r, vdupq_n_f32(1.0f)), p, vmulq_f32(r, r));

    const int32x4_t bits = vshlq_n_s32(vaddq_s32(vcvtq_s32_f32(n), vdupq_n_s32(127)), 23);
    return vmulq_f32(p, vreinterpretq_f32_s32(bits));
}

PYSEG_INLINE float32x4_t vsigmoidq(float32x4_t x) {
    return vdivq_f32(vdupq_n_f32(1.0f), vaddq_f32(vdupq_n_f32(1.0f), vexpq(vnegq_f32(x))));
}

/* tanh x = 2 sigmoid(2x) - 1. The subtraction loses relative accuracy near
 * zero, where sigmoid is close to a half, but the absolute error stays at the
 * rounding level of the result and that is what propagates into the cell
 * state. */
PYSEG_INLINE float32x4_t vtanhq(float32x4_t x) {
    return vfmsq_f32(vdupq_n_f32(-1.0f), vsigmoidq(vaddq_f32(x, x)), vdupq_n_f32(-2.0f));
}

static void lstm_cell_neon(float *h, float *c, const float *rec, const float *in, int hid) {
    for (int i = 0; i < hid; i += 4) {
        const float32x4_t ig = vsigmoidq(vaddq_f32(vld1q_f32(rec + i), vld1q_f32(in + i)));
        const float32x4_t fg =
            vsigmoidq(vaddq_f32(vld1q_f32(rec + hid + i), vld1q_f32(in + hid + i)));
        const float32x4_t gg =
            vtanhq(vaddq_f32(vld1q_f32(rec + 2 * hid + i), vld1q_f32(in + 2 * hid + i)));
        const float32x4_t og =
            vsigmoidq(vaddq_f32(vld1q_f32(rec + 3 * hid + i), vld1q_f32(in + 3 * hid + i)));
        const float32x4_t cn = vfmaq_f32(vmulq_f32(ig, gg), fg, vld1q_f32(c + i));
        vst1q_f32(c + i, cn);
        vst1q_f32(h + i, vmulq_f32(og, vtanhq(cn)));
    }
}

static const pyseg_kernels kernels_simd = {"neon", gemm_nt_neon, lstm_cell_neon};
#endif /* aarch64 NEON */

/* ---- x86-64 AVX2 -----------------------------------------------------
 *
 * Compiled into every x86 build whatever baseline the rest of the file uses,
 * by carrying the instruction set on the functions rather than on the command
 * line, and selected at startup against the running processor. One binary is
 * then both safe on a host without AVX2 and fast on one with it.
 *
 * Every function reachable from an AVX2 entry point needs the same attribute,
 * including the always-inlined ones, or the compiler refuses to inline across
 * the mismatch. */

#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__))
#    define PYSEG_HAVE_SIMD    1
#    define PYSEG_X86_DISPATCH 1
#    include <immintrin.h>

#    define PYSEG_AVX2        __attribute__((target("avx2,fma")))
#    define PYSEG_AVX2_INLINE static inline __attribute__((always_inline, target("avx2,fma")))

PYSEG_AVX2_INLINE float hsum256(__m256 v) {
    __m128 lo = _mm256_castps256_ps128(v);
    __m128 hi = _mm256_extractf128_ps(v, 1);
    lo = _mm_add_ps(lo, hi);
    lo = _mm_add_ps(lo, _mm_movehl_ps(lo, lo));
    lo = _mm_add_ss(lo, _mm_shuffle_ps(lo, lo, 0x55));
    return _mm_cvtss_f32(lo);
}

/* Four output positions against two weight rows: eight accumulators, which is
 * half the register file. That is the shape this instruction set allows. The
 * NEON kernel above blocks four by four because it has thirty-two vector
 * registers to hold sixteen accumulators plus the operands; asking for the
 * same here spills the accumulators to the stack every iteration and costs
 * more than the blocking saves.
 *
 * The window length is a multiple of four but not always of eight, so the
 * eight-wide body is followed by at most one four-wide step, done in scalar
 * code since it is at most four elements of at least sixty. */
PYSEG_AVX2_INLINE void
dot4x2_avx2(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) {
    const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda;
    const float *w0 = w, *w1 = w + k;

    __m256 c00 = _mm256_setzero_ps(), c01 = c00, c10 = c00, c11 = c00;
    __m256 c20 = c00, c21 = c00, c30 = c00, c31 = c00;

    int j = 0;
    for (; j + 8 <= k; j += 8) {
        const __m256 v0 = _mm256_loadu_ps(w0 + j);
        const __m256 v1 = _mm256_loadu_ps(w1 + j);
        const __m256 x0 = _mm256_loadu_ps(a0 + j);
        const __m256 x1 = _mm256_loadu_ps(a1 + j);
        const __m256 x2 = _mm256_loadu_ps(a2 + j);
        const __m256 x3 = _mm256_loadu_ps(a3 + j);
        c00 = _mm256_fmadd_ps(x0, v0, c00);
        c01 = _mm256_fmadd_ps(x0, v1, c01);
        c10 = _mm256_fmadd_ps(x1, v0, c10);
        c11 = _mm256_fmadd_ps(x1, v1, c11);
        c20 = _mm256_fmadd_ps(x2, v0, c20);
        c21 = _mm256_fmadd_ps(x2, v1, c21);
        c30 = _mm256_fmadd_ps(x3, v0, c30);
        c31 = _mm256_fmadd_ps(x3, v1, c31);
    }

    const float *ar[4] = {a0, a1, a2, a3};
    __m256 acc[4][2] = {{c00, c01}, {c10, c11}, {c20, c21}, {c30, c31}};
    for (int s = 0; s < 4; ++s)
        for (int i = 0; i < 2; ++i) {
            float v = hsum256(acc[s][i]);
            for (int t = j; t < k; ++t)
                v += ar[s][t] * w[(size_t)i * k + t];
            y[(size_t)s * ldy + i] = bias ? bias[i] + v : v;
        }
}

/* Single output position, eight weight rows at a time; the matrix-vector shape
 * of the LSTM recurrence. Eight accumulators fit, and one activation load then
 * serves eight weight loads. */
PYSEG_AVX2_INLINE void
row_avx2(float *y, const float *a, const float *w, const float *bias, int n, int k) {
    int i = 0;
    for (; i + 8 <= n; i += 8) {
        const float *w0 = w + (size_t)i * k;
        __m256 c0 = _mm256_setzero_ps(), c1 = c0, c2 = c0, c3 = c0;
        __m256 c4 = c0, c5 = c0, c6 = c0, c7 = c0;
        int j = 0;
        for (; j + 8 <= k; j += 8) {
            const __m256 x = _mm256_loadu_ps(a + j);
            const float *p = w0 + j;
            c0 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p), c0);
            c1 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + k), c1);
            c2 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 2 * k), c2);
            c3 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 3 * k), c3);
            c4 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 4 * k), c4);
            c5 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 5 * k), c5);
            c6 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 6 * k), c6);
            c7 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 7 * k), c7);
        }
        const __m256 acc[8] = {c0, c1, c2, c3, c4, c5, c6, c7};
        for (int q = 0; q < 8; ++q) {
            float v = hsum256(acc[q]);
            for (int t = j; t < k; ++t)
                v += a[t] * w0[(size_t)q * k + t];
            y[i + q] = bias ? bias[i + q] + v : v;
        }
    }
    for (; i < n; ++i) {
        const float *wr = w + (size_t)i * k;
        __m256 acc = _mm256_setzero_ps();
        int j = 0;
        for (; j + 8 <= k; j += 8)
            acc = _mm256_fmadd_ps(_mm256_loadu_ps(a + j), _mm256_loadu_ps(wr + j), acc);
        float v = hsum256(acc);
        for (; j < k; ++j)
            v += a[j] * wr[j];
        y[i] = bias ? bias[i] + v : v;
    }
}

PYSEG_AVX2 static void gemm_nt_avx2(float *y,
                                    int ldy,
                                    const float *a,
                                    int lda,
                                    const float *w,
                                    const float *bias,
                                    int m,
                                    int n,
                                    int k) {
    if (m == 1) {
        row_avx2(y, a, w, bias, n, k);
        return;
    }
    for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) {
        const int mend = min_int(m0 + PYSEG_MPANEL, m);
        const int m4 = m0 + (mend - m0) / 4 * 4;
        int i = 0;
        for (; i + 2 <= n; i += 2)
            for (int s = m0; s < m4; s += 4)
                dot4x2_avx2(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k,
                            bias ? bias + i : NULL, k);
        for (int s = m0; s < m4; ++s)
            if (i < n)
                row_avx2(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k,
                         bias ? bias + i : NULL, n - i, k);
        for (int s = m4; s < mend; ++s)
            row_avx2(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k);
    }
}

/* The same Cephes scheme as the NEON path; see the comment there. */
PYSEG_AVX2_INLINE __m256 vexp256(__m256 x) {
    x = _mm256_min_ps(_mm256_max_ps(x, _mm256_set1_ps(-88.0f)), _mm256_set1_ps(88.0f));

    const __m256 n = _mm256_round_ps(_mm256_mul_ps(x, _mm256_set1_ps(1.44269504088896341f)),
                                     _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
    __m256 r = _mm256_fnmadd_ps(n, _mm256_set1_ps(0.693359375f), x);
    r = _mm256_fnmadd_ps(n, _mm256_set1_ps(-2.12194440e-4f), r);

    __m256 p = _mm256_set1_ps(1.9875691500e-4f);
    p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.3981999507e-3f));
    p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(8.3334519073e-3f));
    p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(4.1665795894e-2f));
    p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.6666665459e-1f));
    p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(5.0000001201e-1f));
    p = _mm256_fmadd_ps(p, _mm256_mul_ps(r, r), _mm256_add_ps(r, _mm256_set1_ps(1.0f)));

    const __m256i bits =
        _mm256_slli_epi32(_mm256_add_epi32(_mm256_cvtps_epi32(n), _mm256_set1_epi32(127)), 23);
    return _mm256_mul_ps(p, _mm256_castsi256_ps(bits));
}

PYSEG_AVX2_INLINE __m256 vsigmoid256(__m256 x) {
    const __m256 one = _mm256_set1_ps(1.0f);
    return _mm256_div_ps(one, _mm256_add_ps(one, vexp256(_mm256_sub_ps(_mm256_setzero_ps(), x))));
}

PYSEG_AVX2_INLINE __m256 vtanh256(__m256 x) {
    return _mm256_fmsub_ps(vsigmoid256(_mm256_add_ps(x, x)), _mm256_set1_ps(2.0f),
                           _mm256_set1_ps(1.0f));
}

PYSEG_AVX2_INLINE __m256 gate256(const float *rec, const float *in, int offset) {
    return _mm256_add_ps(_mm256_loadu_ps(rec + offset), _mm256_loadu_ps(in + offset));
}

PYSEG_AVX2 static void
lstm_cell_avx2(float *h, float *c, const float *rec, const float *in, int hid) {
    int i = 0;
    for (; i + 8 <= hid; i += 8) {
        const __m256 ig = vsigmoid256(gate256(rec, in, i));
        const __m256 fg = vsigmoid256(gate256(rec, in, hid + i));
        const __m256 gg = vtanh256(gate256(rec, in, 2 * hid + i));
        const __m256 og = vsigmoid256(gate256(rec, in, 3 * hid + i));
        const __m256 cn = _mm256_fmadd_ps(fg, _mm256_loadu_ps(c + i), _mm256_mul_ps(ig, gg));
        _mm256_storeu_ps(c + i, cn);
        _mm256_storeu_ps(h + i, _mm256_mul_ps(og, vtanh256(cn)));
    }
    for (; i < hid; ++i) {
        const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i])));
        const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i])));
        const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]);
        const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i])));
        const float cn = fg * c[i] + ig * gg;
        c[i] = cn;
        h[i] = og * tanhf(cn);
    }
}

static const pyseg_kernels kernels_avx2 = {"avx2", gemm_nt_avx2, lstm_cell_avx2};

/* AVX2 also needs the operating system to have enabled the wider register
 * state, which __builtin_cpu_supports checks along with the feature bits. */
static int cpu_has_avx2(void) {
#    if defined(__has_builtin)
#        if __has_builtin(__builtin_cpu_init)
    __builtin_cpu_init();
#        endif
#    elif defined(__GNUC__)
    __builtin_cpu_init();
#    endif
    return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
}
#endif /* x86 */

const pyseg_kernels *pyseg_kernels_simd(void) {
#if defined(PYSEG_X86_DISPATCH)
    /* Resolved once; the answer cannot change while the process runs. */
    static const pyseg_kernels *chosen;
    if (!chosen)
        chosen = cpu_has_avx2() ? &kernels_avx2 : &kernels_scalar;
    return chosen;
#elif defined(PYSEG_HAVE_SIMD)
    return &kernels_simd;
#else
    return &kernels_scalar;
#endif
}