aboutsummaryrefslogtreecommitdiff
path: root/libffwd/avx.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:54:13 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:54:13 +0200
commit91871be6ece9317c5d49353f6474583e7e28168d (patch)
tree8cabf2546322a9e1b724bf8413f32de54378824f /libffwd/avx.c
parent35ab9c12ba4d396e6cba99d6ea947e26becedad4 (diff)
downloadlangdet-91871be6ece9317c5d49353f6474583e7e28168d.tar.gz
langdet-91871be6ece9317c5d49353f6474583e7e28168d.zip
Add native CPU language detection CLI
Diffstat (limited to 'libffwd/avx.c')
-rw-r--r--libffwd/avx.c271
1 files changed, 271 insertions, 0 deletions
diff --git a/libffwd/avx.c b/libffwd/avx.c
new file mode 100644
index 0000000..7cbbd5a
--- /dev/null
+++ b/libffwd/avx.c
@@ -0,0 +1,271 @@
+/* x86 SIMD hot kernels: AVX2+FMA, plus AVX-512 when available.
+ *
+ * BF16 -> F32 conversion:
+ * load 16 uint16 values, zero-extend to 32-bit, then shift left 16.
+ *
+ * Uses AVX-512F+BW for 16-wide BF16 matvec, dominant when seq_len == 1.
+ * Uses AVX2+FMA / AVX-512F for F32 attention helpers, which operate on
+ * cache-resident data.
+ *
+ * BF16 matvec processes 4 output rows at once to reduce instruction overhead
+ * and improve out-of-order execution on memory-bound workloads.
+ */
+
+#include "impl.h"
+
+#if defined(__AVX2__) && defined(__FMA__)
+
+# include <immintrin.h>
+# include <string.h>
+
+/* BF16 matvec: uses AVX-512F+BW when available, AVX2 fallback otherwise */
+
+# if defined(__AVX512F__) && defined(__AVX512BW__)
+
+void bf16_matvec_fused_avx(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
+ int o = 0;
+
+ /* Process 4 output rows at a time */
+ for (; o + 3 < out_dim; o += 4) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = w0 + in_dim;
+ const uint16_t *w2 = w1 + in_dim;
+ const uint16_t *w3 = w2 + in_dim;
+
+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
+ __m512 a4 = _mm512_setzero_ps(), a5 = _mm512_setzero_ps();
+ __m512 a6 = _mm512_setzero_ps(), a7 = _mm512_setzero_ps();
+ int k = 0;
+
+ for (; k + 32 <= in_dim; k += 32) {
+ __m512 xv0 = _mm512_loadu_ps(x + k);
+ __m512 xv1 = _mm512_loadu_ps(x + k + 16);
+
+ /* Row 0 */
+ __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
+ a1 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
+
+ /* Row 1 */
+ __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
+ a3 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
+
+ /* Row 2 */
+ __m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k));
+ __m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16));
+ a4 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4);
+ a5 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5);
+
+ /* Row 3 */
+ __m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k));
+ __m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16));
+ a6 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6);
+ a7 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7);
+ }
+
+ for (; k + 16 <= in_dim; k += 16) {
+ __m512 xv = _mm512_loadu_ps(x + k);
+ __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ __m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k));
+ __m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k));
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
+ a4 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4);
+ a6 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6);
+ }
+
+ float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f);
+ float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o + 2] : 0.0f);
+ float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o + 3] : 0.0f);
+
+ for (; k < in_dim; k++) {
+ float xk = x[k];
+ uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
+ uint32_t b2 = ((uint32_t)w2[k]) << 16, b3 = ((uint32_t)w3[k]) << 16;
+ float v0, v1, v2, v3;
+ memcpy(&v0, &b0, 4);
+ memcpy(&v1, &b1, 4);
+ memcpy(&v2, &b2, 4);
+ memcpy(&v3, &b3, 4);
+ s0 += v0 * xk;
+ s1 += v1 * xk;
+ s2 += v2 * xk;
+ s3 += v3 * xk;
+ }
+ y[o] = s0;
+ y[o + 1] = s1;
+ y[o + 2] = s2;
+ y[o + 3] = s3;
+ }
+
+ /* Handle 2-row remainder */
+ for (; o + 1 < out_dim; o += 2) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = w0 + in_dim;
+ __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
+ __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
+ int k = 0;
+
+ for (; k + 32 <= in_dim; k += 32) {
+ __m512 xv0 = _mm512_loadu_ps(x + k);
+ __m512 xv1 = _mm512_loadu_ps(x + k + 16);
+ __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
+ __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
+ a1 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
+ a3 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
+ }
+ for (; k + 16 <= in_dim; k += 16) {
+ __m512 xv = _mm512_loadu_ps(x + k);
+ __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k));
+ __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
+ a0 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
+ a2 = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
+ }
+
+ float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f);
+
+ for (; k < in_dim; k++) {
+ uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
+ float v0, v1;
+ memcpy(&v0, &b0, 4);
+ memcpy(&v1, &b1, 4);
+ s0 += v0 * x[k];
+ s1 += v1 * x[k];
+ }
+ y[o] = s0;
+ y[o + 1] = s1;
+ }
+
+ /* Single remaining row */
+ for (; o < out_dim; o++) {
+ const uint16_t *w = W_bf16 + (size_t)o * in_dim;
+ __m512 acc = _mm512_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k));
+ acc = _mm512_fmadd_ps(
+ _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)),
+ _mm512_loadu_ps(x + k), acc);
+ }
+ float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f);
+ for (; k < in_dim; k++) {
+ uint32_t bits = ((uint32_t)w[k]) << 16;
+ float wv;
+ memcpy(&wv, &bits, 4);
+ sum += wv * x[k];
+ }
+ y[o] = sum;
+ }
+}
+
+# else /* AVX2 only (or AVX-512F without BW) */
+
+/* Helper: bf16 to f32 for 8 elements */
+static inline __m256 bf16x8_to_f32(__m128i raw) {
+ return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(raw), 16));
+}
+
+static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) {
+ __m256i raw = _mm256_loadu_si256((const __m256i *)src);
+ *lo = bf16x8_to_f32(_mm256_castsi256_si128(raw));
+ *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1));
+}
+
+void bf16_matvec_fused_avx(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
+ int o = 0;
+
+ for (; o + 1 < out_dim; o += 2) {
+ const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
+ const uint16_t *w1 = w0 + in_dim;
+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
+ __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256 xlo = _mm256_loadu_ps(x + k);
+ __m256 xhi = _mm256_loadu_ps(x + k + 8);
+ __m256 wlo, whi;
+ bf16x16_to_f32(w0 + k, &wlo, &whi);
+ a0 = _mm256_fmadd_ps(wlo, xlo, a0);
+ a1 = _mm256_fmadd_ps(whi, xhi, a1);
+ bf16x16_to_f32(w1 + k, &wlo, &whi);
+ a2 = _mm256_fmadd_ps(wlo, xlo, a2);
+ a3 = _mm256_fmadd_ps(whi, xhi, a3);
+ }
+ a0 = _mm256_add_ps(a0, a1);
+ a2 = _mm256_add_ps(a2, a3);
+ __m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
+ __m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1));
+ r0 = _mm_hadd_ps(r0, r1);
+ r0 = _mm_hadd_ps(r0, r0);
+ float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f);
+ float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0, r0, 1)) + (bias ? bias[o + 1] : 0.0f);
+ for (; k < in_dim; k++) {
+ uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
+ float v0, v1;
+ memcpy(&v0, &b0, 4);
+ memcpy(&v1, &b1, 4);
+ s0 += v0 * x[k];
+ s1 += v1 * x[k];
+ }
+ y[o] = s0;
+ y[o + 1] = s1;
+ }
+ for (; o < out_dim; o++) {
+ const uint16_t *w = W_bf16 + (size_t)o * in_dim;
+ __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
+ int k = 0;
+ for (; k + 16 <= in_dim; k += 16) {
+ __m256 wlo, whi;
+ bf16x16_to_f32(w + k, &wlo, &whi);
+ a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x + k), a0);
+ a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x + k + 8), a1);
+ }
+ a0 = _mm256_add_ps(a0, a1);
+ __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
+ r = _mm_hadd_ps(r, r);
+ r = _mm_hadd_ps(r, r);
+ float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f);
+ for (; k < in_dim; k++) {
+ uint32_t bits = ((uint32_t)w[k]) << 16;
+ float wv;
+ memcpy(&wv, &bits, 4);
+ sum += wv * x[k];
+ }
+ y[o] = sum;
+ }
+}
+
+# endif /* AVX-512F+BW vs AVX2 for bf16 */
+
+#endif /* __AVX2__ && __FMA__ */