aboutsummaryrefslogtreecommitdiff
path: root/libffwd/impl.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
commit37cbaef9ceb828c667da8f398027ba5415ee7475 (patch)
tree6103e92a9030766e78a35df00f7a1155d0d0ecf5 /libffwd/impl.h
parentcf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff)
downloadlangdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz
langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip
Rename CPU library API and sources to langdet
Diffstat (limited to 'libffwd/impl.h')
-rw-r--r--libffwd/impl.h254
1 files changed, 0 insertions, 254 deletions
diff --git a/libffwd/impl.h b/libffwd/impl.h
deleted file mode 100644
index f216518..0000000
--- a/libffwd/impl.h
+++ /dev/null
@@ -1,254 +0,0 @@
-/* Inline vector operations keep attention rows in registers across reductions. */
-
-#ifndef KERNELS_IMPL_H
-#define KERNELS_IMPL_H
-
-#include <stdint.h>
-
-float dot_f32_generic(const float *a, const float *b, int n);
-void vec_scale_inplace_generic(float *dst, float scale, int n);
-void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
-void vec_scale_add_generic(float *dst, const float *src, float correction, int n);
-
-#ifdef __ARM_NEON
-
-# include <arm_neon.h>
-
-static inline float dot_f32_impl(const float *a, const float *b, int n) {
- int i = 0;
- float32x4_t acc0 = vdupq_n_f32(0.0f);
- float32x4_t acc1 = vdupq_n_f32(0.0f);
- for (; i + 8 <= n; i += 8) {
- float32x4_t a0 = vld1q_f32(a + i);
- float32x4_t b0 = vld1q_f32(b + i);
- float32x4_t a1 = vld1q_f32(a + i + 4);
- float32x4_t b1 = vld1q_f32(b + i + 4);
- acc0 = vfmaq_f32(acc0, a0, b0);
- acc1 = vfmaq_f32(acc1, a1, b1);
- }
- float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-}
-
-static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
- int i = 0;
- float32x4_t s = vdupq_n_f32(scale);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
- vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-}
-
-static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
- int i = 0;
- float32x4_t a = vdupq_n_f32(alpha);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
- vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-}
-
-static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
- int i = 0;
- float32x4_t c = vdupq_n_f32(correction);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
- vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-}
-
-#elif defined(__AVX2__) && defined(__FMA__)
-
-# include <immintrin.h>
-
-static inline float dot_f32_impl(const float *a, const float *b, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 acc0 = _mm512_setzero_ps();
- __m512 acc1 = _mm512_setzero_ps();
- __m512 acc2 = _mm512_setzero_ps();
- __m512 acc3 = _mm512_setzero_ps();
- for (; i + 64 <= n; i += 64) {
- acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
- acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
- acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
- acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
- }
- __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3));
- for (; i + 16 <= n; i += 16) {
- acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
- }
- float sum = _mm512_reduce_add_ps(acc);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-# else
- int i = 0;
- __m256 acc0 = _mm256_setzero_ps();
- __m256 acc1 = _mm256_setzero_ps();
- __m256 acc2 = _mm256_setzero_ps();
- __m256 acc3 = _mm256_setzero_ps();
- for (; i + 32 <= n; i += 32) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1);
- acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2);
- acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3);
- }
- acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
- for (; i + 8 <= n; i += 8) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- }
- __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
- r = _mm_hadd_ps(r, r);
- r = _mm_hadd_ps(r, r);
- float sum = _mm_cvtss_f32(r);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-# endif
-}
-
-static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 s = _mm512_set1_ps(scale);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
- _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
- _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-# else
- int i = 0;
- __m256 s = _mm256_set1_ps(scale);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s));
- _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s));
- _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-# endif
-}
-
-static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 a = _mm512_set1_ps(alpha);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a,
- _mm512_loadu_ps(dst + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a,
- _mm512_loadu_ps(dst + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a,
- _mm512_loadu_ps(dst + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-# else
- int i = 0;
- __m256 a = _mm256_set1_ps(alpha);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a,
- _mm256_loadu_ps(dst + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a,
- _mm256_loadu_ps(dst + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a,
- _mm256_loadu_ps(dst + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-# endif
-}
-
-static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
-# if defined(__AVX512F__)
- int i = 0;
- __m512 c = _mm512_set1_ps(correction);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c,
- _mm512_loadu_ps(src + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c,
- _mm512_loadu_ps(src + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c,
- _mm512_loadu_ps(src + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-# else
- int i = 0;
- __m256 c = _mm256_set1_ps(correction);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c,
- _mm256_loadu_ps(src + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c,
- _mm256_loadu_ps(src + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c,
- _mm256_loadu_ps(src + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-# endif
-}
-
-#else /* no SIMD path: dispatch to the scalar references */
-
-# define dot_f32_impl dot_f32_generic
-# define vec_scale_inplace_impl vec_scale_inplace_generic
-# define vec_axpy_inplace_impl vec_axpy_inplace_generic
-# define vec_scale_add_impl vec_scale_add_generic
-
-#endif
-
-#endif /* KERNELS_IMPL_H */