aboutsummaryrefslogtreecommitdiff
path: root/gemm.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-10-20 11:44:30 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-10-20 11:44:30 +0200
commit9b9005101e6170a226acb265428aac429be6edbf (patch)
treef89c76f4a5c2a00a72abe9b92468ac9f15083b94 /gemm.c
downloadfacedet-9b9005101e6170a226acb265428aac429be6edbf.tar.gz
facedet-9b9005101e6170a226acb265428aac429be6edbf.zip
Init
Diffstat (limited to 'gemm.c')
-rw-r--r--gemm.c234
1 files changed, 234 insertions, 0 deletions
diff --git a/gemm.c b/gemm.c
new file mode 100644
index 0000000..2105bc3
--- /dev/null
+++ b/gemm.c
@@ -0,0 +1,234 @@
+#include "gemm.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+void yunet_gemm_pack_a(const float *a, int m, int k, float *pack) {
+ int mt = (m + 7) / 8;
+ for (int t = 0; t < mt; t++) {
+ float *p = pack + (size_t)t * 8 * k;
+ for (int kk = 0; kk < k; kk++)
+ for (int r = 0; r < 8; r++) {
+ int row = t * 8 + r;
+ p[(size_t)kk * 8 + r] = row < m ? a[(size_t)row * k + kk] : 0.0f;
+ }
+ }
+}
+
+size_t yunet_gemm_pack_a_bytes(int m, int k) { return (size_t)((m + 7) / 8) * 8 * k * sizeof(float); }
+
+#if defined(__aarch64__) && !defined(YUNET_SCALAR)
+# include <arm_neon.h>
+
+static void kernel_8x12(const float *ap,
+ const float *bp,
+ int k,
+ const float *bias,
+ int relu,
+ float *c,
+ size_t ldc,
+ int mr,
+ int nr) {
+ float32x4_t acc[8][3];
+ for (int r = 0; r < 8; r++)
+ for (int j = 0; j < 3; j++)
+ acc[r][j] = vdupq_n_f32(0.0f);
+ for (int kk = 0; kk < k; kk++) {
+ float32x4_t b0 = vld1q_f32(bp + (size_t)kk * 12);
+ float32x4_t b1 = vld1q_f32(bp + (size_t)kk * 12 + 4);
+ float32x4_t b2 = vld1q_f32(bp + (size_t)kk * 12 + 8);
+ float32x4_t a0 = vld1q_f32(ap + (size_t)kk * 8);
+ float32x4_t a1 = vld1q_f32(ap + (size_t)kk * 8 + 4);
+ acc[0][0] = vfmaq_laneq_f32(acc[0][0], b0, a0, 0);
+ acc[0][1] = vfmaq_laneq_f32(acc[0][1], b1, a0, 0);
+ acc[0][2] = vfmaq_laneq_f32(acc[0][2], b2, a0, 0);
+ acc[1][0] = vfmaq_laneq_f32(acc[1][0], b0, a0, 1);
+ acc[1][1] = vfmaq_laneq_f32(acc[1][1], b1, a0, 1);
+ acc[1][2] = vfmaq_laneq_f32(acc[1][2], b2, a0, 1);
+ acc[2][0] = vfmaq_laneq_f32(acc[2][0], b0, a0, 2);
+ acc[2][1] = vfmaq_laneq_f32(acc[2][1], b1, a0, 2);
+ acc[2][2] = vfmaq_laneq_f32(acc[2][2], b2, a0, 2);
+ acc[3][0] = vfmaq_laneq_f32(acc[3][0], b0, a0, 3);
+ acc[3][1] = vfmaq_laneq_f32(acc[3][1], b1, a0, 3);
+ acc[3][2] = vfmaq_laneq_f32(acc[3][2], b2, a0, 3);
+ acc[4][0] = vfmaq_laneq_f32(acc[4][0], b0, a1, 0);
+ acc[4][1] = vfmaq_laneq_f32(acc[4][1], b1, a1, 0);
+ acc[4][2] = vfmaq_laneq_f32(acc[4][2], b2, a1, 0);
+ acc[5][0] = vfmaq_laneq_f32(acc[5][0], b0, a1, 1);
+ acc[5][1] = vfmaq_laneq_f32(acc[5][1], b1, a1, 1);
+ acc[5][2] = vfmaq_laneq_f32(acc[5][2], b2, a1, 1);
+ acc[6][0] = vfmaq_laneq_f32(acc[6][0], b0, a1, 2);
+ acc[6][1] = vfmaq_laneq_f32(acc[6][1], b1, a1, 2);
+ acc[6][2] = vfmaq_laneq_f32(acc[6][2], b2, a1, 2);
+ acc[7][0] = vfmaq_laneq_f32(acc[7][0], b0, a1, 3);
+ acc[7][1] = vfmaq_laneq_f32(acc[7][1], b1, a1, 3);
+ acc[7][2] = vfmaq_laneq_f32(acc[7][2], b2, a1, 3);
+ }
+ float tmp[12];
+ for (int r = 0; r < mr; r++) {
+ float32x4_t bv = vdupq_n_f32(bias ? bias[r] : 0.0f);
+ float32x4_t v0 = vaddq_f32(acc[r][0], bv);
+ float32x4_t v1 = vaddq_f32(acc[r][1], bv);
+ float32x4_t v2 = vaddq_f32(acc[r][2], bv);
+ if (relu) {
+ float32x4_t z = vdupq_n_f32(0.0f);
+ v0 = vmaxq_f32(v0, z);
+ v1 = vmaxq_f32(v1, z);
+ v2 = vmaxq_f32(v2, z);
+ }
+ if (nr == 12) {
+ vst1q_f32(c + (size_t)r * ldc, v0);
+ vst1q_f32(c + (size_t)r * ldc + 4, v1);
+ vst1q_f32(c + (size_t)r * ldc + 8, v2);
+ } else {
+ vst1q_f32(tmp, v0);
+ vst1q_f32(tmp + 4, v1);
+ vst1q_f32(tmp + 8, v2);
+ for (int j = 0; j < nr; j++)
+ c[(size_t)r * ldc + j] = tmp[j];
+ }
+ }
+}
+#else
+static void kernel_8x12_scalar(const float *ap,
+ const float *bp,
+ int k,
+ const float *bias,
+ int relu,
+ float *c,
+ size_t ldc,
+ int mr,
+ int nr) {
+ float acc[8][12] = {{0}};
+ for (int kk = 0; kk < k; kk++)
+ for (int r = 0; r < 8; r++) {
+ float a = ap[(size_t)kk * 8 + r];
+ for (int j = 0; j < 12; j++)
+ acc[r][j] += a * bp[(size_t)kk * 12 + j];
+ }
+ for (int r = 0; r < mr; r++)
+ for (int j = 0; j < nr; j++) {
+ float v = acc[r][j] + (bias ? bias[r] : 0.0f);
+ c[(size_t)r * ldc + j] = relu && v < 0.0f ? 0.0f : v;
+ }
+}
+
+# if defined(__x86_64__) && !defined(YUNET_SCALAR)
+# include <immintrin.h>
+
+__attribute__((target("avx2,fma"))) static void kernel_8x12_avx2(const float *ap,
+ const float *bp,
+ int k,
+ const float *bias,
+ int relu,
+ float *c,
+ size_t ldc,
+ int mr,
+ int nr) {
+ float tmp[12];
+ for (int half = 0; half < 2; half++) {
+ int r0 = half * 4;
+ if (r0 >= mr)
+ break;
+ __m256 acc0[4];
+ __m128 acc1[4];
+ for (int i = 0; i < 4; i++) {
+ acc0[i] = _mm256_setzero_ps();
+ acc1[i] = _mm_setzero_ps();
+ }
+ for (int kk = 0; kk < k; kk++) {
+ __m256 b0 = _mm256_loadu_ps(bp + (size_t)kk * 12);
+ __m128 b1 = _mm_loadu_ps(bp + (size_t)kk * 12 + 8);
+ for (int i = 0; i < 4; i++) {
+ float a = ap[(size_t)kk * 8 + r0 + i];
+ acc0[i] = _mm256_fmadd_ps(_mm256_set1_ps(a), b0, acc0[i]);
+ acc1[i] = _mm_fmadd_ps(_mm_set1_ps(a), b1, acc1[i]);
+ }
+ }
+ for (int i = 0; i < 4 && r0 + i < mr; i++) {
+ int r = r0 + i;
+ float bv = bias ? bias[r] : 0.0f;
+ __m256 v0 = _mm256_add_ps(acc0[i], _mm256_set1_ps(bv));
+ __m128 v1 = _mm_add_ps(acc1[i], _mm_set1_ps(bv));
+ if (relu) {
+ v0 = _mm256_max_ps(v0, _mm256_setzero_ps());
+ v1 = _mm_max_ps(v1, _mm_setzero_ps());
+ }
+ if (nr == 12) {
+ _mm256_storeu_ps(c + (size_t)r * ldc, v0);
+ _mm_storeu_ps(c + (size_t)r * ldc + 8, v1);
+ } else {
+ _mm256_storeu_ps(tmp, v0);
+ _mm_storeu_ps(tmp + 8, v1);
+ for (int j = 0; j < nr; j++)
+ c[(size_t)r * ldc + j] = tmp[j];
+ }
+ }
+ }
+}
+
+static int have_avx2;
+
+__attribute__((constructor)) static void detect_avx2(void) {
+ have_avx2 = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
+}
+
+static void kernel_8x12(const float *ap,
+ const float *bp,
+ int k,
+ const float *bias,
+ int relu,
+ float *c,
+ size_t ldc,
+ int mr,
+ int nr) {
+ if (have_avx2)
+ kernel_8x12_avx2(ap, bp, k, bias, relu, c, ldc, mr, nr);
+ else
+ kernel_8x12_scalar(ap, bp, k, bias, relu, c, ldc, mr, nr);
+}
+# else
+static void kernel_8x12(const float *ap,
+ const float *bp,
+ int k,
+ const float *bias,
+ int relu,
+ float *c,
+ size_t ldc,
+ int mr,
+ int nr) {
+ kernel_8x12_scalar(ap, bp, k, bias, relu, c, ldc, mr, nr);
+}
+# endif
+#endif
+
+void yunet_gemm(const float *a_pack,
+ int m,
+ int k,
+ const float *b,
+ size_t ldb,
+ int n,
+ const float *bias,
+ int relu,
+ float *c,
+ size_t ldc,
+ float *bscratch) {
+ for (int nt = 0; nt < n; nt += 12) {
+ int nr = n - nt < 12 ? n - nt : 12;
+ float *bp = bscratch;
+ for (int kk = 0; kk < k; kk++) {
+ const float *src = b + (size_t)kk * ldb + nt;
+ float *dst = bp + (size_t)kk * 12;
+ int j = 0;
+ for (; j < nr; j++)
+ dst[j] = src[j];
+ for (; j < 12; j++)
+ dst[j] = 0.0f;
+ }
+ for (int mt = 0; mt < m; mt += 8) {
+ int mr = m - mt < 8 ? m - mt : 8;
+ kernel_8x12(a_pack + (size_t)(mt / 8) * 8 * k, bp, k, bias ? bias + mt : NULL, relu,
+ c + (size_t)mt * ldc + nt, ldc, mr, nr);
+ }
+ }
+}