diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-10-20 11:44:30 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-10-20 11:44:30 +0200 |
| commit | 9b9005101e6170a226acb265428aac429be6edbf (patch) | |
| tree | f89c76f4a5c2a00a72abe9b92468ac9f15083b94 /gemm.c | |
| download | facedet-9b9005101e6170a226acb265428aac429be6edbf.tar.gz facedet-9b9005101e6170a226acb265428aac429be6edbf.zip | |
Init
Diffstat (limited to 'gemm.c')
| -rw-r--r-- | gemm.c | 234 |
1 files changed, 234 insertions, 0 deletions
@@ -0,0 +1,234 @@ +#include "gemm.h" + +#include <stdlib.h> +#include <string.h> + +void yunet_gemm_pack_a(const float *a, int m, int k, float *pack) { + int mt = (m + 7) / 8; + for (int t = 0; t < mt; t++) { + float *p = pack + (size_t)t * 8 * k; + for (int kk = 0; kk < k; kk++) + for (int r = 0; r < 8; r++) { + int row = t * 8 + r; + p[(size_t)kk * 8 + r] = row < m ? a[(size_t)row * k + kk] : 0.0f; + } + } +} + +size_t yunet_gemm_pack_a_bytes(int m, int k) { return (size_t)((m + 7) / 8) * 8 * k * sizeof(float); } + +#if defined(__aarch64__) && !defined(YUNET_SCALAR) +# include <arm_neon.h> + +static void kernel_8x12(const float *ap, + const float *bp, + int k, + const float *bias, + int relu, + float *c, + size_t ldc, + int mr, + int nr) { + float32x4_t acc[8][3]; + for (int r = 0; r < 8; r++) + for (int j = 0; j < 3; j++) + acc[r][j] = vdupq_n_f32(0.0f); + for (int kk = 0; kk < k; kk++) { + float32x4_t b0 = vld1q_f32(bp + (size_t)kk * 12); + float32x4_t b1 = vld1q_f32(bp + (size_t)kk * 12 + 4); + float32x4_t b2 = vld1q_f32(bp + (size_t)kk * 12 + 8); + float32x4_t a0 = vld1q_f32(ap + (size_t)kk * 8); + float32x4_t a1 = vld1q_f32(ap + (size_t)kk * 8 + 4); + acc[0][0] = vfmaq_laneq_f32(acc[0][0], b0, a0, 0); + acc[0][1] = vfmaq_laneq_f32(acc[0][1], b1, a0, 0); + acc[0][2] = vfmaq_laneq_f32(acc[0][2], b2, a0, 0); + acc[1][0] = vfmaq_laneq_f32(acc[1][0], b0, a0, 1); + acc[1][1] = vfmaq_laneq_f32(acc[1][1], b1, a0, 1); + acc[1][2] = vfmaq_laneq_f32(acc[1][2], b2, a0, 1); + acc[2][0] = vfmaq_laneq_f32(acc[2][0], b0, a0, 2); + acc[2][1] = vfmaq_laneq_f32(acc[2][1], b1, a0, 2); + acc[2][2] = vfmaq_laneq_f32(acc[2][2], b2, a0, 2); + acc[3][0] = vfmaq_laneq_f32(acc[3][0], b0, a0, 3); + acc[3][1] = vfmaq_laneq_f32(acc[3][1], b1, a0, 3); + acc[3][2] = vfmaq_laneq_f32(acc[3][2], b2, a0, 3); + acc[4][0] = vfmaq_laneq_f32(acc[4][0], b0, a1, 0); + acc[4][1] = vfmaq_laneq_f32(acc[4][1], b1, a1, 0); + acc[4][2] = vfmaq_laneq_f32(acc[4][2], b2, a1, 0); + acc[5][0] = vfmaq_laneq_f32(acc[5][0], b0, a1, 1); + acc[5][1] = vfmaq_laneq_f32(acc[5][1], b1, a1, 1); + acc[5][2] = vfmaq_laneq_f32(acc[5][2], b2, a1, 1); + acc[6][0] = vfmaq_laneq_f32(acc[6][0], b0, a1, 2); + acc[6][1] = vfmaq_laneq_f32(acc[6][1], b1, a1, 2); + acc[6][2] = vfmaq_laneq_f32(acc[6][2], b2, a1, 2); + acc[7][0] = vfmaq_laneq_f32(acc[7][0], b0, a1, 3); + acc[7][1] = vfmaq_laneq_f32(acc[7][1], b1, a1, 3); + acc[7][2] = vfmaq_laneq_f32(acc[7][2], b2, a1, 3); + } + float tmp[12]; + for (int r = 0; r < mr; r++) { + float32x4_t bv = vdupq_n_f32(bias ? bias[r] : 0.0f); + float32x4_t v0 = vaddq_f32(acc[r][0], bv); + float32x4_t v1 = vaddq_f32(acc[r][1], bv); + float32x4_t v2 = vaddq_f32(acc[r][2], bv); + if (relu) { + float32x4_t z = vdupq_n_f32(0.0f); + v0 = vmaxq_f32(v0, z); + v1 = vmaxq_f32(v1, z); + v2 = vmaxq_f32(v2, z); + } + if (nr == 12) { + vst1q_f32(c + (size_t)r * ldc, v0); + vst1q_f32(c + (size_t)r * ldc + 4, v1); + vst1q_f32(c + (size_t)r * ldc + 8, v2); + } else { + vst1q_f32(tmp, v0); + vst1q_f32(tmp + 4, v1); + vst1q_f32(tmp + 8, v2); + for (int j = 0; j < nr; j++) + c[(size_t)r * ldc + j] = tmp[j]; + } + } +} +#else +static void kernel_8x12_scalar(const float *ap, + const float *bp, + int k, + const float *bias, + int relu, + float *c, + size_t ldc, + int mr, + int nr) { + float acc[8][12] = {{0}}; + for (int kk = 0; kk < k; kk++) + for (int r = 0; r < 8; r++) { + float a = ap[(size_t)kk * 8 + r]; + for (int j = 0; j < 12; j++) + acc[r][j] += a * bp[(size_t)kk * 12 + j]; + } + for (int r = 0; r < mr; r++) + for (int j = 0; j < nr; j++) { + float v = acc[r][j] + (bias ? bias[r] : 0.0f); + c[(size_t)r * ldc + j] = relu && v < 0.0f ? 0.0f : v; + } +} + +# if defined(__x86_64__) && !defined(YUNET_SCALAR) +# include <immintrin.h> + +__attribute__((target("avx2,fma"))) static void kernel_8x12_avx2(const float *ap, + const float *bp, + int k, + const float *bias, + int relu, + float *c, + size_t ldc, + int mr, + int nr) { + float tmp[12]; + for (int half = 0; half < 2; half++) { + int r0 = half * 4; + if (r0 >= mr) + break; + __m256 acc0[4]; + __m128 acc1[4]; + for (int i = 0; i < 4; i++) { + acc0[i] = _mm256_setzero_ps(); + acc1[i] = _mm_setzero_ps(); + } + for (int kk = 0; kk < k; kk++) { + __m256 b0 = _mm256_loadu_ps(bp + (size_t)kk * 12); + __m128 b1 = _mm_loadu_ps(bp + (size_t)kk * 12 + 8); + for (int i = 0; i < 4; i++) { + float a = ap[(size_t)kk * 8 + r0 + i]; + acc0[i] = _mm256_fmadd_ps(_mm256_set1_ps(a), b0, acc0[i]); + acc1[i] = _mm_fmadd_ps(_mm_set1_ps(a), b1, acc1[i]); + } + } + for (int i = 0; i < 4 && r0 + i < mr; i++) { + int r = r0 + i; + float bv = bias ? bias[r] : 0.0f; + __m256 v0 = _mm256_add_ps(acc0[i], _mm256_set1_ps(bv)); + __m128 v1 = _mm_add_ps(acc1[i], _mm_set1_ps(bv)); + if (relu) { + v0 = _mm256_max_ps(v0, _mm256_setzero_ps()); + v1 = _mm_max_ps(v1, _mm_setzero_ps()); + } + if (nr == 12) { + _mm256_storeu_ps(c + (size_t)r * ldc, v0); + _mm_storeu_ps(c + (size_t)r * ldc + 8, v1); + } else { + _mm256_storeu_ps(tmp, v0); + _mm_storeu_ps(tmp + 8, v1); + for (int j = 0; j < nr; j++) + c[(size_t)r * ldc + j] = tmp[j]; + } + } + } +} + +static int have_avx2; + +__attribute__((constructor)) static void detect_avx2(void) { + have_avx2 = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); +} + +static void kernel_8x12(const float *ap, + const float *bp, + int k, + const float *bias, + int relu, + float *c, + size_t ldc, + int mr, + int nr) { + if (have_avx2) + kernel_8x12_avx2(ap, bp, k, bias, relu, c, ldc, mr, nr); + else + kernel_8x12_scalar(ap, bp, k, bias, relu, c, ldc, mr, nr); +} +# else +static void kernel_8x12(const float *ap, + const float *bp, + int k, + const float *bias, + int relu, + float *c, + size_t ldc, + int mr, + int nr) { + kernel_8x12_scalar(ap, bp, k, bias, relu, c, ldc, mr, nr); +} +# endif +#endif + +void yunet_gemm(const float *a_pack, + int m, + int k, + const float *b, + size_t ldb, + int n, + const float *bias, + int relu, + float *c, + size_t ldc, + float *bscratch) { + for (int nt = 0; nt < n; nt += 12) { + int nr = n - nt < 12 ? n - nt : 12; + float *bp = bscratch; + for (int kk = 0; kk < k; kk++) { + const float *src = b + (size_t)kk * ldb + nt; + float *dst = bp + (size_t)kk * 12; + int j = 0; + for (; j < nr; j++) + dst[j] = src[j]; + for (; j < 12; j++) + dst[j] = 0.0f; + } + for (int mt = 0; mt < m; mt += 8) { + int mr = m - mt < 8 ? m - mt : 8; + kernel_8x12(a_pack + (size_t)(mt / 8) * 8 * k, bp, k, bias ? bias + mt : NULL, relu, + c + (size_t)mt * ldc + nt, ldc, mr, nr); + } + } +} |
