#include "gemm.h" #include #include void facedet_gemm_pack_a(const float *a, int m, int k, float *pack) { int mt = (m + 7) / 8; for (int t = 0; t < mt; t++) { float *p = pack + (size_t)t * 8 * k; for (int kk = 0; kk < k; kk++) for (int r = 0; r < 8; r++) { int row = t * 8 + r; p[(size_t)kk * 8 + r] = row < m ? a[(size_t)row * k + kk] : 0.0f; } } } size_t facedet_gemm_pack_a_bytes(int m, int k) { return (size_t)((m + 7) / 8) * 8 * k * sizeof(float); } #if defined(__aarch64__) && !defined(FACEDET_SCALAR) # include static void kernel_8x12(const float *ap, const float *bp, int k, const float *bias, int relu, float *c, size_t ldc, int mr, int nr) { float32x4_t acc[8][3]; for (int r = 0; r < 8; r++) for (int j = 0; j < 3; j++) acc[r][j] = vdupq_n_f32(0.0f); for (int kk = 0; kk < k; kk++) { float32x4_t b0 = vld1q_f32(bp + (size_t)kk * 12); float32x4_t b1 = vld1q_f32(bp + (size_t)kk * 12 + 4); float32x4_t b2 = vld1q_f32(bp + (size_t)kk * 12 + 8); float32x4_t a0 = vld1q_f32(ap + (size_t)kk * 8); float32x4_t a1 = vld1q_f32(ap + (size_t)kk * 8 + 4); acc[0][0] = vfmaq_laneq_f32(acc[0][0], b0, a0, 0); acc[0][1] = vfmaq_laneq_f32(acc[0][1], b1, a0, 0); acc[0][2] = vfmaq_laneq_f32(acc[0][2], b2, a0, 0); acc[1][0] = vfmaq_laneq_f32(acc[1][0], b0, a0, 1); acc[1][1] = vfmaq_laneq_f32(acc[1][1], b1, a0, 1); acc[1][2] = vfmaq_laneq_f32(acc[1][2], b2, a0, 1); acc[2][0] = vfmaq_laneq_f32(acc[2][0], b0, a0, 2); acc[2][1] = vfmaq_laneq_f32(acc[2][1], b1, a0, 2); acc[2][2] = vfmaq_laneq_f32(acc[2][2], b2, a0, 2); acc[3][0] = vfmaq_laneq_f32(acc[3][0], b0, a0, 3); acc[3][1] = vfmaq_laneq_f32(acc[3][1], b1, a0, 3); acc[3][2] = vfmaq_laneq_f32(acc[3][2], b2, a0, 3); acc[4][0] = vfmaq_laneq_f32(acc[4][0], b0, a1, 0); acc[4][1] = vfmaq_laneq_f32(acc[4][1], b1, a1, 0); acc[4][2] = vfmaq_laneq_f32(acc[4][2], b2, a1, 0); acc[5][0] = vfmaq_laneq_f32(acc[5][0], b0, a1, 1); acc[5][1] = vfmaq_laneq_f32(acc[5][1], b1, a1, 1); acc[5][2] = vfmaq_laneq_f32(acc[5][2], b2, a1, 1); acc[6][0] = vfmaq_laneq_f32(acc[6][0], b0, a1, 2); acc[6][1] = vfmaq_laneq_f32(acc[6][1], b1, a1, 2); acc[6][2] = vfmaq_laneq_f32(acc[6][2], b2, a1, 2); acc[7][0] = vfmaq_laneq_f32(acc[7][0], b0, a1, 3); acc[7][1] = vfmaq_laneq_f32(acc[7][1], b1, a1, 3); acc[7][2] = vfmaq_laneq_f32(acc[7][2], b2, a1, 3); } float tmp[12]; for (int r = 0; r < mr; r++) { float32x4_t bv = vdupq_n_f32(bias ? bias[r] : 0.0f); float32x4_t v0 = vaddq_f32(acc[r][0], bv); float32x4_t v1 = vaddq_f32(acc[r][1], bv); float32x4_t v2 = vaddq_f32(acc[r][2], bv); if (relu) { float32x4_t z = vdupq_n_f32(0.0f); v0 = vmaxq_f32(v0, z); v1 = vmaxq_f32(v1, z); v2 = vmaxq_f32(v2, z); } if (nr == 12) { vst1q_f32(c + (size_t)r * ldc, v0); vst1q_f32(c + (size_t)r * ldc + 4, v1); vst1q_f32(c + (size_t)r * ldc + 8, v2); } else { vst1q_f32(tmp, v0); vst1q_f32(tmp + 4, v1); vst1q_f32(tmp + 8, v2); for (int j = 0; j < nr; j++) c[(size_t)r * ldc + j] = tmp[j]; } } } #else static void kernel_8x12_scalar(const float *ap, const float *bp, int k, const float *bias, int relu, float *c, size_t ldc, int mr, int nr) { float acc[8][12] = {{0}}; for (int kk = 0; kk < k; kk++) for (int r = 0; r < 8; r++) { float a = ap[(size_t)kk * 8 + r]; for (int j = 0; j < 12; j++) acc[r][j] += a * bp[(size_t)kk * 12 + j]; } for (int r = 0; r < mr; r++) for (int j = 0; j < nr; j++) { float v = acc[r][j] + (bias ? bias[r] : 0.0f); c[(size_t)r * ldc + j] = relu && v < 0.0f ? 0.0f : v; } } # if defined(__x86_64__) && !defined(FACEDET_SCALAR) # include __attribute__((target("avx2,fma"))) static void kernel_8x12_avx2(const float *ap, const float *bp, int k, const float *bias, int relu, float *c, size_t ldc, int mr, int nr) { float tmp[12]; for (int half = 0; half < 2; half++) { int r0 = half * 4; if (r0 >= mr) break; __m256 acc0[4]; __m128 acc1[4]; for (int i = 0; i < 4; i++) { acc0[i] = _mm256_setzero_ps(); acc1[i] = _mm_setzero_ps(); } for (int kk = 0; kk < k; kk++) { __m256 b0 = _mm256_loadu_ps(bp + (size_t)kk * 12); __m128 b1 = _mm_loadu_ps(bp + (size_t)kk * 12 + 8); for (int i = 0; i < 4; i++) { float a = ap[(size_t)kk * 8 + r0 + i]; acc0[i] = _mm256_fmadd_ps(_mm256_set1_ps(a), b0, acc0[i]); acc1[i] = _mm_fmadd_ps(_mm_set1_ps(a), b1, acc1[i]); } } for (int i = 0; i < 4 && r0 + i < mr; i++) { int r = r0 + i; float bv = bias ? bias[r] : 0.0f; __m256 v0 = _mm256_add_ps(acc0[i], _mm256_set1_ps(bv)); __m128 v1 = _mm_add_ps(acc1[i], _mm_set1_ps(bv)); if (relu) { v0 = _mm256_max_ps(v0, _mm256_setzero_ps()); v1 = _mm_max_ps(v1, _mm_setzero_ps()); } if (nr == 12) { _mm256_storeu_ps(c + (size_t)r * ldc, v0); _mm_storeu_ps(c + (size_t)r * ldc + 8, v1); } else { _mm256_storeu_ps(tmp, v0); _mm_storeu_ps(tmp + 8, v1); for (int j = 0; j < nr; j++) c[(size_t)r * ldc + j] = tmp[j]; } } } } static int have_avx2; __attribute__((constructor)) static void detect_avx2(void) { have_avx2 = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); } static void kernel_8x12(const float *ap, const float *bp, int k, const float *bias, int relu, float *c, size_t ldc, int mr, int nr) { if (have_avx2) kernel_8x12_avx2(ap, bp, k, bias, relu, c, ldc, mr, nr); else kernel_8x12_scalar(ap, bp, k, bias, relu, c, ldc, mr, nr); } # else static void kernel_8x12(const float *ap, const float *bp, int k, const float *bias, int relu, float *c, size_t ldc, int mr, int nr) { kernel_8x12_scalar(ap, bp, k, bias, relu, c, ldc, mr, nr); } # endif #endif void facedet_gemm(const float *a_pack, int m, int k, const float *b, size_t ldb, int n, const float *bias, int relu, float *c, size_t ldc, float *bscratch) { for (int nt = 0; nt < n; nt += 12) { int nr = n - nt < 12 ? n - nt : 12; float *bp = bscratch; for (int kk = 0; kk < k; kk++) { const float *src = b + (size_t)kk * ldb + nt; float *dst = bp + (size_t)kk * 12; int j = 0; for (; j < nr; j++) dst[j] = src[j]; for (; j < 12; j++) dst[j] = 0.0f; } for (int mt = 0; mt < m; mt += 8) { int mr = m - mt < 8 ? m - mt : 8; kernel_8x12(a_pack + (size_t)(mt / 8) * 8 * k, bp, k, bias ? bias + mt : NULL, relu, c + (size_t)mt * ldc + nt, ldc, mr, nr); } } }