From 7500042ac2c33e0501e3d2ebae5a423e61cc39d1 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 16 Jun 2026 19:10:59 +0200 Subject: kernels: split kernels.c into gemm/norm/act/attn/rope by kind --- src/kernels.c | 1271 ---------------------------------------------------- src/kernels_act.c | 95 ++++ src/kernels_attn.c | 469 +++++++++++++++++++ src/kernels_gemm.c | 434 ++++++++++++++++++ src/kernels_norm.c | 245 ++++++++++ src/kernels_rope.c | 137 ++++++ 6 files changed, 1380 insertions(+), 1271 deletions(-) delete mode 100644 src/kernels.c create mode 100644 src/kernels_act.c create mode 100644 src/kernels_attn.c create mode 100644 src/kernels_gemm.c create mode 100644 src/kernels_norm.c create mode 100644 src/kernels_rope.c (limited to 'src') diff --git a/src/kernels.c b/src/kernels.c deleted file mode 100644 index bd57d31..0000000 --- a/src/kernels.c +++ /dev/null @@ -1,1271 +0,0 @@ -#include "kernels.h" -#include "kernels_impl.h" -#include -#include -#include -#include -#include -#if (defined(__AVX512F__) || defined(__AVX2__)) && \ - (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) -#include -#endif -#if defined(__ARM_NEON) || defined(__ARM_NEON__) -#include -#endif -#ifdef __APPLE__ -#include -#else -#include -#endif - -#ifdef USE_BLAS -#ifdef __APPLE__ -#include -#else -#include -#endif -#endif - -#if defined(USE_OPENBLAS) -void openblas_set_num_threads(int num_threads); -#endif - -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif - -/* ======================================================================== - * Basic Element-wise Operations - * ======================================================================== */ - -void embed_add_inplace(float *a, const float *b, int n) { - for (int i = 0; i < n; i++) - a[i] += b[i]; -} - -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ - -void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { -#ifdef USE_BLAS - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); -#else - for (int m = 0; m < M; m++) { - for (int n = 0; n < N; n++) { - float sum = 0.0f; - for (int k = 0; k < K; k++) { - sum += A[m * K + k] * B[n * K + k]; - } - C[m * N + n] = sum; - } - } -#endif -} - -void embed_linear(float *y, - const float *x, - const float *W, - const float *b, - int seq_len, - int in_dim, - int out_dim) { -#ifdef USE_BLAS - if (seq_len == 1) { - cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, - 1); - } else { - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, - in_dim, W, in_dim, 0.0f, y, out_dim); - } - if (b != NULL) { - for (int s = 0; s < seq_len; s++) { - for (int o = 0; o < out_dim; o++) { - y[s * out_dim + o] += b[o]; - } - } - } -#else - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + s * in_dim; - float *y_row = y + s * out_dim; - for (int o = 0; o < out_dim; o++) { - const float *w_row = W + o * in_dim; - float sum = (b != NULL) ? b[o] : 0.0f; - for (int i = 0; i < in_dim; i++) { - sum += x_row[i] * w_row[i]; - } - y_row[o] = sum; - } - } -#endif -} - -void embed_linear_nobias( - float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { - embed_linear(y, x, W, NULL, seq_len, in_dim, out_dim); -} - -/* Convert bf16 buffer to f32 buffer */ -void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { -#if defined(__ARM_NEON) || defined(__ARM_NEON__) - size_t i = 0; - uint32_t *d = (uint32_t *)(void *)dst; - for (; i + 8 <= n; i += 8) { - uint16x8_t v = vld1q_u16(src + i); - uint32x4_t lo = vshlq_n_u32(vmovl_u16(vget_low_u16(v)), 16); - uint32x4_t hi = vshlq_n_u32(vmovl_u16(vget_high_u16(v)), 16); - vst1q_u32(d + i, lo); - vst1q_u32(d + i + 4, hi); - } - for (; i < n; i++) - d[i] = ((uint32_t)src[i]) << 16; -#else - uint32_t *d = (uint32_t *)(void *)dst; - for (size_t i = 0; i < n; i++) - d[i] = ((uint32_t)src[i]) << 16; -#endif -} - -/* - * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias - * Processes 2 output rows at a time to amortize x vector loads. - */ -static void bf16_matvec_fused( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { - embed_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); -} - -/* Threaded matvec: split output rows across threads */ -typedef struct { - float *y; - const float *x; - const uint16_t *W_bf16; - const float *bias; - int in_dim; - int out_dim; -} matvec_task_t; - -static void matvec_worker(int tid, int n_threads, void *arg) { - matvec_task_t *t = (matvec_task_t *)arg; - int chunk = (t->out_dim + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->out_dim) - end = t->out_dim; - if (start >= end) - return; - - bf16_matvec_fused(t->y + start, t->x, t->W_bf16 + (size_t)start * t->in_dim, - t->bias ? t->bias + start : NULL, t->in_dim, end - start); -} - -static void bf16_matvec_threaded( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { - if (embed_num_threads() <= 1) { - bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim); - return; - } - matvec_task_t task = {y, x, W_bf16, bias, in_dim, out_dim}; - parallel_for(matvec_worker, &task); -} - -typedef struct { - float *y; - const float *x; - const uint16_t *W_bf16; - const float *bias; - int seq_len; - int in_dim; - int out_dim; -} bf16_linear_rows_task_t; - -static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) { - bf16_linear_rows_task_t *t = (bf16_linear_rows_task_t *)arg; - int chunk = (t->seq_len + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->seq_len) - end = t->seq_len; - if (start >= end) - return; - - for (int s = start; s < end; s++) { - bf16_matvec_fused(t->y + (size_t)s * t->out_dim, t->x + (size_t)s * t->in_dim, t->W_bf16, - t->bias, t->in_dim, t->out_dim); - } -} - -static void bf16_linear_rows(float *y, - const float *x, - const uint16_t *W_bf16, - const float *bias, - int seq_len, - int in_dim, - int out_dim) { - bf16_linear_rows_task_t task = { - .y = y, - .x = x, - .W_bf16 = W_bf16, - .bias = bias, - .seq_len = seq_len, - .in_dim = in_dim, - .out_dim = out_dim, - }; - - if (embed_num_threads() > 1 && seq_len >= 2) - parallel_for(bf16_linear_rows_worker, &task); - else - bf16_linear_rows_worker(0, 1, &task); -} - -typedef struct { - float *a; - float *b; - const float *x; - const uint16_t *Wa_bf16; - const uint16_t *Wb_bf16; - int seq_len; - int in_dim; - int a_dim; - int b_dim; - int total_dim; -} pair_matvec_task_t; - -static void pair_matvec_range(const pair_matvec_task_t *t, int row, int start, int end) { - const float *x_row = t->x + (size_t)row * t->in_dim; - int a_end = t->a_dim; - int b_end = a_end + t->b_dim; - - if (start < a_end) { - int s = start; - int e = end < a_end ? end : a_end; - if (s < e) { - bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row, - t->Wa_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); - } - } - - if (end > a_end && start < b_end) { - int s = start > a_end ? start - a_end : 0; - int e_abs = end < b_end ? end : b_end; - int e = e_abs - a_end; - if (s < e) { - bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row, - t->Wb_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); - } - } -} - -static void pair_matvec_worker(int tid, int n_threads, void *arg) { - pair_matvec_task_t *t = (pair_matvec_task_t *)arg; - int total_work = t->seq_len * t->total_dim; - int chunk = (total_work + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > total_work) - end = total_work; - if (start >= end) - return; - - while (start < end) { - int row = start / t->total_dim; - int pos = start - row * t->total_dim; - int row_end = (row + 1) * t->total_dim; - int stop = end < row_end ? end : row_end; - pair_matvec_range(t, row, pos, stop - row * t->total_dim); - start = stop; - } -} - -void embed_linear_nobias_bf16_pair(float *a, - float *b, - const float *x, - const uint16_t *Wa_bf16, - const uint16_t *Wb_bf16, - int seq_len, - int in_dim, - int a_dim, - int b_dim) { - if (seq_len <= 0) - return; - - if (embed_num_threads() <= 1) { - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + (size_t)s * in_dim; - bf16_matvec_fused(a + (size_t)s * a_dim, x_row, Wa_bf16, NULL, in_dim, a_dim); - bf16_matvec_fused(b + (size_t)s * b_dim, x_row, Wb_bf16, NULL, in_dim, b_dim); - } - return; - } - - pair_matvec_task_t task = { - .a = a, - .b = b, - .x = x, - .Wa_bf16 = Wa_bf16, - .Wb_bf16 = Wb_bf16, - .seq_len = seq_len, - .in_dim = in_dim, - .a_dim = a_dim, - .b_dim = b_dim, - .total_dim = a_dim + b_dim, - }; - parallel_for(pair_matvec_worker, &task); -} - -typedef struct { - float *q; - float *k; - float *v; - const float *x; - const uint16_t *Wq_bf16; - const uint16_t *Wk_bf16; - const uint16_t *Wv_bf16; - int in_dim; - int q_dim; - int kv_dim; - int seq_len; - int total_dim; -} qkv_matvec_task_t; - -static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int start, int end) { - const float *x_row = t->x + (size_t)row * t->in_dim; - int q_end = t->q_dim; - int k_end = q_end + t->kv_dim; - int v_end = k_end + t->kv_dim; - - if (start < q_end) { - int s = start; - int e = end < q_end ? end : q_end; - if (s < e) { - bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row, - t->Wq_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); - } - } - - if (end > q_end && start < k_end) { - int s = start > q_end ? start - q_end : 0; - int e_abs = end < k_end ? end : k_end; - int e = e_abs - q_end; - if (s < e) { - bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row, - t->Wk_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); - } - } - - if (end > k_end && start < v_end) { - int s = start > k_end ? start - k_end : 0; - int e_abs = end < v_end ? end : v_end; - int e = e_abs - k_end; - if (s < e) { - bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row, - t->Wv_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); - } - } -} - -static void qkv_matvec_worker(int tid, int n_threads, void *arg) { - qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg; - int total_work = t->seq_len * t->total_dim; - int chunk = (total_work + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > total_work) - end = total_work; - if (start >= end) - return; - - while (start < end) { - int row = start / t->total_dim; - int pos = start - row * t->total_dim; - int row_end = (row + 1) * t->total_dim; - int stop = end < row_end ? end : row_end; - qkv_matvec_range(t, row, pos, stop - row * t->total_dim); - start = stop; - } -} - -void embed_linear_nobias_bf16_qkv(float *q, - float *k, - float *v, - const float *x, - const uint16_t *Wq_bf16, - const uint16_t *Wk_bf16, - const uint16_t *Wv_bf16, - int seq_len, - int in_dim, - int q_dim, - int kv_dim) { - if (seq_len <= 0) - return; - - if (embed_num_threads() <= 1) { - for (int s = 0; s < seq_len; s++) { - const float *x_row = x + (size_t)s * in_dim; - bf16_matvec_fused(q + (size_t)s * q_dim, x_row, Wq_bf16, NULL, in_dim, q_dim); - bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, Wk_bf16, NULL, in_dim, kv_dim); - bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, Wv_bf16, NULL, in_dim, kv_dim); - } - return; - } - - qkv_matvec_task_t task = { - .q = q, - .k = k, - .v = v, - .x = x, - .Wq_bf16 = Wq_bf16, - .Wk_bf16 = Wk_bf16, - .Wv_bf16 = Wv_bf16, - .in_dim = in_dim, - .q_dim = q_dim, - .kv_dim = kv_dim, - .seq_len = seq_len, - .total_dim = q_dim + 2 * kv_dim, - }; - parallel_for(qkv_matvec_worker, &task); -} - -void embed_linear_nobias_bf16( - float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) { - if (seq_len == 1) { - bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); - return; - } - /* Callers route longer sequences through an F32-widened weight matrix in - * caller-owned scratch (see embed.c); this per-row path stays correct for - * any length. */ - bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); -} - -/* ======================================================================== - * Normalization - * ======================================================================== */ - -/* Rows split across the pool only when the tensor is big enough to pay - * for the dispatch. */ -#define EMBED_RMS_NORM_PARALLEL_ELEMS 262144 - -static void embed_rms_norm_range( - float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) { - for (int s = start; s < end; s++) { - const float *x_row = x + (size_t)s * hidden; - float *out_row = out + (size_t)s * hidden; - -#if defined(__AVX512F__) && defined(__FMA__) - __m512 accv = _mm512_setzero_ps(); - int i = 0; - for (; i + 16 <= hidden; i += 16) { - __m512 v = _mm512_loadu_ps(x_row + i); - accv = _mm512_fmadd_ps(v, v, accv); - } - float sum_sq = _mm512_reduce_add_ps(accv); - for (; i < hidden; i++) - sum_sq += x_row[i] * x_row[i]; -#elif defined(__AVX2__) && defined(__FMA__) - __m256 accv = _mm256_setzero_ps(); - int i = 0; - for (; i + 8 <= hidden; i += 8) { - __m256 v = _mm256_loadu_ps(x_row + i); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float sum_sq = _mm_cvtss_f32(acc128); - for (; i < hidden; i++) - sum_sq += x_row[i] * x_row[i]; -#else - float sum_sq = 0.0f; - for (int i = 0; i < hidden; i++) { - sum_sq += x_row[i] * x_row[i]; - } -#endif - float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps); - -#if defined(__AVX512F__) - __m512 scale = _mm512_set1_ps(rms_inv); - int j = 0; - for (; j + 16 <= hidden; j += 16) { - __m512 vx = _mm512_loadu_ps(x_row + j); - __m512 vw = _mm512_loadu_ps(weight + j); - _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale)); - } - for (; j < hidden; j++) - out_row[j] = x_row[j] * rms_inv * weight[j]; -#elif defined(__AVX2__) - __m256 scale = _mm256_set1_ps(rms_inv); - int j = 0; - for (; j + 8 <= hidden; j += 8) { - __m256 vx = _mm256_loadu_ps(x_row + j); - __m256 vw = _mm256_loadu_ps(weight + j); - _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale)); - } - for (; j < hidden; j++) - out_row[j] = x_row[j] * rms_inv * weight[j]; -#else - for (int i = 0; i < hidden; i++) { - out_row[i] = x_row[i] * rms_inv * weight[i]; - } -#endif - } -} - -typedef struct { - float *out; - const float *x; - const float *weight; - int seq_len; - int hidden; - float eps; -} rms_norm_task_t; - -static void rms_norm_worker(int tid, int n_threads, void *arg) { - rms_norm_task_t *t = (rms_norm_task_t *)arg; - int chunk = (t->seq_len + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->seq_len) - end = t->seq_len; - if (start >= end) - return; - embed_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps); -} - -void embed_rms_norm( - float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) { - if (seq_len <= 0 || hidden <= 0) - return; - - long long elems = (long long)seq_len * hidden; - if (embed_num_threads() > 1 && elems >= EMBED_RMS_NORM_PARALLEL_ELEMS) { - rms_norm_task_t task = { - .out = out, - .x = x, - .weight = weight, - .seq_len = seq_len, - .hidden = hidden, - .eps = eps, - }; - parallel_for(rms_norm_worker, &task); - } else { - embed_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); - } -} - -void embed_rms_norm_per_head( - float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) { - /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ - int hidden = n_heads * head_dim; - for (int s = 0; s < seq_len; s++) { - for (int h = 0; h < n_heads; h++) { - float *vec = x + s * hidden + h * head_dim; - -#if defined(__AVX512F__) && defined(__FMA__) - __m512 accv = _mm512_setzero_ps(); - int d = 0; - for (; d + 16 <= head_dim; d += 16) { - __m512 v = _mm512_loadu_ps(vec + d); - accv = _mm512_fmadd_ps(v, v, accv); - } - float sum_sq = _mm512_reduce_add_ps(accv); - for (; d < head_dim; d++) - sum_sq += vec[d] * vec[d]; -#elif defined(__AVX2__) && defined(__FMA__) - __m256 accv = _mm256_setzero_ps(); - int d = 0; - for (; d + 8 <= head_dim; d += 8) { - __m256 v = _mm256_loadu_ps(vec + d); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = - _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float sum_sq = _mm_cvtss_f32(acc128); - for (; d < head_dim; d++) - sum_sq += vec[d] * vec[d]; -#else - float sum_sq = 0.0f; - for (int d = 0; d < head_dim; d++) { - sum_sq += vec[d] * vec[d]; - } -#endif - float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps); - -#if defined(__AVX512F__) - __m512 scale = _mm512_set1_ps(rms_inv); - int j = 0; - for (; j + 16 <= head_dim; j += 16) { - __m512 v = _mm512_loadu_ps(vec + j); - __m512 w = _mm512_loadu_ps(weight + j); - _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale)); - } - for (; j < head_dim; j++) - vec[j] = vec[j] * rms_inv * weight[j]; -#elif defined(__AVX2__) - __m256 scale = _mm256_set1_ps(rms_inv); - int j = 0; - for (; j + 8 <= head_dim; j += 8) { - __m256 v = _mm256_loadu_ps(vec + j); - __m256 w = _mm256_loadu_ps(weight + j); - _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale)); - } - for (; j < head_dim; j++) - vec[j] = vec[j] * rms_inv * weight[j]; -#else - for (int d = 0; d < head_dim; d++) { - vec[d] = vec[d] * rms_inv * weight[d]; - } -#endif - } - } -} - -/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each - * row's mean and variance are reduced before the row is rewritten. Scalar for - * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */ -void embed_layer_norm(float *out, - const float *x, - const float *gamma, - const float *beta, - int seq_len, - int hidden, - float eps) { - if (seq_len <= 0 || hidden <= 0) - return; - float inv_h = 1.0f / (float)hidden; - for (int s = 0; s < seq_len; s++) { - const float *row = x + (size_t)s * hidden; - float *dst = out + (size_t)s * hidden; - float mean = 0.0f; - for (int d = 0; d < hidden; d++) - mean += row[d]; - mean *= inv_h; - float var = 0.0f; - for (int d = 0; d < hidden; d++) { - float diff = row[d] - mean; - var += diff * diff; - } - var *= inv_h; - float inv_std = 1.0f / sqrtf(var + eps); - for (int d = 0; d < hidden; d++) - dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; - } -} - -/* ======================================================================== - * Activation Functions - * ======================================================================== */ - -void embed_silu_mul_inplace(float *gate, const float *up, int n) { -#if defined(__APPLE__) && defined(USE_BLAS) - enum { CHUNK = 4096 }; - float exp_neg[CHUNK]; - - for (int base = 0; base < n; base += CHUNK) { - int len = n - base; - if (len > CHUNK) - len = CHUNK; - - for (int i = 0; i < len; i++) - exp_neg[i] = -gate[base + i]; - vvexpf(exp_neg, exp_neg, &len); - - for (int i = 0; i < len; i++) { - float g = gate[base + i]; - gate[base + i] = (g / (1.0f + exp_neg[i])) * up[base + i]; - } - } -#else - for (int i = 0; i < n; i++) { - float g = gate[i]; - gate[i] = (g / (1.0f + expf(-g))) * up[i]; - } -#endif -} - -void embed_gelu_inplace(float *x, int n) { - /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ - for (int i = 0; i < n; i++) - x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); -} - -void embed_gelu_tanh_inplace(float *x, int n) { - /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */ - for (int i = 0; i < n; i++) { - float v = x[i]; - float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); - x[i] = 0.5f * v * (1.0f + tanhf(inner)); - } -} - -void embed_softmax(float *x, int rows, int cols) { - for (int r = 0; r < rows; r++) { - float *row = x + r * cols; - float max_val = row[0]; - for (int c = 1; c < cols; c++) { - if (row[c] > max_val) - max_val = row[c]; - } - float sum = 0.0f; - for (int c = 0; c < cols; c++) { - row[c] = expf(row[c] - max_val); - sum += row[c]; - } - float inv_sum = 1.0f / sum; - for (int c = 0; c < cols; c++) { - row[c] *= inv_sum; - } - } -} - -/* ======================================================================== - * Attention Operations - * ======================================================================== */ - -float embed_dot_f32(const float *a, const float *b, int n) { return embed_dot_f32_impl(a, b, n); } - -static inline float embed_dot_f32_fast(const float *a, const float *b, int n) { - return embed_dot_f32_impl(a, b, n); -} - -/* dst = dst * scale */ -static inline void embed_vec_scale_inplace(float *dst, float scale, int n) { - embed_vec_scale_inplace_impl(dst, scale, n); -} - -/* dst += alpha * src */ -static inline void embed_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - embed_vec_axpy_inplace_impl(dst, src, alpha, n); -} - -/* dst = dst * correction + src */ -static inline void embed_vec_scale_add(float *dst, const float *src, float correction, int n) { - embed_vec_scale_add_impl(dst, src, correction, n); -} - -#define EMBED_PACKED_ATTN_QUERY_TILE 32 -#define EMBED_PACKED_ATTN_BLAS_MIN_SEQ 128 - -static void embed_bidirectional_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - - for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) - o_row[d] = 0.0f; - - for (int j = start; j < end; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - - float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - embed_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - embed_vec_scale_inplace(o_row, inv_sum, head_dim); - } - } -} - -#ifdef USE_BLAS -static void embed_bidirectional_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - int rows = query_end - query_start; - int keys = end - start; - - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, - k, kv_hidden, 0.0f, scores, keys); - embed_softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); -} -#endif - -typedef struct { - float *out; - const float *Q; - const float *K; - const float *V; - const int *offsets; - int batch; - int n_heads, n_kv_heads; - int head_dim; - float scale; - float *scores; - int max_seq; -} packed_gqa_attn_task_t; - -static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; - int task_id = 0; -#ifdef USE_BLAS - float *scores = - t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; -#endif - - for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; -#ifdef USE_BLAS - if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { - embed_bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, - t->head_dim, t->scale, h, q0, q1, scores); - continue; - } -#endif - embed_bidirectional_gqa_attention_packed_online_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); - } - } - } -} - -size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { -#ifdef USE_BLAS - if (!offsets || batch <= 0) - return 0; - - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) - max_seq = len; - } - if (max_seq < EMBED_PACKED_ATTN_BLAS_MIN_SEQ) - return 0; - if ((size_t)max_seq > - SIZE_MAX / (sizeof(float) * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)embed_num_threads())) - return 0; - return (size_t)embed_num_threads() * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); -#else - (void)offsets; - (void)batch; - return 0; -#endif -} - -void embed_bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * len * n_heads; - if (len > max_seq) - max_seq = len; - } - - float *scores = NULL; -#ifdef USE_BLAS - size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) - scores = scratch; -#else - (void)scratch; - (void)scratch_bytes; -#endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; - if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { - parallel_for(packed_gqa_attn_worker, &task); - } else { - packed_gqa_attn_worker(0, 1, &task); - } -} - -void embed_bidirectional_gqa_attention_packed(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale) { - size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - embed_bidirectional_gqa_attention_packed_with_scratch( - out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes); - free(scores); -} - -static void embed_causal_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - - for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) - o_row[d] = 0.0f; - - for (int j = start; j <= i; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - embed_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) - embed_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); - } -} - -#ifdef USE_BLAS -static void embed_causal_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - int rows = query_end - query_start; - int keys = query_end - start; - - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, - k, kv_hidden, 0.0f, scores, keys); - for (int r = 0; r < rows; r++) { - int allowed = query_start + r - start + 1; - float *row = scores + (size_t)r * keys; - for (int j = allowed; j < keys; j++) - row[j] = -INFINITY; - } - embed_softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); -} -#endif - -static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; - int task_id = 0; -#ifdef USE_BLAS - float *scores = - t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; -#endif - - for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; -#ifdef USE_BLAS - if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { - embed_causal_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1, scores); - continue; - } -#endif - embed_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, - t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); - } - } - } -} - -void embed_causal_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * (len + 1) / 2 * n_heads; - if (len > max_seq) - max_seq = len; - } - - float *scores = NULL; -#ifdef USE_BLAS - size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) - scores = scratch; -#else - (void)scratch; - (void)scratch_bytes; -#endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; - if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) - parallel_for(packed_causal_gqa_attn_worker, &task); - else - packed_causal_gqa_attn_worker(0, 1, &task); -} - -void embed_causal_gqa_attention_packed(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale) { - size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - embed_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, - head_dim, scale, scores, scratch_bytes); - free(scores); -} - -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ - -void embed_compute_rope_neox( - float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { - int half = head_dim / 2; - - for (int s = 0; s < seq; s++) { - float pos = (float)positions[s]; - for (int d = 0; d < half; d++) { - float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); - float angle = pos * freq; - float c = cosf(angle); - float sn = sinf(angle); - /* Duplicate for full head_dim */ - cos_out[s * head_dim + d] = c; - cos_out[s * head_dim + half + d] = c; - sin_out[s * head_dim + d] = sn; - sin_out[s * head_dim + half + d] = sn; - } - } -} - -void embed_apply_rope_neox( - float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) { - /* - * NeoX split-half style: - * x1 = x[..., :half], x2 = x[..., half:] - * rotated = cat(-x2, x1) - * result = x * cos + rotated * sin - */ - int half = head_dim / 2; - int hidden = n_heads * head_dim; - - for (int s = 0; s < seq; s++) { - const float *c = cos_vals + s * head_dim; - const float *sn = sin_vals + s * head_dim; - - for (int h = 0; h < n_heads; h++) { - float *vec = x + s * hidden + h * head_dim; - -#if defined(__AVX512F__) && defined(__FMA__) - int d = 0; - for (; d + 16 <= half; d += 16) { - __m512 x1 = _mm512_loadu_ps(vec + d); - __m512 x2 = _mm512_loadu_ps(vec + half + d); - /* RoPE cache duplicates cos/sin across halves. */ - __m512 cc = _mm512_loadu_ps(c + d); - __m512 ss = _mm512_loadu_ps(sn + d); - __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss)); - __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss)); - _mm512_storeu_ps(vec + d, new1); - _mm512_storeu_ps(vec + half + d, new2); - } - for (; d < half; d++) { - float x1 = vec[d]; - float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#elif defined(__AVX2__) && defined(__FMA__) - int d = 0; - for (; d + 8 <= half; d += 8) { - __m256 x1 = _mm256_loadu_ps(vec + d); - __m256 x2 = _mm256_loadu_ps(vec + half + d); - __m256 cc = _mm256_loadu_ps(c + d); - __m256 ss = _mm256_loadu_ps(sn + d); - __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss)); - __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss)); - _mm256_storeu_ps(vec + d, new1); - _mm256_storeu_ps(vec + half + d, new2); - } - for (; d < half; d++) { - float x1 = vec[d]; - float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#elif defined(__ARM_NEON) || defined(__ARM_NEON__) - int d = 0; - for (; d + 4 <= half; d += 4) { - float32x4_t x1 = vld1q_f32(vec + d); - float32x4_t x2 = vld1q_f32(vec + half + d); - float32x4_t cc = vld1q_f32(c + d); - float32x4_t ss = vld1q_f32(sn + d); - float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss); - float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss); - vst1q_f32(vec + d, new1); - vst1q_f32(vec + half + d, new2); - } - for (; d < half; d++) { - float x1 = vec[d]; - float x2 = vec[half + d]; - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#else - for (int d = 0; d < half; d++) { - float x1 = vec[d]; /* first half */ - float x2 = vec[half + d]; /* second half */ - vec[d] = x1 * c[d] + (-x2) * sn[d]; - vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; - } -#endif - } - } -} diff --git a/src/kernels_act.c b/src/kernels_act.c new file mode 100644 index 0000000..abbade4 --- /dev/null +++ b/src/kernels_act.c @@ -0,0 +1,95 @@ +#include "kernels.h" +#include "kernels_impl.h" + +#include +#include +#include + +#if (defined(__AVX512F__) || defined(__AVX2__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include +#endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU + * variants. */ + +/* ======================================================================== + * Activation Functions + * ======================================================================== */ + +void embed_silu_mul_inplace(float *gate, const float *up, int n) { +#if defined(__APPLE__) && defined(USE_BLAS) + enum { CHUNK = 4096 }; + float exp_neg[CHUNK]; + + for (int base = 0; base < n; base += CHUNK) { + int len = n - base; + if (len > CHUNK) + len = CHUNK; + + for (int i = 0; i < len; i++) + exp_neg[i] = -gate[base + i]; + vvexpf(exp_neg, exp_neg, &len); + + for (int i = 0; i < len; i++) { + float g = gate[base + i]; + gate[base + i] = (g / (1.0f + exp_neg[i])) * up[base + i]; + } + } +#else + for (int i = 0; i < n; i++) { + float g = gate[i]; + gate[i] = (g / (1.0f + expf(-g))) * up[i]; + } +#endif +} + +void embed_gelu_inplace(float *x, int n) { + /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ + for (int i = 0; i < n; i++) + x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); +} + +void embed_gelu_tanh_inplace(float *x, int n) { + /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */ + for (int i = 0; i < n; i++) { + float v = x[i]; + float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); + x[i] = 0.5f * v * (1.0f + tanhf(inner)); + } +} + +void embed_softmax(float *x, int rows, int cols) { + for (int r = 0; r < rows; r++) { + float *row = x + r * cols; + float max_val = row[0]; + for (int c = 1; c < cols; c++) { + if (row[c] > max_val) + max_val = row[c]; + } + float sum = 0.0f; + for (int c = 0; c < cols; c++) { + row[c] = expf(row[c] - max_val); + sum += row[c]; + } + float inv_sum = 1.0f / sum; + for (int c = 0; c < cols; c++) { + row[c] *= inv_sum; + } + } +} diff --git a/src/kernels_attn.c b/src/kernels_attn.c new file mode 100644 index 0000000..66f5ca5 --- /dev/null +++ b/src/kernels_attn.c @@ -0,0 +1,469 @@ +#include "kernels.h" +#include "kernels_impl.h" + +#include +#include +#include + +#if (defined(__AVX512F__) || defined(__AVX2__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include +#endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE) + * GQA, each with an online-softmax and a BLAS-GEMM path. */ + +/* ======================================================================== + * Attention Operations + * ======================================================================== */ + +float embed_dot_f32(const float *a, const float *b, int n) { return embed_dot_f32_impl(a, b, n); } + +static inline float embed_dot_f32_fast(const float *a, const float *b, int n) { + return embed_dot_f32_impl(a, b, n); +} + +/* dst = dst * scale */ +static inline void embed_vec_scale_inplace(float *dst, float scale, int n) { + embed_vec_scale_inplace_impl(dst, scale, n); +} + +/* dst += alpha * src */ +static inline void embed_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { + embed_vec_axpy_inplace_impl(dst, src, alpha, n); +} + +/* dst = dst * correction + src */ +static inline void embed_vec_scale_add(float *dst, const float *src, float correction, int n) { + embed_vec_scale_add_impl(dst, src, correction, n); +} + +#define EMBED_PACKED_ATTN_QUERY_TILE 32 +#define EMBED_PACKED_ATTN_BLAS_MIN_SEQ 128 + +static void embed_bidirectional_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; + + for (int j = start; j < end; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + + float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + embed_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + embed_vec_scale_inplace(o_row, inv_sum, head_dim); + } + } +} + +#ifdef USE_BLAS +static void embed_bidirectional_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, + k, kv_hidden, 0.0f, scores, keys); + embed_softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); +} +#endif + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + const int *offsets; + int batch; + int n_heads, n_kv_heads; + int head_dim; + float scale; + float *scores; + int max_seq; +} packed_gqa_attn_task_t; + +static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = + t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; +#ifdef USE_BLAS + if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { + embed_bidirectional_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, + t->head_dim, t->scale, h, q0, q1, scores); + continue; + } +#endif + embed_bidirectional_gqa_attention_packed_online_rows( + t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1); + } + } + } +} + +size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { +#ifdef USE_BLAS + if (!offsets || batch <= 0) + return 0; + + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + if (len > max_seq) + max_seq = len; + } + if (max_seq < EMBED_PACKED_ATTN_BLAS_MIN_SEQ) + return 0; + if ((size_t)max_seq > + SIZE_MAX / (sizeof(float) * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)embed_num_threads())) + return 0; + return (size_t)embed_num_threads() * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); +#else + (void)offsets; + (void)batch; + return 0; +#endif +} + +void embed_bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * len * n_heads; + if (len > max_seq) + max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; + if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { + parallel_for(packed_gqa_attn_worker, &task); + } else { + packed_gqa_attn_worker(0, 1, &task); + } +} + +void embed_bidirectional_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale) { + size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + embed_bidirectional_gqa_attention_packed_with_scratch( + out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes); + free(scores); +} + +static void embed_causal_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; + + for (int j = start; j <= i; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + embed_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) + embed_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); + } +} + +#ifdef USE_BLAS +static void embed_causal_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = query_end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, + k, kv_hidden, 0.0f, scores, keys); + for (int r = 0; r < rows; r++) { + int allowed = query_start + r - start + 1; + float *row = scores + (size_t)r * keys; + for (int j = allowed; j < keys; j++) + row[j] = -INFINITY; + } + embed_softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); +} +#endif + +static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = + t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; +#ifdef USE_BLAS + if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { + embed_causal_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1, scores); + continue; + } +#endif + embed_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, + t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1); + } + } + } +} + +void embed_causal_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * (len + 1) / 2 * n_heads; + if (len > max_seq) + max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; + if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) + parallel_for(packed_causal_gqa_attn_worker, &task); + else + packed_causal_gqa_attn_worker(0, 1, &task); +} + +void embed_causal_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale) { + size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + embed_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, + head_dim, scale, scores, scratch_bytes); + free(scores); +} diff --git a/src/kernels_gemm.c b/src/kernels_gemm.c new file mode 100644 index 0000000..aa8dadf --- /dev/null +++ b/src/kernels_gemm.c @@ -0,0 +1,434 @@ +#include "kernels.h" +#include "kernels_impl.h" + +#include +#include +#include + +#if (defined(__AVX512F__) || defined(__AVX2__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include +#endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* Matrix and vector ops: GEMM, the dtype-dispatch linear kernels, and the + * BF16-fused matvec/QKV/pair workers, plus the elementwise residual add. */ + +/* ======================================================================== + * Basic Element-wise Operations + * ======================================================================== */ + +void embed_add_inplace(float *a, const float *b, int n) { + for (int i = 0; i < n; i++) + a[i] += b[i]; +} + +/* ======================================================================== + * Matrix Operations + * ======================================================================== */ + +void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { +#ifdef USE_BLAS + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); +#else + for (int m = 0; m < M; m++) { + for (int n = 0; n < N; n++) { + float sum = 0.0f; + for (int k = 0; k < K; k++) { + sum += A[m * K + k] * B[n * K + k]; + } + C[m * N + n] = sum; + } + } +#endif +} + +void embed_linear(float *y, + const float *x, + const float *W, + const float *b, + int seq_len, + int in_dim, + int out_dim) { +#ifdef USE_BLAS + if (seq_len == 1) { + cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, + 1); + } else { + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, + in_dim, W, in_dim, 0.0f, y, out_dim); + } + if (b != NULL) { + for (int s = 0; s < seq_len; s++) { + for (int o = 0; o < out_dim; o++) { + y[s * out_dim + o] += b[o]; + } + } + } +#else + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + s * in_dim; + float *y_row = y + s * out_dim; + for (int o = 0; o < out_dim; o++) { + const float *w_row = W + o * in_dim; + float sum = (b != NULL) ? b[o] : 0.0f; + for (int i = 0; i < in_dim; i++) { + sum += x_row[i] * w_row[i]; + } + y_row[o] = sum; + } + } +#endif +} + +void embed_linear_nobias( + float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { + embed_linear(y, x, W, NULL, seq_len, in_dim, out_dim); +} + +/* Convert bf16 buffer to f32 buffer */ +void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { +#if defined(__ARM_NEON) || defined(__ARM_NEON__) + size_t i = 0; + uint32_t *d = (uint32_t *)(void *)dst; + for (; i + 8 <= n; i += 8) { + uint16x8_t v = vld1q_u16(src + i); + uint32x4_t lo = vshlq_n_u32(vmovl_u16(vget_low_u16(v)), 16); + uint32x4_t hi = vshlq_n_u32(vmovl_u16(vget_high_u16(v)), 16); + vst1q_u32(d + i, lo); + vst1q_u32(d + i + 4, hi); + } + for (; i < n; i++) + d[i] = ((uint32_t)src[i]) << 16; +#else + uint32_t *d = (uint32_t *)(void *)dst; + for (size_t i = 0; i < n; i++) + d[i] = ((uint32_t)src[i]) << 16; +#endif +} + +/* + * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias + * Processes 2 output rows at a time to amortize x vector loads. + */ +static void bf16_matvec_fused( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { + embed_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); +} + +/* Threaded matvec: split output rows across threads */ +typedef struct { + float *y; + const float *x; + const uint16_t *W_bf16; + const float *bias; + int in_dim; + int out_dim; +} matvec_task_t; + +static void matvec_worker(int tid, int n_threads, void *arg) { + matvec_task_t *t = (matvec_task_t *)arg; + int chunk = (t->out_dim + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->out_dim) + end = t->out_dim; + if (start >= end) + return; + + bf16_matvec_fused(t->y + start, t->x, t->W_bf16 + (size_t)start * t->in_dim, + t->bias ? t->bias + start : NULL, t->in_dim, end - start); +} + +static void bf16_matvec_threaded( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { + if (embed_num_threads() <= 1) { + bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim); + return; + } + matvec_task_t task = {y, x, W_bf16, bias, in_dim, out_dim}; + parallel_for(matvec_worker, &task); +} + +typedef struct { + float *y; + const float *x; + const uint16_t *W_bf16; + const float *bias; + int seq_len; + int in_dim; + int out_dim; +} bf16_linear_rows_task_t; + +static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) { + bf16_linear_rows_task_t *t = (bf16_linear_rows_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->seq_len) + end = t->seq_len; + if (start >= end) + return; + + for (int s = start; s < end; s++) { + bf16_matvec_fused(t->y + (size_t)s * t->out_dim, t->x + (size_t)s * t->in_dim, t->W_bf16, + t->bias, t->in_dim, t->out_dim); + } +} + +static void bf16_linear_rows(float *y, + const float *x, + const uint16_t *W_bf16, + const float *bias, + int seq_len, + int in_dim, + int out_dim) { + bf16_linear_rows_task_t task = { + .y = y, + .x = x, + .W_bf16 = W_bf16, + .bias = bias, + .seq_len = seq_len, + .in_dim = in_dim, + .out_dim = out_dim, + }; + + if (embed_num_threads() > 1 && seq_len >= 2) + parallel_for(bf16_linear_rows_worker, &task); + else + bf16_linear_rows_worker(0, 1, &task); +} + +typedef struct { + float *a; + float *b; + const float *x; + const uint16_t *Wa_bf16; + const uint16_t *Wb_bf16; + int seq_len; + int in_dim; + int a_dim; + int b_dim; + int total_dim; +} pair_matvec_task_t; + +static void pair_matvec_range(const pair_matvec_task_t *t, int row, int start, int end) { + const float *x_row = t->x + (size_t)row * t->in_dim; + int a_end = t->a_dim; + int b_end = a_end + t->b_dim; + + if (start < a_end) { + int s = start; + int e = end < a_end ? end : a_end; + if (s < e) { + bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row, + t->Wa_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); + } + } + + if (end > a_end && start < b_end) { + int s = start > a_end ? start - a_end : 0; + int e_abs = end < b_end ? end : b_end; + int e = e_abs - a_end; + if (s < e) { + bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row, + t->Wb_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); + } + } +} + +static void pair_matvec_worker(int tid, int n_threads, void *arg) { + pair_matvec_task_t *t = (pair_matvec_task_t *)arg; + int total_work = t->seq_len * t->total_dim; + int chunk = (total_work + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > total_work) + end = total_work; + if (start >= end) + return; + + while (start < end) { + int row = start / t->total_dim; + int pos = start - row * t->total_dim; + int row_end = (row + 1) * t->total_dim; + int stop = end < row_end ? end : row_end; + pair_matvec_range(t, row, pos, stop - row * t->total_dim); + start = stop; + } +} + +void embed_linear_nobias_bf16_pair(float *a, + float *b, + const float *x, + const uint16_t *Wa_bf16, + const uint16_t *Wb_bf16, + int seq_len, + int in_dim, + int a_dim, + int b_dim) { + if (seq_len <= 0) + return; + + if (embed_num_threads() <= 1) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + (size_t)s * in_dim; + bf16_matvec_fused(a + (size_t)s * a_dim, x_row, Wa_bf16, NULL, in_dim, a_dim); + bf16_matvec_fused(b + (size_t)s * b_dim, x_row, Wb_bf16, NULL, in_dim, b_dim); + } + return; + } + + pair_matvec_task_t task = { + .a = a, + .b = b, + .x = x, + .Wa_bf16 = Wa_bf16, + .Wb_bf16 = Wb_bf16, + .seq_len = seq_len, + .in_dim = in_dim, + .a_dim = a_dim, + .b_dim = b_dim, + .total_dim = a_dim + b_dim, + }; + parallel_for(pair_matvec_worker, &task); +} + +typedef struct { + float *q; + float *k; + float *v; + const float *x; + const uint16_t *Wq_bf16; + const uint16_t *Wk_bf16; + const uint16_t *Wv_bf16; + int in_dim; + int q_dim; + int kv_dim; + int seq_len; + int total_dim; +} qkv_matvec_task_t; + +static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int start, int end) { + const float *x_row = t->x + (size_t)row * t->in_dim; + int q_end = t->q_dim; + int k_end = q_end + t->kv_dim; + int v_end = k_end + t->kv_dim; + + if (start < q_end) { + int s = start; + int e = end < q_end ? end : q_end; + if (s < e) { + bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row, + t->Wq_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); + } + } + + if (end > q_end && start < k_end) { + int s = start > q_end ? start - q_end : 0; + int e_abs = end < k_end ? end : k_end; + int e = e_abs - q_end; + if (s < e) { + bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row, + t->Wk_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); + } + } + + if (end > k_end && start < v_end) { + int s = start > k_end ? start - k_end : 0; + int e_abs = end < v_end ? end : v_end; + int e = e_abs - k_end; + if (s < e) { + bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row, + t->Wv_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s); + } + } +} + +static void qkv_matvec_worker(int tid, int n_threads, void *arg) { + qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg; + int total_work = t->seq_len * t->total_dim; + int chunk = (total_work + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > total_work) + end = total_work; + if (start >= end) + return; + + while (start < end) { + int row = start / t->total_dim; + int pos = start - row * t->total_dim; + int row_end = (row + 1) * t->total_dim; + int stop = end < row_end ? end : row_end; + qkv_matvec_range(t, row, pos, stop - row * t->total_dim); + start = stop; + } +} + +void embed_linear_nobias_bf16_qkv(float *q, + float *k, + float *v, + const float *x, + const uint16_t *Wq_bf16, + const uint16_t *Wk_bf16, + const uint16_t *Wv_bf16, + int seq_len, + int in_dim, + int q_dim, + int kv_dim) { + if (seq_len <= 0) + return; + + if (embed_num_threads() <= 1) { + for (int s = 0; s < seq_len; s++) { + const float *x_row = x + (size_t)s * in_dim; + bf16_matvec_fused(q + (size_t)s * q_dim, x_row, Wq_bf16, NULL, in_dim, q_dim); + bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, Wk_bf16, NULL, in_dim, kv_dim); + bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, Wv_bf16, NULL, in_dim, kv_dim); + } + return; + } + + qkv_matvec_task_t task = { + .q = q, + .k = k, + .v = v, + .x = x, + .Wq_bf16 = Wq_bf16, + .Wk_bf16 = Wk_bf16, + .Wv_bf16 = Wv_bf16, + .in_dim = in_dim, + .q_dim = q_dim, + .kv_dim = kv_dim, + .seq_len = seq_len, + .total_dim = q_dim + 2 * kv_dim, + }; + parallel_for(qkv_matvec_worker, &task); +} + +void embed_linear_nobias_bf16( + float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) { + if (seq_len == 1) { + bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); + return; + } + /* Callers route longer sequences through an F32-widened weight matrix in + * caller-owned scratch (see embed.c); this per-row path stays correct for + * any length. */ + bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); +} diff --git a/src/kernels_norm.c b/src/kernels_norm.c new file mode 100644 index 0000000..6bae71d --- /dev/null +++ b/src/kernels_norm.c @@ -0,0 +1,245 @@ +#include "kernels.h" +#include "kernels_impl.h" + +#include +#include +#include + +#if (defined(__AVX512F__) || defined(__AVX2__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include +#endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the + * per-head RMSNorm used for QK-norm. */ + +/* ======================================================================== + * Normalization + * ======================================================================== */ + +/* Rows split across the pool only when the tensor is big enough to pay + * for the dispatch. */ +#define EMBED_RMS_NORM_PARALLEL_ELEMS 262144 + +static void embed_rms_norm_range( + float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) { + for (int s = start; s < end; s++) { + const float *x_row = x + (size_t)s * hidden; + float *out_row = out + (size_t)s * hidden; + +#if defined(__AVX512F__) && defined(__FMA__) + __m512 accv = _mm512_setzero_ps(); + int i = 0; + for (; i + 16 <= hidden; i += 16) { + __m512 v = _mm512_loadu_ps(x_row + i); + accv = _mm512_fmadd_ps(v, v, accv); + } + float sum_sq = _mm512_reduce_add_ps(accv); + for (; i < hidden; i++) + sum_sq += x_row[i] * x_row[i]; +#elif defined(__AVX2__) && defined(__FMA__) + __m256 accv = _mm256_setzero_ps(); + int i = 0; + for (; i + 8 <= hidden; i += 8) { + __m256 v = _mm256_loadu_ps(x_row + i); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float sum_sq = _mm_cvtss_f32(acc128); + for (; i < hidden; i++) + sum_sq += x_row[i] * x_row[i]; +#else + float sum_sq = 0.0f; + for (int i = 0; i < hidden; i++) { + sum_sq += x_row[i] * x_row[i]; + } +#endif + float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps); + +#if defined(__AVX512F__) + __m512 scale = _mm512_set1_ps(rms_inv); + int j = 0; + for (; j + 16 <= hidden; j += 16) { + __m512 vx = _mm512_loadu_ps(x_row + j); + __m512 vw = _mm512_loadu_ps(weight + j); + _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale)); + } + for (; j < hidden; j++) + out_row[j] = x_row[j] * rms_inv * weight[j]; +#elif defined(__AVX2__) + __m256 scale = _mm256_set1_ps(rms_inv); + int j = 0; + for (; j + 8 <= hidden; j += 8) { + __m256 vx = _mm256_loadu_ps(x_row + j); + __m256 vw = _mm256_loadu_ps(weight + j); + _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale)); + } + for (; j < hidden; j++) + out_row[j] = x_row[j] * rms_inv * weight[j]; +#else + for (int i = 0; i < hidden; i++) { + out_row[i] = x_row[i] * rms_inv * weight[i]; + } +#endif + } +} + +typedef struct { + float *out; + const float *x; + const float *weight; + int seq_len; + int hidden; + float eps; +} rms_norm_task_t; + +static void rms_norm_worker(int tid, int n_threads, void *arg) { + rms_norm_task_t *t = (rms_norm_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->seq_len) + end = t->seq_len; + if (start >= end) + return; + embed_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps); +} + +void embed_rms_norm( + float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) { + if (seq_len <= 0 || hidden <= 0) + return; + + long long elems = (long long)seq_len * hidden; + if (embed_num_threads() > 1 && elems >= EMBED_RMS_NORM_PARALLEL_ELEMS) { + rms_norm_task_t task = { + .out = out, + .x = x, + .weight = weight, + .seq_len = seq_len, + .hidden = hidden, + .eps = eps, + }; + parallel_for(rms_norm_worker, &task); + } else { + embed_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); + } +} + +void embed_rms_norm_per_head( + float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) { + /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ + int hidden = n_heads * head_dim; + for (int s = 0; s < seq_len; s++) { + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + +#if defined(__AVX512F__) && defined(__FMA__) + __m512 accv = _mm512_setzero_ps(); + int d = 0; + for (; d + 16 <= head_dim; d += 16) { + __m512 v = _mm512_loadu_ps(vec + d); + accv = _mm512_fmadd_ps(v, v, accv); + } + float sum_sq = _mm512_reduce_add_ps(accv); + for (; d < head_dim; d++) + sum_sq += vec[d] * vec[d]; +#elif defined(__AVX2__) && defined(__FMA__) + __m256 accv = _mm256_setzero_ps(); + int d = 0; + for (; d + 8 <= head_dim; d += 8) { + __m256 v = _mm256_loadu_ps(vec + d); + accv = _mm256_fmadd_ps(v, v, accv); + } + __m128 acc128 = + _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); + acc128 = _mm_hadd_ps(acc128, acc128); + acc128 = _mm_hadd_ps(acc128, acc128); + float sum_sq = _mm_cvtss_f32(acc128); + for (; d < head_dim; d++) + sum_sq += vec[d] * vec[d]; +#else + float sum_sq = 0.0f; + for (int d = 0; d < head_dim; d++) { + sum_sq += vec[d] * vec[d]; + } +#endif + float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps); + +#if defined(__AVX512F__) + __m512 scale = _mm512_set1_ps(rms_inv); + int j = 0; + for (; j + 16 <= head_dim; j += 16) { + __m512 v = _mm512_loadu_ps(vec + j); + __m512 w = _mm512_loadu_ps(weight + j); + _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale)); + } + for (; j < head_dim; j++) + vec[j] = vec[j] * rms_inv * weight[j]; +#elif defined(__AVX2__) + __m256 scale = _mm256_set1_ps(rms_inv); + int j = 0; + for (; j + 8 <= head_dim; j += 8) { + __m256 v = _mm256_loadu_ps(vec + j); + __m256 w = _mm256_loadu_ps(weight + j); + _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale)); + } + for (; j < head_dim; j++) + vec[j] = vec[j] * rms_inv * weight[j]; +#else + for (int d = 0; d < head_dim; d++) { + vec[d] = vec[d] * rms_inv * weight[d]; + } +#endif + } + } +} + +/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each + * row's mean and variance are reduced before the row is rewritten. Scalar for + * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */ +void embed_layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps) { + if (seq_len <= 0 || hidden <= 0) + return; + float inv_h = 1.0f / (float)hidden; + for (int s = 0; s < seq_len; s++) { + const float *row = x + (size_t)s * hidden; + float *dst = out + (size_t)s * hidden; + float mean = 0.0f; + for (int d = 0; d < hidden; d++) + mean += row[d]; + mean *= inv_h; + float var = 0.0f; + for (int d = 0; d < hidden; d++) { + float diff = row[d] - mean; + var += diff * diff; + } + var *= inv_h; + float inv_std = 1.0f / sqrtf(var + eps); + for (int d = 0; d < hidden; d++) + dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; + } +} diff --git a/src/kernels_rope.c b/src/kernels_rope.c new file mode 100644 index 0000000..ef1709a --- /dev/null +++ b/src/kernels_rope.c @@ -0,0 +1,137 @@ +#include "kernels.h" +#include "kernels_impl.h" + +#include +#include +#include + +#if (defined(__AVX512F__) || defined(__AVX2__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include +#endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include +#else +#include +#endif +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ + +/* ======================================================================== + * Position Embeddings + * ======================================================================== */ + +void embed_compute_rope_neox( + float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { + int half = head_dim / 2; + + for (int s = 0; s < seq; s++) { + float pos = (float)positions[s]; + for (int d = 0; d < half; d++) { + float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = pos * freq; + float c = cosf(angle); + float sn = sinf(angle); + /* Duplicate for full head_dim */ + cos_out[s * head_dim + d] = c; + cos_out[s * head_dim + half + d] = c; + sin_out[s * head_dim + d] = sn; + sin_out[s * head_dim + half + d] = sn; + } + } +} + +void embed_apply_rope_neox( + float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) { + /* + * NeoX split-half style: + * x1 = x[..., :half], x2 = x[..., half:] + * rotated = cat(-x2, x1) + * result = x * cos + rotated * sin + */ + int half = head_dim / 2; + int hidden = n_heads * head_dim; + + for (int s = 0; s < seq; s++) { + const float *c = cos_vals + s * head_dim; + const float *sn = sin_vals + s * head_dim; + + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + +#if defined(__AVX512F__) && defined(__FMA__) + int d = 0; + for (; d + 16 <= half; d += 16) { + __m512 x1 = _mm512_loadu_ps(vec + d); + __m512 x2 = _mm512_loadu_ps(vec + half + d); + /* RoPE cache duplicates cos/sin across halves. */ + __m512 cc = _mm512_loadu_ps(c + d); + __m512 ss = _mm512_loadu_ps(sn + d); + __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss)); + __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss)); + _mm512_storeu_ps(vec + d, new1); + _mm512_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#elif defined(__AVX2__) && defined(__FMA__) + int d = 0; + for (; d + 8 <= half; d += 8) { + __m256 x1 = _mm256_loadu_ps(vec + d); + __m256 x2 = _mm256_loadu_ps(vec + half + d); + __m256 cc = _mm256_loadu_ps(c + d); + __m256 ss = _mm256_loadu_ps(sn + d); + __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss)); + __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss)); + _mm256_storeu_ps(vec + d, new1); + _mm256_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) + int d = 0; + for (; d + 4 <= half; d += 4) { + float32x4_t x1 = vld1q_f32(vec + d); + float32x4_t x2 = vld1q_f32(vec + half + d); + float32x4_t cc = vld1q_f32(c + d); + float32x4_t ss = vld1q_f32(sn + d); + float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss); + float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss); + vst1q_f32(vec + d, new1); + vst1q_f32(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#else + for (int d = 0; d < half; d++) { + float x1 = vec[d]; /* first half */ + float x2 = vec[half + d]; /* second half */ + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#endif + } + } +} -- cgit v1.3.1