#include "kernels.h" #include "impl.h" #include "threadpool.h" #include #ifdef USE_BLAS # ifdef __APPLE__ # include # else # include # endif #endif #define ATTN_QUERY_TILE 32 #define ATTN_BLAS_MIN_SEQ 128 static void attention_online_rows(float *out, const float *Q, const float *K, const float *V, int seq, int n_heads, int head_dim, float scale, int h, int query_start, int query_end) { int hidden = n_heads * head_dim; for (int i = query_start; i < query_end; i++) { const float *q_row = Q + (size_t)i * hidden + h * head_dim; float *o_row = out + (size_t)i * hidden + h * head_dim; float max_score = -1e30f; float sum_exp = 0.0f; for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; for (int j = 0; j < seq; j++) { const float *k_row = K + (size_t)j * hidden + h * head_dim; const float *v_row = V + (size_t)j * hidden + h * head_dim; float score = dot_f32_impl(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); sum_exp = sum_exp * correction + 1.0f; vec_scale_add_impl(o_row, v_row, correction, head_dim); max_score = score; } else { float wt = expf(score - max_score); sum_exp += wt; vec_axpy_inplace_impl(o_row, v_row, wt, head_dim); } } if (sum_exp > 0.0f) { float inv_sum = 1.0f / sum_exp; vec_scale_inplace_impl(o_row, inv_sum, head_dim); } } } #ifdef USE_BLAS static void attention_blas_rows(float *out, const float *Q, const float *K, const float *V, int seq, int n_heads, int head_dim, float scale, int h, int query_start, int query_end, float *scores) { int hidden = n_heads * head_dim; int rows = query_end - query_start; const float *q = Q + (size_t)query_start * hidden + h * head_dim; const float *k = K + h * head_dim; const float *v = V + h * head_dim; float *o = out + (size_t)query_start * hidden + h * head_dim; cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, seq, head_dim, scale, q, hidden, k, hidden, 0.0f, scores, seq); softmax(scores, rows, seq); cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, seq, 1.0f, scores, seq, v, hidden, 0.0f, o, hidden); } #endif typedef struct { float *out; const float *Q; const float *K; const float *V; int seq; int n_heads; int head_dim; float scale; float *scores; } attention_task_t; static void attention_worker(int tid, int n_threads, void *arg) { attention_task_t *t = (attention_task_t *)arg; int task_id = 0; #ifdef USE_BLAS float *scores = t->scores ? t->scores + (size_t)tid * ATTN_QUERY_TILE * t->seq : NULL; #endif for (int h = 0; h < t->n_heads; h++) { for (int q0 = 0; q0 < t->seq; q0 += ATTN_QUERY_TILE) { int q1 = q0 + ATTN_QUERY_TILE; if (q1 > t->seq) q1 = t->seq; if (task_id++ % n_threads != tid) continue; #ifdef USE_BLAS if (scores && t->seq >= ATTN_BLAS_MIN_SEQ) { attention_blas_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads, t->head_dim, t->scale, h, q0, q1, scores); continue; } #endif attention_online_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads, t->head_dim, t->scale, h, q0, q1); } } } size_t attention_scratch_bytes(int seq) { #ifdef USE_BLAS if (seq < ATTN_BLAS_MIN_SEQ) return 0; if ((size_t)seq > SIZE_MAX / (sizeof(float) * ATTN_QUERY_TILE * (size_t)tp_num_threads())) return 0; return (size_t)tp_num_threads() * ATTN_QUERY_TILE * (size_t)seq * sizeof(float); #else (void)seq; return 0; #endif } void attention_with_scratch(float *out, const float *Q, const float *K, const float *V, int seq, int n_heads, int head_dim, float scale, float *scratch, size_t scratch_bytes) { long long qk_work = (long long)seq * seq * n_heads; float *scores = NULL; #ifdef USE_BLAS size_t required = attention_scratch_bytes(seq); if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; #else (void)scratch; (void)scratch_bytes; #endif attention_task_t task = {.out = out, .Q = Q, .K = K, .V = V, .seq = seq, .n_heads = n_heads, .head_dim = head_dim, .scale = scale, .scores = scores}; if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { parallel_for(attention_worker, &task); } else { attention_worker(0, 1, &task); } }