diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 15:19:12 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 15:19:12 +0200 |
| commit | f463018cd700690336898d109226d489deca036d (patch) | |
| tree | 4e1d83f55a5e231af519aa4d647141db5de23af7 /qwen_kernels.c | |
| parent | d98ee1568c0bd88304a12e59d12104066e01e8d0 (diff) | |
| download | ffwd-f463018cd700690336898d109226d489deca036d.tar.gz ffwd-f463018cd700690336898d109226d489deca036d.zip | |
all: add support for Qwen3-Embedding + causal GQA attention
Diffstat (limited to 'qwen_kernels.c')
| -rw-r--r-- | qwen_kernels.c | 152 |
1 files changed, 152 insertions, 0 deletions
diff --git a/qwen_kernels.c b/qwen_kernels.c index 85871d4..dcd724c 100644 --- a/qwen_kernels.c +++ b/qwen_kernels.c @@ -1015,6 +1015,158 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, free(scores); } +static void qwen_causal_gqa_attention_packed_online_rows( + float *out, const float *Q, const float *K, const float *V, + int start, int n_heads, int n_kv_heads, int head_dim, + float scale, int h, int query_start, int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; + + for (int j = start; j <= i; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + qwen_vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) + qwen_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); + } +} + +#ifdef USE_BLAS +static void qwen_causal_gqa_attention_packed_blas_rows( + float *out, const float *Q, const float *K, const float *V, + int start, int n_heads, int n_kv_heads, int head_dim, + float scale, int h, int query_start, int query_end, float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = query_end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, + rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden, + 0.0f, scores, keys); + for (int r = 0; r < rows; r++) { + int allowed = query_start + r - start + 1; + float *row = scores + (size_t)r * keys; + for (int j = allowed; j < keys; j++) row[j] = -INFINITY; + } + qwen_softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, + rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, + 0.0f, o, q_hidden); +} +#endif + +static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = t->scores + ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq + : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; + if (q1 > end) q1 = end; + if (task_id++ % n_threads != tid) continue; +#ifdef USE_BLAS + if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { + qwen_causal_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, + scores); + continue; + } +#endif + qwen_causal_gqa_attention_packed_online_rows( + t->out, t->Q, t->K, t->V, start, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); + } + } + } +} + +void qwen_causal_gqa_attention_packed_with_scratch( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * (len + 1) / 2 * n_heads; + if (len > max_seq) max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes( + offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = { + .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, + .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, + .head_dim = head_dim, .scale = scale, .scores = scores, + .max_seq = max_seq + }; + if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) + parallel_for(packed_causal_gqa_attn_worker, &task); + else + packed_causal_gqa_attn_worker(0, 1, &task); +} + +void qwen_causal_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale) { + size_t scratch_bytes = + qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + qwen_causal_gqa_attention_packed_with_scratch( + out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, + scores, scratch_bytes); + free(scores); +} + /* ======================================================================== * Position Embeddings * ======================================================================== */ |
