aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-21 02:45:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:00 +0200
commita3010d08c8806ca86a0e0b2793d8486f26f988e6 (patch)
treecb79581a1a0b7b88470505ad920a866cfcf46583
parent7897110119db9aca574472ed067d547ecba63b7c (diff)
downloadffwd-a3010d08c8806ca86a0e0b2793d8486f26f988e6.tar.gz
ffwd-a3010d08c8806ca86a0e0b2793d8486f26f988e6.zip
kernels: avoid tiny packed attention dispatch
-rw-r--r--qwen_asr_kernels.c8
1 files changed, 7 insertions, 1 deletions
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index bc93f8a..70ead7b 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -1215,7 +1215,13 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
const int *offsets, int batch,
int n_heads, int n_kv_heads,
int head_dim, float scale) {
- if (tp.n_threads > 1 && n_heads >= 2 && batch > 0) {
+ long long qk_work = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ qk_work += (long long)len * len * n_heads;
+ }
+
+ if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) {
packed_gqa_attn_task_t task = {
.out = out, .Q = Q, .K = K, .V = V, .offsets = offsets,
.batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads,