diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-21 02:45:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:18:00 +0200 |
| commit | a3010d08c8806ca86a0e0b2793d8486f26f988e6 (patch) | |
| tree | cb79581a1a0b7b88470505ad920a866cfcf46583 | |
| parent | 7897110119db9aca574472ed067d547ecba63b7c (diff) | |
| download | ffwd-a3010d08c8806ca86a0e0b2793d8486f26f988e6.tar.gz ffwd-a3010d08c8806ca86a0e0b2793d8486f26f988e6.zip | |
kernels: avoid tiny packed attention dispatch
| -rw-r--r-- | qwen_asr_kernels.c | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index bc93f8a..70ead7b 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -1215,7 +1215,13 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, const int *offsets, int batch, int n_heads, int n_kv_heads, int head_dim, float scale) { - if (tp.n_threads > 1 && n_heads >= 2 && batch > 0) { + long long qk_work = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * len * n_heads; + } + + if (tp.n_threads > 1 && n_heads >= 2 && qk_work >= 4096) { packed_gqa_attn_task_t task = { .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, .batch = batch, .n_heads = n_heads, .n_kv_heads = n_kv_heads, |
