aboutsummaryrefslogtreecommitdiff
path: root/qwen_asr_kernels.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-02 09:08:54 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit8d412f8a225f3e337dac271ea2491ac3c1c099af (patch)
tree6e43dd291c46f5b440d109f1967d5423c5dc84c9 /qwen_asr_kernels.c
parent3f248b27a5482514f80deb687450dbd3a11d41b2 (diff)
downloadffwd-8d412f8a225f3e337dac271ea2491ac3c1c099af.tar.gz
ffwd-8d412f8a225f3e337dac271ea2491ac3c1c099af.zip
cpu: reuse tiled attention scratch buffers
Diffstat (limited to 'qwen_asr_kernels.c')
-rw-r--r--qwen_asr_kernels.c58
1 files changed, 46 insertions, 12 deletions
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index 369e050..efa606c 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -1451,11 +1451,33 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
}
}
-void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
- const float *K, const float *V,
- const int *offsets, int batch,
- int n_heads, int n_kv_heads,
- int head_dim, float scale) {
+size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(
+ const int *offsets, int batch) {
+#ifdef USE_BLAS
+ if (!offsets || batch <= 0) return 0;
+
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ if (len > max_seq) max_seq = len;
+ }
+ if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) return 0;
+ if ((size_t)max_seq > SIZE_MAX / (sizeof(float) *
+ QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads))
+ return 0;
+ return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE *
+ (size_t)max_seq * sizeof(float);
+#else
+ (void)offsets;
+ (void)batch;
+ return 0;
+#endif
+}
+
+void qwen_bidirectional_gqa_attention_packed_with_scratch(
+ float *out, const float *Q, const float *K, const float *V,
+ const int *offsets, int batch, int n_heads, int n_kv_heads,
+ int head_dim, float scale, float *scratch, size_t scratch_bytes) {
long long qk_work = 0;
int max_seq = 0;
for (int b = 0; b < batch; b++) {
@@ -1466,13 +1488,12 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
float *scores = NULL;
#ifdef USE_BLAS
- /* Keep score scratch linear in sequence length; OOM falls back online. */
- if (max_seq >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ &&
- (size_t)max_seq <= SIZE_MAX / (sizeof(float) *
- QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) {
- size_t n = (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * max_seq;
- scores = malloc(n * sizeof(*scores));
- }
+ size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(
+ offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required) scores = scratch;
+#else
+ (void)scratch;
+ (void)scratch_bytes;
#endif
packed_gqa_attn_task_t task = {
.out = out, .Q = Q, .K = K, .V = V, .offsets = offsets,
@@ -1485,6 +1506,19 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
} else {
packed_gqa_attn_worker(0, 1, &task);
}
+}
+
+void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q,
+ const float *K, const float *V,
+ const int *offsets, int batch,
+ int n_heads, int n_kv_heads,
+ int head_dim, float scale) {
+ size_t scratch_bytes =
+ qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
+ qwen_bidirectional_gqa_attention_packed_with_scratch(
+ out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale,
+ scores, scratch_bytes);
free(scores);
}