diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-02 09:08:54 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 8d412f8a225f3e337dac271ea2491ac3c1c099af (patch) | |
| tree | 6e43dd291c46f5b440d109f1967d5423c5dc84c9 | |
| parent | 3f248b27a5482514f80deb687450dbd3a11d41b2 (diff) | |
| download | ffwd-8d412f8a225f3e337dac271ea2491ac3c1c099af.tar.gz ffwd-8d412f8a225f3e337dac271ea2491ac3c1c099af.zip | |
cpu: reuse tiled attention scratch buffers
| -rw-r--r-- | pplx_embed.c | 26 | ||||
| -rw-r--r-- | qwen_asr_kernels.c | 58 | ||||
| -rw-r--r-- | qwen_asr_kernels.h | 8 |
3 files changed, 77 insertions, 15 deletions
diff --git a/pplx_embed.c b/pplx_embed.c index 4aaeda1..1acd06f 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -53,6 +53,8 @@ struct pplx_workspace { float *proj_out; /* [seq, hidden] */ float *ffn_gate; /* [seq, intermediate] */ float *ffn_up; /* [seq, intermediate] */ + float *attn_scores; /* reusable BLAS attention score tiles */ + size_t attn_scores_bytes; int *offsets; /* [batch + 1] packed sequence offsets */ int offsets_cap; @@ -497,6 +499,19 @@ static int ensure_buffers(pplx_workspace_t *ws, const pplx_config_t *c, int seq) return 0; } +static void ensure_attention_scores(pplx_workspace_t *ws, const int *offsets, + int batch) +{ + size_t bytes = + qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (bytes <= ws->attn_scores_bytes) return; + + void *p = realloc(ws->attn_scores, bytes); + if (!p) return; + ws->attn_scores = (float *)p; + ws->attn_scores_bytes = bytes; +} + /* ======================================================================== * RoPE cache * ======================================================================== */ @@ -722,6 +737,7 @@ void pplx_workspace_free(pplx_workspace_t *ws) free(ws->q); free(ws->k); free(ws->v); free(ws->attn_out); free(ws->proj_out); free(ws->ffn_gate); free(ws->ffn_up); + free(ws->attn_scores); free(ws->offsets); free(ws->rope_cos); free(ws->rope_sin); free(ws); @@ -786,6 +802,8 @@ size_t pplx_workspace_nbytes(const pplx_workspace_t *ws) return SIZE_MAX; if (add_size(total, offset_bytes, &total) != 0) return SIZE_MAX; + if (add_size(total, ws->attn_scores_bytes, &total) != 0) + return SIZE_MAX; return total; } @@ -853,6 +871,7 @@ static int forward_packed_slice_inplace(const pplx_model_t *model, if (ensure_buffers(ws, cfg, total_seq) != 0) return -1; if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1; + ensure_attention_scores(ws, offsets, batch); const float *rope_cos = ws->rope_cos; const float *rope_sin = ws->rope_sin; @@ -912,9 +931,10 @@ static int forward_packed_slice_inplace(const pplx_model_t *model, rope_cos, rope_sin, len, n_kv_heads, head_dim); } - qwen_bidirectional_gqa_attention_packed(attn_out, q_buf, k_buf, v_buf, - offsets, batch, n_heads, - n_kv_heads, head_dim, scale); + qwen_bidirectional_gqa_attention_packed_with_scratch( + attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, + n_kv_heads, head_dim, scale, ws->attn_scores, + ws->attn_scores_bytes); linear_nobias_weight(proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); qwen_add_inplace(x, proj_out, total_seq * hidden); diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index 369e050..efa606c 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -1451,11 +1451,33 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { } } -void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, - const float *K, const float *V, - const int *offsets, int batch, - int n_heads, int n_kv_heads, - int head_dim, float scale) { +size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( + const int *offsets, int batch) { +#ifdef USE_BLAS + if (!offsets || batch <= 0) return 0; + + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + if (len > max_seq) max_seq = len; + } + if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) return 0; + if ((size_t)max_seq > SIZE_MAX / (sizeof(float) * + QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) + return 0; + return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * + (size_t)max_seq * sizeof(float); +#else + (void)offsets; + (void)batch; + return 0; +#endif +} + +void qwen_bidirectional_gqa_attention_packed_with_scratch( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes) { long long qk_work = 0; int max_seq = 0; for (int b = 0; b < batch; b++) { @@ -1466,13 +1488,12 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, float *scores = NULL; #ifdef USE_BLAS - /* Keep score scratch linear in sequence length; OOM falls back online. */ - if (max_seq >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ && - (size_t)max_seq <= SIZE_MAX / (sizeof(float) * - QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) { - size_t n = (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * max_seq; - scores = malloc(n * sizeof(*scores)); - } + size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes( + offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; #endif packed_gqa_attn_task_t task = { .out = out, .Q = Q, .K = K, .V = V, .offsets = offsets, @@ -1485,6 +1506,19 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, } else { packed_gqa_attn_worker(0, 1, &task); } +} + +void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, + const float *K, const float *V, + const int *offsets, int batch, + int n_heads, int n_kv_heads, + int head_dim, float scale) { + size_t scratch_bytes = + qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + qwen_bidirectional_gqa_attention_packed_with_scratch( + out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, + scores, scratch_bytes); free(scores); } diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h index 92082f1..c32ec3a 100644 --- a/qwen_asr_kernels.h +++ b/qwen_asr_kernels.h @@ -150,6 +150,14 @@ void qwen_bidirectional_gqa_attention_packed(float *out, const float *Q, int n_heads, int n_kv_heads, int head_dim, float scale); +/* Optional reusable scratch for the BLAS-tiled packed attention path. */ +size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes( + const int *offsets, int batch); +void qwen_bidirectional_gqa_attention_packed_with_scratch( + float *out, const float *Q, const float *K, const float *V, + const int *offsets, int batch, int n_heads, int n_kv_heads, + int head_dim, float scale, float *scratch, size_t scratch_bytes); + /* ======================================================================== * Position Embeddings * ======================================================================== */ |
