aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-25 20:44:28 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-25 20:44:28 +0200
commit6a9aa3302362e7d129b804cd621d39a1c1e7c43d (patch)
treef87be5f15b0ec63b4c24dcf70f7b1eab98de6863
parentc09ea7df78546e6c4ff73fe4e8e66cd5a964ea97 (diff)
downloadffwd-6a9aa3302362e7d129b804cd621d39a1c1e7c43d.tar.gz
ffwd-6a9aa3302362e7d129b804cd621d39a1c1e7c43d.zip
cuda: skip O(L^2) score buffer for qwen flash path (enables 32k)
-rw-r--r--libffwd/cuda.cu8
1 files changed, 7 insertions, 1 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index a80fed5..a06d5d0 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -3866,7 +3866,13 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
ctx->rope_cos = NULL;
ctx->rope_sin = NULL;
long long score_elems = (long long)c->n_heads * max_seq * max_seq;
- int need_materialized_scores = !is_bert || !bp.flash;
+ // The Qwen BF16 HD=128 tc3 path is streaming flash attention: it writes the
+ // output directly and never materializes the N*N scores. Skip the O(L^2)
+ // attn_scores/attn_probs allocation for it (as BERT flash already does),
+ // which otherwise OOMs at long context (e.g. 32k) and caps ffwd's usable
+ // sequence length below the model's 32k. mat/f16 fallbacks still allocate.
+ int qwen_stream_flash = use_streaming_attention(c, ctx->weights_bf16, c->head_dim);
+ int need_materialized_scores = (!is_bert || !bp.flash) && !qwen_stream_flash;
if (need_materialized_scores && score_elems > ctx->attn_scores_elems) {
cudaFree(ctx->attn_scores);
ctx->attn_scores = NULL;