From 6a9aa3302362e7d129b804cd621d39a1c1e7c43d Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Thu, 25 Jun 2026 20:44:28 +0200 Subject: cuda: skip O(L^2) score buffer for qwen flash path (enables 32k) --- libffwd/cuda.cu | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index a80fed5..a06d5d0 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -3866,7 +3866,13 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se ctx->rope_cos = NULL; ctx->rope_sin = NULL; long long score_elems = (long long)c->n_heads * max_seq * max_seq; - int need_materialized_scores = !is_bert || !bp.flash; + // The Qwen BF16 HD=128 tc3 path is streaming flash attention: it writes the + // output directly and never materializes the N*N scores. Skip the O(L^2) + // attn_scores/attn_probs allocation for it (as BERT flash already does), + // which otherwise OOMs at long context (e.g. 32k) and caps ffwd's usable + // sequence length below the model's 32k. mat/f16 fallbacks still allocate. + int qwen_stream_flash = use_streaming_attention(c, ctx->weights_bf16, c->head_dim); + int need_materialized_scores = (!is_bert || !bp.flash) && !qwen_stream_flash; if (need_materialized_scores && score_elems > ctx->attn_scores_elems) { cudaFree(ctx->attn_scores); ctx->attn_scores = NULL; -- cgit v1.3.1