aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/cuda.cu8
1 files changed, 7 insertions, 1 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index a80fed5..a06d5d0 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -3866,7 +3866,13 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se
ctx->rope_cos = NULL;
ctx->rope_sin = NULL;
long long score_elems = (long long)c->n_heads * max_seq * max_seq;
- int need_materialized_scores = !is_bert || !bp.flash;
+ // The Qwen BF16 HD=128 tc3 path is streaming flash attention: it writes the
+ // output directly and never materializes the N*N scores. Skip the O(L^2)
+ // attn_scores/attn_probs allocation for it (as BERT flash already does),
+ // which otherwise OOMs at long context (e.g. 32k) and caps ffwd's usable
+ // sequence length below the model's 32k. mat/f16 fallbacks still allocate.
+ int qwen_stream_flash = use_streaming_attention(c, ctx->weights_bf16, c->head_dim);
+ int need_materialized_scores = (!is_bert || !bp.flash) && !qwen_stream_flash;
if (need_materialized_scores && score_elems > ctx->attn_scores_elems) {
cudaFree(ctx->attn_scores);
ctx->attn_scores = NULL;