diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-25 20:44:28 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-25 20:44:28 +0200 |
| commit | 6a9aa3302362e7d129b804cd621d39a1c1e7c43d (patch) | |
| tree | f87be5f15b0ec63b4c24dcf70f7b1eab98de6863 /libffwd | |
| parent | c09ea7df78546e6c4ff73fe4e8e66cd5a964ea97 (diff) | |
| download | ffwd-6a9aa3302362e7d129b804cd621d39a1c1e7c43d.tar.gz ffwd-6a9aa3302362e7d129b804cd621d39a1c1e7c43d.zip | |
cuda: skip O(L^2) score buffer for qwen flash path (enables 32k)
Diffstat (limited to 'libffwd')
| -rw-r--r-- | libffwd/cuda.cu | 8 |
1 files changed, 7 insertions, 1 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index a80fed5..a06d5d0 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -3866,7 +3866,13 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se ctx->rope_cos = NULL; ctx->rope_sin = NULL; long long score_elems = (long long)c->n_heads * max_seq * max_seq; - int need_materialized_scores = !is_bert || !bp.flash; + // The Qwen BF16 HD=128 tc3 path is streaming flash attention: it writes the + // output directly and never materializes the N*N scores. Skip the O(L^2) + // attn_scores/attn_probs allocation for it (as BERT flash already does), + // which otherwise OOMs at long context (e.g. 32k) and caps ffwd's usable + // sequence length below the model's 32k. mat/f16 fallbacks still allocate. + int qwen_stream_flash = use_streaming_attention(c, ctx->weights_bf16, c->head_dim); + int need_materialized_scores = (!is_bert || !bp.flash) && !qwen_stream_flash; if (need_materialized_scores && score_elems > ctx->attn_scores_elems) { cudaFree(ctx->attn_scores); ctx->attn_scores = NULL; |
