diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 20:18:05 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 20:18:05 +0200 |
| commit | 4d79e6337c05d0a768de80eeb164f1de3b1624d3 (patch) | |
| tree | b5b8b140e2eceaf7c69b28d31f7d05464d9dae12 /libffwd | |
| parent | 6a10d3577232f1fd646a20f22a4a1ab4ec29f28a (diff) | |
| download | ffwd-4d79e6337c05d0a768de80eeb164f1de3b1624d3.tar.gz ffwd-4d79e6337c05d0a768de80eeb164f1de3b1624d3.zip | |
cuda: default cuDNN attention to Qwen only, BERT opt-in
Diffstat (limited to 'libffwd')
| -rw-r--r-- | libffwd/cuda.cu | 31 |
1 files changed, 19 insertions, 12 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index 1884b73..ec3f15a 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -2863,15 +2863,20 @@ static int use_streaming_attention(const ffwd_config_t *c, int out_bf16, int hea return (mode == 0 || mode == 8) && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128; } -// cuDNN fused attention (cuda_sdpa.h). Unset/1 dispatches the 16-bit flash -// shapes to cuDNN whenever the runtime supports them; FFWD_CUDA_SDPA=0 forces -// the built-in kernels (the A/B instrument and the rescue switch). +// cuDNN fused attention (cuda_sdpa.h). Unset routes the Qwen bf16 hd128 shape +// to cuDNN (measured faster at every length, growing with context) and keeps +// BERT on the built-in kernels (whose direct packed-QKV staging still beats +// the extra Q-pack/expand the cuDNN path needs at BERT lengths). 1/on extends +// cuDNN to the BERT flash shapes; 0/off forces the built-in kernels everywhere +// (the A/B instrument and the rescue switch). static int sdpa_mode(void) { static int init = 0, mode = 0; if (!init) { const char *e = getenv("FFWD_CUDA_SDPA"); if (e && (!strcmp(e, "0") || !strcmp(e, "off"))) mode = -1; + else if (e && (!strcmp(e, "1") || !strcmp(e, "on") || !strcmp(e, "force"))) + mode = 1; init = 1; } return mode; @@ -3412,8 +3417,8 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx, // hand-written kernels, and any cuDNN refusal falls back to them). It // consumes the staged contiguous K/V, so the BERT direct-QKV read shortcut // is bypassed while it is active. - int use_sdpa = sdpa_mode() >= 0 && ctx->sdpa && ctx->q16 && ctx->sdpa_batch_bucket > 0 && - (use_tc_kexp_bf16 || use_bert_flash); + int use_sdpa = ctx->sdpa && ctx->q16 && ctx->sdpa_batch_bucket > 0 && + ((use_tc_kexp_bf16 && sdpa_mode() >= 0) || (use_bert_flash && sdpa_mode() > 0)); if (use_sdpa) use_bert_direct_kv = 0; @@ -3869,11 +3874,11 @@ static void bert_cuda_plan(ffwd_cuda_ctx_t *ctx, int total, bert_cuda_plan_t *p) p->qkv16 = p->flash && p->cublaslt_avail && qkv16_mode >= 0 && (qkv16_mode > 0 || total >= BERT_QKV16_MIN_TOKENS); // The cuDNN fused-attention path reads staged contiguous K/V, so while it - // is a candidate the direct packed-QKV read is not planned and the K/V - // expansion buffers stay allocated (the dispatch relies on them, including - // for its fallback). + // is a candidate (BERT: forced on only) the direct packed-QKV read is not + // planned and the K/V expansion buffers stay allocated (the dispatch + // relies on them, including for its fallback). p->direct_kv = - p->flash && p->qkv16 && bert_direct_kv_mode() >= 0 && !(ctx->sdpa && sdpa_mode() >= 0); + p->flash && p->qkv16 && bert_direct_kv_mode() >= 0 && !(ctx->sdpa && sdpa_mode() > 0); } static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_seq) { @@ -3902,9 +3907,11 @@ static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_se int act_cols = 2 * c->intermediate_size; int ffn_gate_up_cols = 2 * c->intermediate_size; // Packed 16-bit Q for the cuDNN fused-attention path: any shape whose - // 16-bit flash kernel could dispatch to cuDNN needs it staged. - int need_q16 = ctx->sdpa && sdpa_mode() >= 0 && - (is_bert ? bp.flash : (ctx->weights_bf16 && c->head_dim == 128)); + // 16-bit flash kernel could dispatch to cuDNN needs it staged. BERT only + // dispatches when cuDNN is forced on (see sdpa_mode). + int need_q16 = + ctx->sdpa && (is_bert ? (bp.flash && sdpa_mode() > 0) + : (ctx->weights_bf16 && c->head_dim == 128 && sdpa_mode() >= 0)); if (is_bert) { need_x_norm = 0; |
