diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 16:17:11 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 16:17:11 +0200 |
| commit | 88aed70366f99520a7ed4e86a101a41d33aa4542 (patch) | |
| tree | 37a699f3091718d48bf0428c3d4651eacfcb5a6f | |
| parent | d97c79be3f738b3c50af1df3bfdf9fdf662658f6 (diff) | |
| download | ffwd-88aed70366f99520a7ed4e86a101a41d33aa4542.tar.gz ffwd-88aed70366f99520a7ed4e86a101a41d33aa4542.zip | |
cuda: broaden bert flash kv pipeline to all flash batches
| -rw-r--r-- | libffwd/cuda.cu | 18 |
1 files changed, 10 insertions, 8 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index da60f73..ae601bb 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -4060,10 +4060,10 @@ static int bert_qkv16_mode(void) { } // BERT flash K/V-staging pipeline. Diagnostic switch over the single release -// path: 0/default rides the residual16 gate (the shipped behavior), 1/on forces -// the cp.async pipeline on, 0/off forces the legacy single-buffer staging. Used -// to A/B the pipeline at one token count without rebuilding while the -// non-residual16 bf16 correctness question is open. +// path: env unset/default runs the cp.async pipeline whenever BERT flash is +// active (the shipped behavior), 1/on forces it on, 0/off forces the legacy +// single-buffer staging. Used to A/B the pipeline at one token count without +// rebuilding. static int bert_flash_pipeline_mode(void) { static int init = 0, mode = 0; if (!init) { @@ -4187,11 +4187,13 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp return -1; } // Flash writes reduced-precision attention output into act16; the - // materialized path writes F32 attn_out. The K/V-staging pipeline rides - // the residual16 gate by default; the diagnostic env knob forces it - // on/off to validate that single release path. + // materialized path writes F32 attn_out. The K/V-staging pipeline runs + // whenever flash is active: it is numerically identical to single-buffer + // staging (it only overlaps the cp.async K/V loads) and is faster across + // the reduced-precision batch cells. The diagnostic env knob forces it + // on/off for a same-host A/B. int pipe_mode = bert_flash_pipeline_mode(); - int bert_pipeline = pipe_mode > 0 ? 1 : (pipe_mode < 0 ? 0 : use_resid16); + int bert_pipeline = pipe_mode > 0 ? 1 : (pipe_mode < 0 ? 0 : flash); if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, qkv_dim, qkv_src, qkv_dtype, scale, flash ? act16 : NULL, flash ? act_dtype : CUDA_DTYPE_F32, bert_pipeline) != 0) |
