diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-22 14:15:41 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-22 14:15:41 +0200 |
| commit | 3e8108cee394f44381193ee847515f81546f3383 (patch) | |
| tree | 0a47e57d30d361d0438a2ad11a7b3de832b94485 /libffwd | |
| parent | 3259a306f2898bd0525aabccf0d15a66608636c2 (diff) | |
| download | ffwd-3e8108cee394f44381193ee847515f81546f3383.tar.gz ffwd-3e8108cee394f44381193ee847515f81546f3383.zip | |
cuda: make bf16 BERT flash attention the default on the head_dim-64 path
Diffstat (limited to 'libffwd')
| -rw-r--r-- | libffwd/cuda.cu | 15 |
1 files changed, 9 insertions, 6 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index 985d7bd..138a70d 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -2738,15 +2738,18 @@ attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim, i return 0; } -// Opt-in BF16 tensor-core flash attention for the BERT shape (head_dim 64, -// bidirectional), replacing the materialized N×N scores/probs path. Enabled by -// FFWD_CUDA_BERT_FLASH=1; gated further on bf16 output + head_dim 64 at the call -// site. FFWD_TC_WARPS (2/4/8) tunes the query tiles per block (default 8). +// BF16 tensor-core flash attention for the BERT shape (head_dim 64, +// bidirectional), replacing the materialized N×N scores/probs path. It is bit- +// accurate (cosine vs F32 >= the materialized bf16 path) and faster at every +// length (the win grows with length as it removes the O(n^2) scores HBM), so it +// is the DEFAULT on the bf16w-bf16 head_dim-64 path. FFWD_CUDA_BERT_FLASH=0 +// forces the materialized path back for A/B. FFWD_TC_WARPS (2/4/8) tunes the +// query tiles per block (default 8). static int bert_flash_enabled(void) { - static int init = 0, on = 0; + static int init = 0, on = 1; if (!init) { const char *e = getenv("FFWD_CUDA_BERT_FLASH"); - on = (e && (!strcmp(e, "1") || !strcmp(e, "on"))); + on = !(e && (!strcmp(e, "0") || !strcmp(e, "off"))); init = 1; } return on; |
