diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 16:38:45 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 16:38:45 +0200 |
| commit | a2907badacf6bb86e90d82f054419955be572d50 (patch) | |
| tree | 4bea85140cf0908f62cc18d87ad07dc3d11c0877 | |
| parent | a09b41dc4824eb33e11b1e71d95ec94d2411ead8 (diff) | |
| download | ffwd-a2907badacf6bb86e90d82f054419955be572d50.tar.gz ffwd-a2907badacf6bb86e90d82f054419955be572d50.zip | |
cuda: fix bf16 BERT batched-attention buffer-layout mismatch
| -rw-r--r-- | libffwd/cuda.cu | 20 | ||||
| -rwxr-xr-x | tests/check_cuda_fast_gemm_parity.py | 6 |
2 files changed, 22 insertions, 4 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index 7dffcfa..5208f76 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -892,7 +892,15 @@ static int gemm_strided_batched(cublasHandle_t blas, // layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged // batches, single sequences, or when one sequence's scores exceed the // budget. -static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) { +// use_bf16 must match the out_bf16 branch the caller will take in +// cuda_attention_gemm: the V/probability/output pointer layout chosen here has to +// agree with the data type that GEMM reads. BERT runs attention in F32 (passes +// out_bf16=NULL) even with BF16 weights, so it must pass use_bf16=0; the Qwen +// path runs BF16 attention when its weights are BF16. Keying this off the global +// g_weights_bf16 instead silently mismatched the BERT path (BF16 layout, F32 +// GEMM) and corrupted equal-length batched attention. +static int +attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim, int use_bf16) { const ffwd_config_t *c = &ctx->config; ctx->attn_G = 0; if (batch < 2) @@ -913,7 +921,7 @@ static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int if (G < 2) return 0; - int bf16 = g_weights_bf16; + int bf16 = use_bf16; long long want = (long long)G * per; if (want > ctx->attn_scores_elems) { cudaFree(ctx->attn_scores); @@ -1504,7 +1512,9 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp if (launch_check() != 0) return -1; - if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0) + // BERT attention runs in F32 (cuda_attention_gemm is called with NULL below), + // so the batched layout must be F32 too, regardless of weight storage dtype. + if (attn_batched_setup(ctx, batch, q_offset, qkv_dim, 0) != 0) return -1; for (int layer = 0; layer < c->n_layers; layer++) { @@ -1593,7 +1603,9 @@ static int cuda_forward_batch(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, // projection GEMM consumes before any later producer reuses the buffer // (single stream). No cast launches remain in the layer loop. __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16; - if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0) + // BF16-weight layers run attention in BF16 (cuda_attention_gemm gets `act`); + // the batched layout must match. Weight dtype is uniform across layers. + if (attn_batched_setup(ctx, batch, q_offset, qkv_dim, g_weights_bf16) != 0) return -1; for (int layer = 0; layer < c->n_layers; layer++) { cuda_layer_t *l = &ctx->layers[layer]; diff --git a/tests/check_cuda_fast_gemm_parity.py b/tests/check_cuda_fast_gemm_parity.py index bc201f7..57adf72 100755 --- a/tests/check_cuda_fast_gemm_parity.py +++ b/tests/check_cuda_fast_gemm_parity.py @@ -42,6 +42,12 @@ DEFAULT_TEXTS = [ # (BERT/MiniLM/BGE max_position 512, XLM-R 514) accept it; larger 8x/14x inputs # overflowed their position tables and were (correctly) rejected, which looked # like a CUDA failure. Still long enough to stress packed-sequence attention. +# +# NOTE on coverage: these defaults are all DIFFERENT lengths, so they exercise +# only the per-sequence (ragged) attention path. The CUDA batched-attention path +# (cublasGemmBatchedEx) engages only when every sequence in the batch is the SAME +# length (see attn_batched_setup). To exercise it, pass >=2 identical texts as +# positional args, e.g. the same sentence four times with --batch-size 4. def run_cuda(binary: str, model_dir: str, mode: str, weights: str, |
