aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--libffwd/cuda.cu20
-rwxr-xr-xtests/check_cuda_fast_gemm_parity.py6
2 files changed, 22 insertions, 4 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index 7dffcfa..5208f76 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -892,7 +892,15 @@ static int gemm_strided_batched(cublasHandle_t blas,
// layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged
// batches, single sequences, or when one sequence's scores exceed the
// budget.
-static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) {
+// use_bf16 must match the out_bf16 branch the caller will take in
+// cuda_attention_gemm: the V/probability/output pointer layout chosen here has to
+// agree with the data type that GEMM reads. BERT runs attention in F32 (passes
+// out_bf16=NULL) even with BF16 weights, so it must pass use_bf16=0; the Qwen
+// path runs BF16 attention when its weights are BF16. Keying this off the global
+// g_weights_bf16 instead silently mismatched the BERT path (BF16 layout, F32
+// GEMM) and corrupted equal-length batched attention.
+static int
+attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim, int use_bf16) {
const ffwd_config_t *c = &ctx->config;
ctx->attn_G = 0;
if (batch < 2)
@@ -913,7 +921,7 @@ static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int
if (G < 2)
return 0;
- int bf16 = g_weights_bf16;
+ int bf16 = use_bf16;
long long want = (long long)G * per;
if (want > ctx->attn_scores_elems) {
cudaFree(ctx->attn_scores);
@@ -1504,7 +1512,9 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp
if (launch_check() != 0)
return -1;
- if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
+ // BERT attention runs in F32 (cuda_attention_gemm is called with NULL below),
+ // so the batched layout must be F32 too, regardless of weight storage dtype.
+ if (attn_batched_setup(ctx, batch, q_offset, qkv_dim, 0) != 0)
return -1;
for (int layer = 0; layer < c->n_layers; layer++) {
@@ -1593,7 +1603,9 @@ static int cuda_forward_batch(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs,
// projection GEMM consumes before any later producer reuses the buffer
// (single stream). No cast launches remain in the layer loop.
__nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16;
- if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
+ // BF16-weight layers run attention in BF16 (cuda_attention_gemm gets `act`);
+ // the batched layout must match. Weight dtype is uniform across layers.
+ if (attn_batched_setup(ctx, batch, q_offset, qkv_dim, g_weights_bf16) != 0)
return -1;
for (int layer = 0; layer < c->n_layers; layer++) {
cuda_layer_t *l = &ctx->layers[layer];
diff --git a/tests/check_cuda_fast_gemm_parity.py b/tests/check_cuda_fast_gemm_parity.py
index bc201f7..57adf72 100755
--- a/tests/check_cuda_fast_gemm_parity.py
+++ b/tests/check_cuda_fast_gemm_parity.py
@@ -42,6 +42,12 @@ DEFAULT_TEXTS = [
# (BERT/MiniLM/BGE max_position 512, XLM-R 514) accept it; larger 8x/14x inputs
# overflowed their position tables and were (correctly) rejected, which looked
# like a CUDA failure. Still long enough to stress packed-sequence attention.
+#
+# NOTE on coverage: these defaults are all DIFFERENT lengths, so they exercise
+# only the per-sequence (ragged) attention path. The CUDA batched-attention path
+# (cublasGemmBatchedEx) engages only when every sequence in the batch is the SAME
+# length (see attn_batched_setup). To exercise it, pass >=2 identical texts as
+# positional args, e.g. the same sentence four times with --batch-size 4.
def run_cuda(binary: str, model_dir: str, mode: str, weights: str,