aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/cuda.cu20
1 files changed, 16 insertions, 4 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index 7dffcfa..5208f76 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -892,7 +892,15 @@ static int gemm_strided_batched(cublasHandle_t blas,
// layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged
// batches, single sequences, or when one sequence's scores exceed the
// budget.
-static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) {
+// use_bf16 must match the out_bf16 branch the caller will take in
+// cuda_attention_gemm: the V/probability/output pointer layout chosen here has to
+// agree with the data type that GEMM reads. BERT runs attention in F32 (passes
+// out_bf16=NULL) even with BF16 weights, so it must pass use_bf16=0; the Qwen
+// path runs BF16 attention when its weights are BF16. Keying this off the global
+// g_weights_bf16 instead silently mismatched the BERT path (BF16 layout, F32
+// GEMM) and corrupted equal-length batched attention.
+static int
+attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim, int use_bf16) {
const ffwd_config_t *c = &ctx->config;
ctx->attn_G = 0;
if (batch < 2)
@@ -913,7 +921,7 @@ static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int
if (G < 2)
return 0;
- int bf16 = g_weights_bf16;
+ int bf16 = use_bf16;
long long want = (long long)G * per;
if (want > ctx->attn_scores_elems) {
cudaFree(ctx->attn_scores);
@@ -1504,7 +1512,9 @@ static int cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inp
if (launch_check() != 0)
return -1;
- if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
+ // BERT attention runs in F32 (cuda_attention_gemm is called with NULL below),
+ // so the batched layout must be F32 too, regardless of weight storage dtype.
+ if (attn_batched_setup(ctx, batch, q_offset, qkv_dim, 0) != 0)
return -1;
for (int layer = 0; layer < c->n_layers; layer++) {
@@ -1593,7 +1603,9 @@ static int cuda_forward_batch(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs,
// projection GEMM consumes before any later producer reuses the buffer
// (single stream). No cast launches remain in the layer loop.
__nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16;
- if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
+ // BF16-weight layers run attention in BF16 (cuda_attention_gemm gets `act`);
+ // the batched layout must match. Weight dtype is uniform across layers.
+ if (attn_batched_setup(ctx, batch, q_offset, qkv_dim, g_weights_bf16) != 0)
return -1;
for (int layer = 0; layer < c->n_layers; layer++) {
cuda_layer_t *l = &ctx->layers[layer];