aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-22 07:43:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-22 07:43:55 +0200
commitf4876ad9f4a2b58ce222cc52d4039fe779e3adb1 (patch)
treebae0c4a45fe1888a918d97c4e8913d1f1c13879b
parent1a14da36fc4ee9f7fc02462a60db6c94e23656e7 (diff)
downloadffwd-f4876ad9f4a2b58ce222cc52d4039fe779e3adb1.tar.gz
ffwd-f4876ad9f4a2b58ce222cc52d4039fe779e3adb1.zip
cuda: add qwen bf16 tensor-core flash attention
-rw-r--r--libffwd/cuda.cu463
1 files changed, 459 insertions, 4 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index 2c64266..41fc4c8 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -912,6 +912,427 @@ __global__ static void attn_stream_wmma_kernel(OUT_T *out,
}
}
+__device__ static __forceinline__ uint32_t bf16_pair_bits(__nv_bfloat16 lo, __nv_bfloat16 hi) {
+ return (uint32_t)__bfloat16_as_ushort(lo) | ((uint32_t)__bfloat16_as_ushort(hi) << 16);
+}
+
+__device__ static __forceinline__ uint32_t bf16_pair_bits(float lo, float hi) {
+ return bf16_pair_bits(__float2bfloat16(lo), __float2bfloat16(hi));
+}
+
+__device__ static __forceinline__ float warp_group4_sum(float v) {
+ v += __shfl_xor_sync(0xffffffffu, v, 1);
+ v += __shfl_xor_sync(0xffffffffu, v, 2);
+ return v;
+}
+
+__device__ static __forceinline__ float warp_group4_max(float v) {
+ v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 2));
+ v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 1));
+ return v;
+}
+
+__device__ static __forceinline__ void mma_m16n16k16_bf16(float c[8],
+ const uint32_t a[4],
+ const uint32_t b[4]) {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+ asm volatile(
+ "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
+ : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3])
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]),
+ "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3]));
+ asm volatile(
+ "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
+ : "=f"(c[4]), "=f"(c[5]), "=f"(c[6]), "=f"(c[7])
+ : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[2]), "r"(b[3]),
+ "f"(c[4]), "f"(c[5]), "f"(c[6]), "f"(c[7]));
+#else
+ (void)c;
+ (void)a;
+ (void)b;
+#endif
+}
+
+__device__ static __forceinline__ void tc_load_a_q_bf16(uint32_t a[4],
+ const __nv_bfloat16 *qsh,
+ int d0) {
+ int lane = threadIdx.x & 31;
+#pragma unroll
+ for (int r = 0; r < 4; r++) {
+ int row = ((r & 1) << 3) + (lane >> 2);
+ int d = d0 + ((r >> 1) << 3) + ((lane & 3) << 1);
+ a[r] = bf16_pair_bits(qsh[row * 128 + d], qsh[row * 128 + d + 1]);
+ }
+}
+
+__device__ static __forceinline__ void tc_load_b_qk_bf16(uint32_t b[4],
+ const __nv_bfloat16 *ksh,
+ int d0) {
+ int lane = threadIdx.x & 31;
+#pragma unroll
+ for (int r = 0; r < 4; r++) {
+ int key_col = ((r >> 1) << 3) + (lane >> 2);
+ int d = d0 + ((r & 1) << 3) + ((lane & 3) << 1);
+ b[r] = bf16_pair_bits(ksh[key_col * 128 + d], ksh[key_col * 128 + d + 1]);
+ }
+}
+
+__device__ static __forceinline__ void tc_load_b_pv_bf16(uint32_t b[4],
+ const __nv_bfloat16 *vsh,
+ int d0) {
+ int lane = threadIdx.x & 31;
+#pragma unroll
+ for (int r = 0; r < 4; r++) {
+ int col = d0 + ((r >> 1) << 3) + (lane >> 2);
+ int key = ((r & 1) << 3) + ((lane & 3) << 1);
+ b[r] = bf16_pair_bits(vsh[key * 128 + col], vsh[(key + 1) * 128 + col]);
+ }
+}
+
+__device__ static __forceinline__ void tc_probs_to_a_bf16(uint32_t a[4], const float s[8]) {
+#pragma unroll
+ for (int r = 0; r < 4; r++)
+ a[r] = bf16_pair_bits(s[2 * r], s[2 * r + 1]);
+}
+
+__device__ static __forceinline__ void tc_load_a_p_bf16(uint32_t a[4],
+ const __nv_bfloat16 *psh) {
+ int lane = threadIdx.x & 31;
+#pragma unroll
+ for (int r = 0; r < 4; r++) {
+ int row = ((r & 1) << 3) + (lane >> 2);
+ int key = ((r >> 1) << 3) + ((lane & 3) << 1);
+ a[r] = bf16_pair_bits(psh[row * 16 + key], psh[row * 16 + key + 1]);
+ }
+}
+
+// BF16 tensor-core flash attention for the Qwen-family HD=128 BF16-output path.
+// One warp owns a 16-query tile for one (sequence, head). Q/K are rounded to
+// BF16 before QK, matching the reduced-precision materialized baseline; scores,
+// online softmax state, and the 16x128 output accumulator stay in registers.
+// K/V use shared memory only as the current 16-key staging tile.
+template <int CAUSAL>
+__global__ static void attn_stream_tc128_bf16_kernel(__nv_bfloat16 *out,
+ const float *qkv,
+ const float *kexp,
+ const __nv_bfloat16 *vexp,
+ const int *offsets,
+ int q_dim,
+ int qkv_dim,
+ int q_offset,
+ float scale) {
+ enum { QT = 16, KT = 16, HD = 128, DCHUNKS = 8 };
+ __shared__ __nv_bfloat16 qsh[QT * HD];
+ __shared__ __nv_bfloat16 ksh[KT * HD];
+ __shared__ __nv_bfloat16 vsh[KT * HD];
+
+ int lane = threadIdx.x & 31;
+ int q0 = blockIdx.x * QT;
+ int head = blockIdx.y;
+ int b = blockIdx.z;
+ int start = offsets[b];
+ int L = offsets[b + 1] - start;
+ if (q0 >= L)
+ return;
+ size_t hv = (size_t)head * HD;
+
+ for (int idx = lane; idx < QT * HD; idx += 32) {
+ int qr = idx / HD;
+ int d = idx - qr * HD;
+ int row = q0 + qr;
+ float qv = row < L ? qkv[(size_t)(start + row) * qkv_dim + q_offset + hv + d] : 0.0f;
+ qsh[idx] = __float2bfloat16(qv);
+ }
+ __syncwarp();
+
+ float o[DCHUNKS][8];
+#pragma unroll
+ for (int dc = 0; dc < DCHUNKS; dc++)
+#pragma unroll
+ for (int r = 0; r < 8; r++)
+ o[dc][r] = 0.0f;
+
+ float m[2] = {-3.402823466e+38F, -3.402823466e+38F};
+ float denom[2] = {0.0f, 0.0f};
+ int key_hi = L;
+ if (CAUSAL) {
+ int lim = q0 + QT;
+ if (lim < key_hi)
+ key_hi = lim;
+ }
+
+ for (int k0 = 0; k0 < key_hi; k0 += KT) {
+ for (int idx = lane; idx < KT * HD; idx += 32) {
+ int kk = idx / HD;
+ int d = idx - kk * HD;
+ int key = k0 + kk;
+ size_t kb = (size_t)(start + key) * q_dim + hv + d;
+ ksh[idx] = key < L ? __float2bfloat16(kexp[kb]) : __float2bfloat16(0.0f);
+ vsh[idx] = key < L ? vexp[kb] : __float2bfloat16(0.0f);
+ }
+ __syncwarp();
+
+ float s[8];
+#pragma unroll
+ for (int r = 0; r < 8; r++)
+ s[r] = 0.0f;
+#pragma unroll
+ for (int d0 = 0; d0 < HD; d0 += 16) {
+ uint32_t a[4], bb[4];
+ tc_load_a_q_bf16(a, qsh, d0);
+ tc_load_b_qk_bf16(bb, ksh, d0);
+ mma_m16n16k16_bf16(s, a, bb);
+ }
+
+ const float neg_inf = -3.402823466e+38F;
+#pragma unroll
+ for (int r = 0; r < 8; r++) {
+ int qr = (lane >> 2) + (((r % 4) >> 1) << 3);
+ int kc = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
+ int qrow = q0 + qr;
+ int key = k0 + kc;
+ if (qrow < L && key < L && (!CAUSAL || key <= qrow))
+ s[r] *= scale;
+ else
+ s[r] = neg_inf;
+ }
+
+#pragma unroll
+ for (int j = 0; j < 2; j++) {
+ int r0 = j * 2;
+ float tile_m = fmaxf(fmaxf(s[r0 + 0], s[r0 + 1]), fmaxf(s[r0 + 4], s[r0 + 5]));
+ float m_new = fmaxf(m[j], warp_group4_max(tile_m));
+ float corr = (m_new == neg_inf) ? 1.0f : __expf(m[j] - m_new);
+ denom[j] *= corr;
+#pragma unroll
+ for (int dc = 0; dc < DCHUNKS; dc++) {
+ o[dc][r0 + 0] *= corr;
+ o[dc][r0 + 1] *= corr;
+ o[dc][r0 + 4] *= corr;
+ o[dc][r0 + 5] *= corr;
+ }
+
+ float part = 0.0f;
+ float p0 = (s[r0 + 0] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 0] - m_new);
+ float p1 = (s[r0 + 1] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 1] - m_new);
+ float p4 = (s[r0 + 4] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 4] - m_new);
+ float p5 = (s[r0 + 5] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 5] - m_new);
+ s[r0 + 0] = p0;
+ s[r0 + 1] = p1;
+ s[r0 + 4] = p4;
+ s[r0 + 5] = p5;
+ part = p0 + p1 + p4 + p5;
+ denom[j] += warp_group4_sum(part);
+ m[j] = m_new;
+ }
+
+ uint32_t pfrag[4];
+ tc_probs_to_a_bf16(pfrag, s);
+#pragma unroll
+ for (int dc = 0; dc < DCHUNKS; dc++) {
+ uint32_t vb[4];
+ tc_load_b_pv_bf16(vb, vsh, dc * 16);
+ mma_m16n16k16_bf16(o[dc], pfrag, vb);
+ }
+ __syncwarp();
+ }
+
+#pragma unroll
+ for (int j = 0; j < 2; j++) {
+ int qrow = q0 + (lane >> 2) + j * 8;
+ if (qrow >= L)
+ continue;
+ float inv = denom[j] > 0.0f ? 1.0f / denom[j] : 0.0f;
+ size_t ob = (size_t)(start + qrow) * q_dim + hv;
+#pragma unroll
+ for (int dc = 0; dc < DCHUNKS; dc++) {
+ int col = dc * 16 + ((lane & 3) << 1);
+ store_act(out, ob + col + 0, o[dc][j * 2 + 0] * inv);
+ store_act(out, ob + col + 1, o[dc][j * 2 + 1] * inv);
+ store_act(out, ob + col + 8, o[dc][j * 2 + 4] * inv);
+ store_act(out, ob + col + 9, o[dc][j * 2 + 5] * inv);
+ }
+ }
+}
+
+// PV-parallel variant of the BF16 tensor-core tile. Warp 0 computes QK and the
+// online-softmax recurrence once per key tile, writing the 16x16 unnormalized
+// probability tile plus row rescale factors to shared memory. Warps 1..8 each
+// own one 16-column V/O chunk and keep their output accumulator in registers.
+// This preserves the register-resident output state while exposing the P*V MMA
+// work to eight tensor-core warps instead of serializing it in one warp.
+template <int CAUSAL>
+__global__ static void attn_stream_tc128_bf16_pv_kernel(__nv_bfloat16 *out,
+ const float *qkv,
+ const float *kexp,
+ const __nv_bfloat16 *vexp,
+ const int *offsets,
+ int q_dim,
+ int qkv_dim,
+ int q_offset,
+ float scale) {
+ enum { QT = 16, KT = 16, HD = 128, DCHUNKS = 8, WARPS = 9 };
+ __shared__ __nv_bfloat16 qsh[QT * HD];
+ __shared__ __nv_bfloat16 ksh[KT * HD];
+ __shared__ __nv_bfloat16 vsh[KT * HD];
+ __shared__ __nv_bfloat16 psh[QT * KT];
+ __shared__ float corr_sh[QT];
+ __shared__ float denom_sh[QT];
+
+ int lane = threadIdx.x & 31;
+ int warp = threadIdx.y;
+ int tid = warp * 32 + lane;
+ int q0 = blockIdx.x * QT;
+ int head = blockIdx.y;
+ int b = blockIdx.z;
+ int start = offsets[b];
+ int L = offsets[b + 1] - start;
+ if (q0 >= L)
+ return;
+ size_t hv = (size_t)head * HD;
+
+ for (int idx = tid; idx < QT * HD; idx += WARPS * 32) {
+ int qr = idx / HD;
+ int d = idx - qr * HD;
+ int row = q0 + qr;
+ float qv = row < L ? qkv[(size_t)(start + row) * qkv_dim + q_offset + hv + d] : 0.0f;
+ qsh[idx] = __float2bfloat16(qv);
+ }
+ if (tid < QT) {
+ corr_sh[tid] = 1.0f;
+ denom_sh[tid] = 0.0f;
+ }
+ __syncthreads();
+
+ float m[2] = {-3.402823466e+38F, -3.402823466e+38F};
+ float denom[2] = {0.0f, 0.0f};
+ float o[8];
+#pragma unroll
+ for (int r = 0; r < 8; r++)
+ o[r] = 0.0f;
+
+ int key_hi = L;
+ if (CAUSAL) {
+ int lim = q0 + QT;
+ if (lim < key_hi)
+ key_hi = lim;
+ }
+
+ for (int k0 = 0; k0 < key_hi; k0 += KT) {
+ for (int idx = tid; idx < KT * HD; idx += WARPS * 32) {
+ int kk = idx / HD;
+ int d = idx - kk * HD;
+ int key = k0 + kk;
+ size_t kb = (size_t)(start + key) * q_dim + hv + d;
+ ksh[idx] = key < L ? __float2bfloat16(kexp[kb]) : __float2bfloat16(0.0f);
+ vsh[idx] = key < L ? vexp[kb] : __float2bfloat16(0.0f);
+ }
+ __syncthreads();
+
+ if (warp == 0) {
+ float s[8];
+#pragma unroll
+ for (int r = 0; r < 8; r++)
+ s[r] = 0.0f;
+#pragma unroll
+ for (int d0 = 0; d0 < HD; d0 += 16) {
+ uint32_t a[4], bb[4];
+ tc_load_a_q_bf16(a, qsh, d0);
+ tc_load_b_qk_bf16(bb, ksh, d0);
+ mma_m16n16k16_bf16(s, a, bb);
+ }
+
+ const float neg_inf = -3.402823466e+38F;
+#pragma unroll
+ for (int r = 0; r < 8; r++) {
+ int qr = (lane >> 2) + (((r % 4) >> 1) << 3);
+ int kc = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
+ int qrow = q0 + qr;
+ int key = k0 + kc;
+ if (qrow < L && key < L && (!CAUSAL || key <= qrow))
+ s[r] *= scale;
+ else
+ s[r] = neg_inf;
+ }
+
+#pragma unroll
+ for (int j = 0; j < 2; j++) {
+ int r0 = j * 2;
+ int row = (lane >> 2) + j * 8;
+ float tile_m = fmaxf(fmaxf(s[r0 + 0], s[r0 + 1]), fmaxf(s[r0 + 4], s[r0 + 5]));
+ float m_new = fmaxf(m[j], warp_group4_max(tile_m));
+ float corr = (m_new == neg_inf) ? 1.0f : __expf(m[j] - m_new);
+ denom[j] *= corr;
+
+ float p0 =
+ (s[r0 + 0] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 0] - m_new);
+ float p1 =
+ (s[r0 + 1] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 1] - m_new);
+ float p4 =
+ (s[r0 + 4] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 4] - m_new);
+ float p5 =
+ (s[r0 + 5] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 5] - m_new);
+ s[r0 + 0] = p0;
+ s[r0 + 1] = p1;
+ s[r0 + 4] = p4;
+ s[r0 + 5] = p5;
+ denom[j] += warp_group4_sum(p0 + p1 + p4 + p5);
+ m[j] = m_new;
+ if ((lane & 3) == 0) {
+ corr_sh[row] = corr;
+ denom_sh[row] = denom[j];
+ }
+ }
+
+#pragma unroll
+ for (int r = 0; r < 8; r++) {
+ int row = (lane >> 2) + (((r % 4) >> 1) << 3);
+ int col = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
+ psh[row * KT + col] = __float2bfloat16(s[r]);
+ }
+ }
+ __syncthreads();
+
+ if (warp > 0) {
+ int dc = warp - 1;
+#pragma unroll
+ for (int j = 0; j < 2; j++) {
+ float corr = corr_sh[(lane >> 2) + j * 8];
+ int r0 = j * 2;
+ o[r0 + 0] *= corr;
+ o[r0 + 1] *= corr;
+ o[r0 + 4] *= corr;
+ o[r0 + 5] *= corr;
+ }
+ uint32_t pfrag[4], vb[4];
+ tc_load_a_p_bf16(pfrag, psh);
+ tc_load_b_pv_bf16(vb, vsh, dc * 16);
+ mma_m16n16k16_bf16(o, pfrag, vb);
+ }
+ __syncthreads();
+ }
+
+ if (warp > 0) {
+ int dc = warp - 1;
+#pragma unroll
+ for (int j = 0; j < 2; j++) {
+ int qrow = q0 + (lane >> 2) + j * 8;
+ if (qrow >= L)
+ continue;
+ float d = denom_sh[(lane >> 2) + j * 8];
+ float inv = d > 0.0f ? 1.0f / d : 0.0f;
+ size_t ob = (size_t)(start + qrow) * q_dim + hv;
+ int col = dc * 16 + ((lane & 3) << 1);
+ store_act(out, ob + col + 0, o[j * 2 + 0] * inv);
+ store_act(out, ob + col + 1, o[j * 2 + 1] * inv);
+ store_act(out, ob + col + 8, o[j * 2 + 4] * inv);
+ store_act(out, ob + col + 9, o[j * 2 + 5] * inv);
+ }
+ }
+}
+
// Register-resident streaming (online-softmax) attention: the FlashAttention-2
// tile for our packed layout. One warp owns RQ query rows of one (sequence,
// head); its 32 lanes split head_dim into DPL = HD/32 lane-local elements. The
@@ -1216,12 +1637,13 @@ static cublasComputeType_t gemm_compute(void) {
}
// Experimental streaming attention selector. Off by default; use
-// FFWD_CUDA_STREAMING_ATTN=reg for the register-resident FlashAttention tile,
-// =bert for the scalar BERT tile, or =wmma for the half-rounded tensor-core
-// prototype. =1/=all forces the generic row kernel for every head_dim<=128 call.
+// FFWD_CUDA_STREAMING_ATTN=tc for the BF16 tensor-core Qwen tile, =tc1 for the
+// one-warp BF16 tensor-core fallback, =reg for the F32 register-resident tile,
+// =bert for the scalar BERT tile, or =wmma for the shared-memory WMMA prototype.
+// =1/=all forces the generic row kernel for every head_dim<=128 call.
static int streaming_attention_mode(void) {
static int init = 0;
- static int mode = 0; // 0 off, 1 all, 2 BERT scalar tile, 3 WMMA tile, 4 register tile
+ static int mode = 0; // 0 off, 1 all, 2 BERT scalar, 3 WMMA, 4 reg, 5 BF16 TC, 6 TC fallback
if (!init) {
const char *e = getenv("FFWD_CUDA_STREAMING_ATTN");
if (e && (!strcmp(e, "1") || !strcmp(e, "all")))
@@ -1232,6 +1654,10 @@ static int streaming_attention_mode(void) {
mode = 3;
else if (e && (!strcmp(e, "reg") || !strcmp(e, "flash")))
mode = 4;
+ else if (e && (!strcmp(e, "tc") || !strcmp(e, "mma") || !strcmp(e, "mma_bf16")))
+ mode = 5;
+ else if (e && (!strcmp(e, "tc1") || !strcmp(e, "mma1") || !strcmp(e, "mma_bf16_1warp")))
+ mode = 6;
init = 1;
}
return mode;
@@ -1249,6 +1675,10 @@ static int use_streaming_attention(const ffwd_config_t *c, int out_bf16, int hea
// Register tile handles both shapes: head_dim 64 (BERT/BGE) and 128 (Qwen).
if (mode == 4)
return head_dim == 64 || head_dim == 128;
+ if (mode == 5)
+ return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
+ if (mode == 6)
+ return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
return 1;
}
@@ -1537,6 +1967,31 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
if (L > max_L)
max_L = L;
}
+ if (stream_mode == 5 && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && hd == 128) {
+ dim3 grid((max_L + 15) / 16, H, batch);
+ dim3 block(32, 9);
+ if (causal)
+ attn_stream_tc128_bf16_pv_kernel<1><<<grid, block, 0, ctx->stream>>>(
+ out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp, ctx->offsets,
+ q_dim, qkv_dim, q_offset, scale);
+ else
+ attn_stream_tc128_bf16_pv_kernel<0><<<grid, block, 0, ctx->stream>>>(
+ out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp, ctx->offsets,
+ q_dim, qkv_dim, q_offset, scale);
+ return launch_check();
+ }
+ if (stream_mode == 6 && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && hd == 128) {
+ dim3 grid((max_L + 15) / 16, H, batch);
+ if (causal)
+ attn_stream_tc128_bf16_kernel<1><<<grid, 32, 0, ctx->stream>>>(
+ out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp, ctx->offsets,
+ q_dim, qkv_dim, q_offset, scale);
+ else
+ attn_stream_tc128_bf16_kernel<0><<<grid, 32, 0, ctx->stream>>>(
+ out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp, ctx->offsets,
+ q_dim, qkv_dim, q_offset, scale);
+ return launch_check();
+ }
if (stream_mode == 4) {
// Register-resident tile: one warp owns RQ query rows, a block owns
// blockDim.y*RQ = 32 rows. hd=64: 4 warps x RQ=8, BC=16 keys/tile;