aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--libffwd/cuda.cu105
1 files changed, 65 insertions, 40 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index ae601bb..a238798 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -852,6 +852,42 @@ tc_load_b_qk_ldmatrix_swz_f16(uint32_t b[4], const __half *ktsh, int d0) {
tc_ldmatrix_x4_trans_b16(b, ktsh + tc_swz_idx<16>(row, col));
}
+// Direct-read QK B-operand loaders for the BERT flash kernels. Unlike the
+// transposed loaders above, these read K straight from the cp.async-staged
+// ROW-MAJOR (key, d) shared tile (stride HD, grain XOR-swizzled), so the kernel
+// pays no shared->shared K transpose and no extra barrier per key tile.
+//
+// The QK MMA is m16n8k16.row.col, so its B operand is column-major (d, key). A
+// plain (non-transposing) ldmatrix.x4 over a (key, d) row-major source produces
+// exactly that fragment: thread t ends up with the pair
+// {K[key=t/4][d=2*(t%4)], K[key=t/4][d=2*(t%4)+1]}, which is {k=2*(t%4),
+// n=t/4} = the B operand the mma expects (k = head-dim, n = key). So QK^T is
+// bit-for-bit identical to the transposed-staging loader; only the shared layout
+// it reads from differs. The four ldmatrix sub-tiles map to (key block, d half):
+// mat&2 selects keys 0..7 vs 8..15, mat&1 selects the low vs high 8 of the
+// 16-wide head-dim chunk. `ksh` points at the current 16-key sub-tile base
+// (krow + sub*16*HD); since the 16-key offset is a multiple of 8 the grain
+// swizzle is the same whether computed on the local or the absolute key.
+template <int HD>
+__device__ static __forceinline__ void
+tc_load_b_qk_keyd_swz_bf16(uint32_t b[4], const __nv_bfloat16 *ksh, int d0) {
+ int lane = threadIdx.x & 31;
+ int mat = lane >> 3;
+ int key = ((mat >> 1) << 3) + (lane & 7);
+ int d = d0 + ((mat & 1) << 3);
+ tc_ldmatrix_x4_b16(b, ksh + tc_swz_idx<HD>(key, d));
+}
+
+template <int HD>
+__device__ static __forceinline__ void
+tc_load_b_qk_keyd_swz_f16(uint32_t b[4], const __half *ksh, int d0) {
+ int lane = threadIdx.x & 31;
+ int mat = lane >> 3;
+ int key = ((mat >> 1) << 3) + (lane & 7);
+ int d = d0 + ((mat & 1) << 3);
+ tc_ldmatrix_x4_b16(b, ksh + tc_swz_idx<HD>(key, d));
+}
+
__device__ static __forceinline__ void
tc_load_b_pv_bf16(uint32_t b[4], const __nv_bfloat16 *vsh, int d0) {
int lane = threadIdx.x & 31;
@@ -924,28 +960,6 @@ __device__ static __forceinline__ void tc_load_b_pv_hd_f16(uint32_t b[4], const
}
}
-template <int HD>
-__device__ static __forceinline__ void
-bert_transpose_k_tile_bf16(__nv_bfloat16 *ktsh, const __nv_bfloat16 *krow, int tid, int nthreads) {
- enum { KT = 64 };
- for (int idx = tid; idx < KT * HD; idx += nthreads) {
- int kk = idx / HD;
- int d = idx - kk * HD;
- tc_store_swz_bf16<16>(ktsh + (kk >> 4) * 16 * HD, d, kk & 15, krow[idx]);
- }
-}
-
-template <int HD>
-__device__ static __forceinline__ void
-bert_transpose_k_tile_f16(__half *ktsh, const __half *krow, int tid, int nthreads) {
- enum { KT = 64 };
- for (int idx = tid; idx < KT * HD; idx += nthreads) {
- int kk = idx / HD;
- int d = idx - kk * HD;
- tc_store_swz_f16<16>(ktsh + (kk >> 4) * 16 * HD, d, kk & 15, krow[idx]);
- }
-}
-
__device__ static __forceinline__ void tc_cp_async_cg_16(void *dst, const void *src) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
uint32_t smem = (uint32_t)__cvta_generic_to_shared(dst);
@@ -1004,7 +1018,11 @@ __device__ static __forceinline__ int bert_stage_kv_tile_bf16(__nv_bfloat16 *ksh
ksrc = kexp + base;
vsrc = vexp + base;
}
- tc_cp_async_cg_16(ksh + kk * HD + d, ksrc);
+ // K lands grain-swizzled in (key, d) so the QK ldmatrix reads it
+ // directly with no transpose; V stays plain row-major for the PV
+ // gather. The 16-byte cp.async grain (8 head-dims) stays contiguous
+ // under the swizzle, which only permutes the 16-byte slot.
+ tc_cp_async_cg_16(ksh + tc_swz_idx<HD>(kk, d), ksrc);
tc_cp_async_cg_16(vsh + kk * HD + d, vsrc);
}
tc_cp_async_commit_group();
@@ -1028,7 +1046,7 @@ __device__ static __forceinline__ int bert_stage_kv_tile_bf16(__nv_bfloat16 *ksh
vval = vexp[base];
}
}
- ksh[idx] = kval;
+ ksh[tc_swz_idx<HD>(kk, d)] = kval; // (key, d) swizzled, matches the fast path
vsh[idx] = vval;
}
return 0;
@@ -1070,7 +1088,9 @@ __device__ static __forceinline__ int bert_stage_kv_tile_f16(__half *ksh,
ksrc = kexp + base;
vsrc = vexp + base;
}
- tc_cp_async_cg_16(ksh + kk * HD + d, ksrc);
+ // K grain-swizzled in (key, d) for the direct QK ldmatrix read; V
+ // stays plain row-major (see the bf16 sibling for the rationale).
+ tc_cp_async_cg_16(ksh + tc_swz_idx<HD>(kk, d), ksrc);
tc_cp_async_cg_16(vsh + kk * HD + d, vsrc);
}
tc_cp_async_commit_group();
@@ -1094,7 +1114,7 @@ __device__ static __forceinline__ int bert_stage_kv_tile_f16(__half *ksh,
vval = vexp[base];
}
}
- ksh[idx] = kval;
+ ksh[tc_swz_idx<HD>(kk, d)] = kval; // (key, d) swizzled, matches the fast path
vsh[idx] = vval;
}
return 0;
@@ -1275,7 +1295,11 @@ __global__ static void attn_stream_bert_flash_kernel(__nv_bfloat16 *out,
float scale) {
enum { QT = 16, KT = 64, DCHUNKS = HD / 16, KSUB = KT / 16 };
enum { Q_ELEMS = WARPS * QT * HD, KV_LEGACY_ELEMS = Q_ELEMS + 2 * KT * HD };
- enum { KV_PIPE_ELEMS = 5 * KT * HD };
+ // Pipelined K/V double-buffer: two K stages + two V stages. K is staged in
+ // (key, d) row-major and read by the QK ldmatrix directly, so there is no
+ // separate transposed-K buffer; dropping that 5th KT*HD tile frees ~8 KB of
+ // shared memory (helping occupancy) and one barrier per key tile.
+ enum { KV_PIPE_ELEMS = 4 * KT * HD };
enum {
SH_ELEMS = PIPELINED ? ((Q_ELEMS > KV_PIPE_ELEMS) ? Q_ELEMS : KV_PIPE_ELEMS) : KV_LEGACY_ELEMS
};
@@ -1336,7 +1360,6 @@ __global__ static void attn_stream_bert_flash_kernel(__nv_bfloat16 *out,
__nv_bfloat16 *kstage1 = kstage0 + KT * HD;
__nv_bfloat16 *vstage0 = kstage1 + KT * HD;
__nv_bfloat16 *vstage1 = vstage0 + KT * HD;
- __nv_bfloat16 *ktsh = vstage1 + KT * HD;
int stage = 0;
int async_stage = bert_stage_kv_tile_bf16<HD, QKV_T, DIRECT_KV>(
@@ -1345,7 +1368,9 @@ __global__ static void attn_stream_bert_flash_kernel(__nv_bfloat16 *out,
// K/V staging is double-buffered. While one 64-key tile feeds the
// unchanged QK/softmax/PV recurrence, the next tile is already moving
- // from global to the other shared stage.
+ // from global to the other shared stage. K is staged in the (key, d)
+ // layout the QK ldmatrix reads directly, so the single barrier below
+ // (publishing the freshly staged tile) is the only one per key tile.
for (int k0 = 0; k0 < L; k0 += KT) { // bidirectional: every query attends every key
if (async_stage)
tc_cp_async_wait_group<0>();
@@ -1353,8 +1378,6 @@ __global__ static void attn_stream_bert_flash_kernel(__nv_bfloat16 *out,
__nv_bfloat16 *krow = stage ? kstage1 : kstage0;
__nv_bfloat16 *vsh = stage ? vstage1 : vstage0;
- bert_transpose_k_tile_bf16<HD>(ktsh, krow, tid, nthreads);
- __syncthreads();
int next_k0 = k0 + KT;
int next_stage = stage ^ 1;
@@ -1368,7 +1391,7 @@ __global__ static void attn_stream_bert_flash_kernel(__nv_bfloat16 *out,
#pragma unroll
for (int sub = 0; sub < KSUB; sub++) {
- const __nv_bfloat16 *ksh_s = ktsh + sub * 16 * HD;
+ const __nv_bfloat16 *ksh_s = krow + sub * 16 * HD;
const __nv_bfloat16 *vsh_s = vsh + sub * 16 * HD;
int ksub0 = k0 + sub * 16;
if (ksub0 >= L)
@@ -1381,7 +1404,7 @@ __global__ static void attn_stream_bert_flash_kernel(__nv_bfloat16 *out,
#pragma unroll
for (int d0 = 0; d0 < HD; d0 += 16) {
uint32_t bb[4];
- tc_load_b_qk_ldmatrix_swz_bf16(bb, ksh_s, d0);
+ tc_load_b_qk_keyd_swz_bf16<HD>(bb, ksh_s, d0);
mma_m16n16k16_bf16(s, a_q[d0 >> 4], bb);
}
@@ -1579,7 +1602,10 @@ __global__ static void attn_stream_bert_flash_f16_kernel(__half *out,
float scale) {
enum { QT = 16, KT = 64, DCHUNKS = HD / 16, KSUB = KT / 16 };
enum { Q_ELEMS = WARPS * QT * HD, KV_LEGACY_ELEMS = Q_ELEMS + 2 * KT * HD };
- enum { KV_PIPE_ELEMS = 5 * KT * HD };
+ // See the bf16 sibling: K is read directly from its (key, d) staged tile, so
+ // the pipeline needs only two K and two V stages, not a fifth transposed-K
+ // tile.
+ enum { KV_PIPE_ELEMS = 4 * KT * HD };
enum {
SH_ELEMS = PIPELINED ? ((Q_ELEMS > KV_PIPE_ELEMS) ? Q_ELEMS : KV_PIPE_ELEMS) : KV_LEGACY_ELEMS
};
@@ -1637,7 +1663,6 @@ __global__ static void attn_stream_bert_flash_f16_kernel(__half *out,
__half *kstage1 = kstage0 + KT * HD;
__half *vstage0 = kstage1 + KT * HD;
__half *vstage1 = vstage0 + KT * HD;
- __half *ktsh = vstage1 + KT * HD;
int stage = 0;
int async_stage = bert_stage_kv_tile_f16<HD, QKV_T, DIRECT_KV>(
@@ -1646,7 +1671,9 @@ __global__ static void attn_stream_bert_flash_f16_kernel(__half *out,
// K/V staging is double-buffered. While one 64-key tile feeds the
// unchanged QK/softmax/PV recurrence, the next tile is already moving
- // from global to the other shared stage.
+ // from global to the other shared stage. K is staged in the (key, d)
+ // layout the QK ldmatrix reads directly, so the single barrier below is
+ // the only one per key tile.
for (int k0 = 0; k0 < L; k0 += KT) {
if (async_stage)
tc_cp_async_wait_group<0>();
@@ -1654,8 +1681,6 @@ __global__ static void attn_stream_bert_flash_f16_kernel(__half *out,
__half *krow = stage ? kstage1 : kstage0;
__half *vsh = stage ? vstage1 : vstage0;
- bert_transpose_k_tile_f16<HD>(ktsh, krow, tid, nthreads);
- __syncthreads();
int next_k0 = k0 + KT;
int next_stage = stage ^ 1;
@@ -1669,7 +1694,7 @@ __global__ static void attn_stream_bert_flash_f16_kernel(__half *out,
#pragma unroll
for (int sub = 0; sub < KSUB; sub++) {
- const __half *ksh_s = ktsh + sub * 16 * HD;
+ const __half *ksh_s = krow + sub * 16 * HD;
const __half *vsh_s = vsh + sub * 16 * HD;
int ksub0 = k0 + sub * 16;
if (ksub0 >= L)
@@ -1682,7 +1707,7 @@ __global__ static void attn_stream_bert_flash_f16_kernel(__half *out,
#pragma unroll
for (int d0 = 0; d0 < HD; d0 += 16) {
uint32_t bb[4];
- tc_load_b_qk_ldmatrix_swz_f16(bb, ksh_s, d0);
+ tc_load_b_qk_keyd_swz_f16<HD>(bb, ksh_s, d0);
mma_m16n16k16_f16(s, a_q[d0 >> 4], bb);
}