diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-26 00:52:05 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-26 00:52:05 +0200 |
| commit | 733ec2cb70551a85032b44281623d1b3b43da511 (patch) | |
| tree | b89f229407be6e27f27195e08d576c1e6e12e07c /libffwd | |
| parent | fe3fe5d737e61e6e179b63f0480216c421810fd8 (diff) | |
| download | ffwd-733ec2cb70551a85032b44281623d1b3b43da511.tar.gz ffwd-733ec2cb70551a85032b44281623d1b3b43da511.zip | |
cuda: load Qwen flash V with ldmatrix.trans instead of gather
Diffstat (limited to 'libffwd')
| -rw-r--r-- | libffwd/cuda.cu | 118 |
1 files changed, 83 insertions, 35 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu index a06d5d0..3a9fffa 100644 --- a/libffwd/cuda.cu +++ b/libffwd/cuda.cu @@ -949,6 +949,22 @@ tc_load_b_pv_hd(uint32_t b[4], const __nv_bfloat16 *vsh, int d0) { } } +// PV B-operand loaded with ldmatrix.trans instead of the scalar gather above. +// Requires V staged in the SAME (key,d) grain-swizzle as K (SWZ_V). The PV operand +// has k=key, n=d, so mat&1 selects the key block and mat>>1 the d block (swapped vs +// the QK loader, whose n=key); ldmatrix.trans then yields (k=key, n=d) directly. +// The scalar gather is ~half the kernel at long context; this is the lever that +// lifts the tc128 path from ~48% to ~70% of bf16 peak (bit-identical V values). +template <int HD> +__device__ static __forceinline__ void +tc_load_b_pv_ldm(uint32_t b[4], const __nv_bfloat16 *vsh, int d0) { + int lane = threadIdx.x & 31; + int mat = lane >> 3; + int key = ((mat & 1) << 3) + (lane & 7); + int d = d0 + ((mat >> 1) << 3); + tc_ldmatrix_x4_trans_b16(b, vsh + tc_swz_idx<HD>(key, d)); +} + template <int HD> __device__ static __forceinline__ void tc_load_b_pv_hd_f16(uint32_t b[4], const __half *vsh, int d0) { int lane = threadIdx.x & 31; @@ -982,7 +998,7 @@ template <int N> __device__ static __forceinline__ void tc_cp_async_wait_group() #endif } -template <int HD, int KT, typename QKV_T, int DIRECT_KV> +template <int HD, int KT, typename QKV_T, int DIRECT_KV, int SWZ_V = 0> __device__ static __forceinline__ int bert_stage_kv_tile_bf16(__nv_bfloat16 *ksh, __nv_bfloat16 *vsh, const QKV_T *qkv, @@ -1019,11 +1035,12 @@ __device__ static __forceinline__ int bert_stage_kv_tile_bf16(__nv_bfloat16 *ksh vsrc = vexp + base; } // K lands grain-swizzled in (key, d) so the QK ldmatrix reads it - // directly with no transpose; V stays plain row-major for the PV - // gather. The 16-byte cp.async grain (8 head-dims) stays contiguous - // under the swizzle, which only permutes the 16-byte slot. + // directly with no transpose. V is plain row-major for the scalar PV + // gather, or the same (key, d) grain-swizzle when SWZ_V so the PV + // B-operand can be read with ldmatrix.trans. The 16-byte cp.async grain + // (8 head-dims) stays contiguous under the swizzle. tc_cp_async_cg_16(ksh + tc_swz_idx<HD>(kk, d), ksrc); - tc_cp_async_cg_16(vsh + kk * HD + d, vsrc); + tc_cp_async_cg_16(vsh + (SWZ_V ? tc_swz_idx<HD>(kk, d) : kk * HD + d), vsrc); } tc_cp_async_commit_group(); return 1; @@ -1047,7 +1064,7 @@ __device__ static __forceinline__ int bert_stage_kv_tile_bf16(__nv_bfloat16 *ksh } } ksh[tc_swz_idx<HD>(kk, d)] = kval; // (key, d) swizzled, matches the fast path - vsh[idx] = vval; + vsh[SWZ_V ? tc_swz_idx<HD>(kk, d) : idx] = vval; } return 0; } @@ -1128,7 +1145,7 @@ __device__ static __forceinline__ int bert_stage_kv_tile_f16(__half *ksh, // QK/softmax/PV recurrence; the legacy single-buffer path (PIPELINED=0) stages K // transposed+swizzled for ldmatrix.x4.trans and is kept only as a same-binary // A/B baseline. Both paths are numerically identical. -template <int CAUSAL, int WARPS, int PIPELINED> +template <int CAUSAL, int WARPS, int PIPELINED, int LDMV> __global__ static void attn_stream_tc128_bf16_kv_ldm_kernel(__nv_bfloat16 *out, const float *qkv, const __nv_bfloat16 *kexp, @@ -1219,9 +1236,9 @@ __global__ static void attn_stream_tc128_bf16_kv_ldm_kernel(__nv_bfloat16 *out, __nv_bfloat16 *vstage1 = vstage0 + KT * HD; int stage = 0; - int async_stage = bert_stage_kv_tile_bf16<HD, KT, float, 0>(kstage0, vstage0, qkv, kexp, vexp, - start, L, 0, kv_stride, qkv_dim, - 0, 0, kvhv, 0, tid, nthreads); + int async_stage = bert_stage_kv_tile_bf16<HD, KT, float, 0, LDMV>( + kstage0, vstage0, qkv, kexp, vexp, start, L, 0, kv_stride, qkv_dim, 0, 0, kvhv, 0, tid, + nthreads); for (int k0 = 0; k0 < key_hi; k0 += KT) { if (async_stage) @@ -1235,7 +1252,7 @@ __global__ static void attn_stream_tc128_bf16_kv_ldm_kernel(__nv_bfloat16 *out, int next_stage = stage ^ 1; int async_next = 0; if (next_k0 < key_hi) { - async_next = bert_stage_kv_tile_bf16<HD, KT, float, 0>( + async_next = bert_stage_kv_tile_bf16<HD, KT, float, 0, LDMV>( next_stage ? kstage1 : kstage0, next_stage ? vstage1 : vstage0, qkv, kexp, vexp, start, L, next_k0, kv_stride, qkv_dim, 0, 0, kvhv, 0, tid, nthreads); } @@ -1301,7 +1318,10 @@ __global__ static void attn_stream_tc128_bf16_kv_ldm_kernel(__nv_bfloat16 *out, #pragma unroll for (int dc = 0; dc < DCHUNKS; dc++) { uint32_t vb[4]; - tc_load_b_pv_hd<HD>(vb, vsh, dc * 16); + if constexpr (LDMV) + tc_load_b_pv_ldm<HD>(vb, vsh, dc * 16); + else + tc_load_b_pv_hd<HD>(vb, vsh, dc * 16); mma_m16n16k16_bf16(o[dc], pfrag, vb); } } @@ -2752,6 +2772,22 @@ static int qwen_compact_kv(void) { return on; } +// PV V-operand load mode for the Qwen tc128 flash path. Default 1 = ldmatrix.trans +// (V staged in the same (key,d) grain-swizzle as K; reads the whole 8x8 fragment in +// one instruction). env 0/off restores the scalar smem gather for A/B. The gather +// costs ~half the kernel at long context; ldmatrix lifts it from ~48% to ~70% of +// bf16 peak, bit-identical (same V values). Only affects the pipelined path. +static int qwen_ldmv_env(void) { + static int init = 0, on = 1; + if (!init) { + const char *e = getenv("FFWD_CUDA_QWEN_LDMV"); + if (e && (!strcmp(e, "0") || !strcmp(e, "off"))) + on = 0; + init = 1; + } + return on; +} + static int use_streaming_attention(const ffwd_config_t *c, int out_bf16, int head_dim) { int mode = streaming_attention_mode(); if (mode < 0) // FFWD_CUDA_STREAMING_ATTN=mat/off: force the materialized path @@ -3470,35 +3506,47 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx, int compact_kv = qwen_compact_kv(); int kv_stride = compact_kv ? c->n_kv_heads * hd : q_dim; // compact vs expanded row int kv_head_div = compact_kv ? H / c->n_kv_heads : 1; // query heads per kv head -#define TC3_LAUNCH(W, P) \ - do { \ - dim3 grid((max_L + (W) * 16 - 1) / ((W) * 16), H, batch); \ - dim3 block(32, (W)); \ - if (causal) \ - attn_stream_tc128_bf16_kv_ldm_kernel<1, (W), (P)><<<grid, block, 0, ctx->stream>>>( \ - (__nv_bfloat16 *)out_16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16, \ - (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale, \ - cskip, kv_stride, kv_head_div); \ - else \ - attn_stream_tc128_bf16_kv_ldm_kernel<0, (W), (P)><<<grid, block, 0, ctx->stream>>>( \ - (__nv_bfloat16 *)out_16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16, \ - (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale, \ - cskip, kv_stride, kv_head_div); \ +#define TC3_LAUNCH(W, P, LV) \ + do { \ + dim3 grid((max_L + (W) * 16 - 1) / ((W) * 16), H, batch); \ + dim3 block(32, (W)); \ + if (causal) \ + attn_stream_tc128_bf16_kv_ldm_kernel<1, (W), (P), (LV)> \ + <<<grid, block, 0, ctx->stream>>>( \ + (__nv_bfloat16 *)out_16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16, \ + (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale, \ + cskip, kv_stride, kv_head_div); \ + else \ + attn_stream_tc128_bf16_kv_ldm_kernel<0, (W), (P), (LV)> \ + <<<grid, block, 0, ctx->stream>>>( \ + (__nv_bfloat16 *)out_16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16, \ + (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale, \ + cskip, kv_stride, kv_head_div); \ } while (0) + int ldmv = qwen_ldmv_env(); if (qpipe) { - if (w == 2) - TC3_LAUNCH(2, 1); - else if (w == 8) - TC3_LAUNCH(8, 1); - else - TC3_LAUNCH(4, 1); + if (ldmv) { + if (w == 2) + TC3_LAUNCH(2, 1, 1); + else if (w == 8) + TC3_LAUNCH(8, 1, 1); + else + TC3_LAUNCH(4, 1, 1); + } else { + if (w == 2) + TC3_LAUNCH(2, 1, 0); + else if (w == 8) + TC3_LAUNCH(8, 1, 0); + else + TC3_LAUNCH(4, 1, 0); + } } else { if (w == 2) - TC3_LAUNCH(2, 0); + TC3_LAUNCH(2, 0, 0); else if (w == 8) - TC3_LAUNCH(8, 0); + TC3_LAUNCH(8, 0, 0); else - TC3_LAUNCH(4, 0); + TC3_LAUNCH(4, 0, 0); } #undef TC3_LAUNCH return launch_check(); |
