aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-22 17:11:26 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-22 17:11:26 +0200
commit74fc389fb382cc79f67b67591569144b3fa393bd (patch)
treee5b4925f4096a463aa6566585f4cd22433380a29 /libffwd
parentebdf85d4d3c96fd7015192a44b94bb3bcad19735 (diff)
downloadffwd-74fc389fb382cc79f67b67591569144b3fa393bd.tar.gz
ffwd-74fc389fb382cc79f67b67591569144b3fa393bd.zip
cuda: prune attention experiments
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/cuda.cu1347
1 files changed, 17 insertions, 1330 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index e36f0a8..b308ca1 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -7,7 +7,6 @@ extern "C" {
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cublas_v2.h>
-#include <mma.h>
#include <limits.h>
#include <math.h>
@@ -416,12 +415,6 @@ __device__ static inline float load_embedding_value(const void *emb, int emb_bf1
return emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[i]) : ((const float *)emb)[i];
}
-__device__ static inline float load_attn_value(const float *v, size_t i) { return v[i]; }
-
-__device__ static inline float load_attn_value(const __nv_bfloat16 *v, size_t i) {
- return __bfloat162float(v[i]);
-}
-
// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight
// projection GEMM directly (same rounding as a separate cast kernel).
template <typename OUT_T>
@@ -637,283 +630,6 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, int
store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f);
}
-// One block computes one (sequence, head, query) row using the online-softmax
-// recurrence. Scores stay in registers and the output accumulator; no LxL score
-// matrix is written to global memory. This first version is intentionally
-// simple and handles the head_dim <= blockDim.x shapes used by BERT/BGE and the
-// current Qwen blocks.
-template <typename V_T, typename OUT_T>
-__global__ static void attn_stream_row_kernel(OUT_T *out,
- const float *qkv,
- const float *kexp,
- const V_T *vexp,
- const int *offsets,
- int head_dim,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale,
- int causal) {
- int qrow = blockIdx.x;
- int head = blockIdx.y;
- int b = blockIdx.z;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
- if (qrow >= L || head_dim > blockDim.x)
- return;
-
- int tid = threadIdx.x;
- size_t q_base = (size_t)(start + qrow) * qkv_dim + q_offset + (size_t)head * head_dim;
- size_t hv = (size_t)head * head_dim;
- float qv = tid < head_dim ? qkv[q_base + tid] : 0.0f;
- float acc = 0.0f;
- float m = -3.402823466e+38F;
- float denom = 0.0f;
- int keys = causal ? qrow + 1 : L;
-
- for (int j = 0; j < keys; j++) {
- size_t k_base = (size_t)(start + j) * q_dim + hv;
- float dot_part = tid < head_dim ? qv * kexp[k_base + tid] : 0.0f;
- float score = block_sum(dot_part) * scale;
- float m_new = fmaxf(m, score);
- float old_scale = __expf(m - m_new);
- float p = __expf(score - m_new);
- denom = denom * old_scale + p;
- if (tid < head_dim) {
- float vv = load_attn_value(vexp, k_base + tid);
- acc = acc * old_scale + p * vv;
- }
- m = m_new;
- }
-
- if (tid < head_dim)
- store_act(out, (size_t)(start + qrow) * q_dim + hv + tid, acc / denom);
-}
-
-// BERT/BGE non-causal head_dim=64 specialization. One block owns one query
-// tile for one (sequence, head); each warp accumulates one query row while the
-// block loads a K/V row into shared memory once and reuses it across the query
-// warps.
-__global__ static void attn_stream_bert64_kernel(float *out,
- const float *qkv,
- const float *kexp,
- const float *vexp,
- const int *offsets,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale) {
- __shared__ float kbuf[64];
- __shared__ float vbuf[64];
- int q_per_block = blockDim.x >> 5;
- int q_tile = blockIdx.x * q_per_block;
- int head = blockIdx.y;
- int b = blockIdx.z;
- int lane = threadIdx.x & 31;
- int warp = threadIdx.x >> 5;
- int qrow = q_tile + warp;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
- int active = qrow < L;
- size_t hv = (size_t)head * 64;
- size_t q_base = (size_t)(start + qrow) * qkv_dim + q_offset + hv;
- float q0 = active ? qkv[q_base + lane] : 0.0f;
- float q1 = active ? qkv[q_base + lane + 32] : 0.0f;
- float acc0 = 0.0f;
- float acc1 = 0.0f;
- float m = -3.402823466e+38F;
- float denom = 0.0f;
-
- for (int j = 0; j < L; j++) {
- size_t kv_base = (size_t)(start + j) * q_dim + hv;
- if (threadIdx.x < 64) {
- kbuf[threadIdx.x] = kexp[kv_base + threadIdx.x];
- vbuf[threadIdx.x] = vexp[kv_base + threadIdx.x];
- }
- __syncthreads();
-
- float dot = q0 * kbuf[lane] + q1 * kbuf[lane + 32];
- float score = warp_sum(dot) * scale;
- if (active) {
- float m_new = fmaxf(m, score);
- float old_scale = __expf(m - m_new);
- float p = __expf(score - m_new);
- denom = denom * old_scale + p;
- acc0 = acc0 * old_scale + p * vbuf[lane];
- acc1 = acc1 * old_scale + p * vbuf[lane + 32];
- m = m_new;
- }
- __syncthreads();
- }
-
- if (active) {
- size_t o_base = (size_t)(start + qrow) * q_dim + hv;
- out[o_base + lane] = acc0 / denom;
- out[o_base + lane + 32] = acc1 / denom;
- }
-}
-
-// Tensor-core prototype for 16-row query tiles. It keeps the online-softmax
-// state in shared memory and computes each 16x16 QK/PV tile with WMMA using
-// half-rounded Q/K/P/V operands. The output accumulator stays F32.
-template <int HD, typename V_T, typename OUT_T, int CAUSAL>
-__global__ static void attn_stream_wmma_kernel(OUT_T *out,
- const float *qkv,
- const float *kexp,
- const V_T *vexp,
- const int *offsets,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale) {
- enum { QT = 16, KT = 16, DCHUNKS = HD / 16 };
- __shared__ half qh[QT * HD];
- __shared__ half kh[HD * KT];
- __shared__ half ph[QT * KT];
- __shared__ half vh[KT * HD];
- __shared__ float scores[QT * KT];
- __shared__ float acc[QT * HD];
- __shared__ float delta[DCHUNKS * QT * 16];
- __shared__ float row_m[QT];
- __shared__ float row_d[QT];
- __shared__ float row_old_scale[QT];
-
- int tid = threadIdx.x;
- int q_tile = blockIdx.x * QT;
- int head = blockIdx.y;
- int b = blockIdx.z;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
- size_t hv = (size_t)head * HD;
-
- for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
- int qr = idx / HD;
- int d = idx - qr * HD;
- int qrow = q_tile + qr;
- float qv = qrow < L ? qkv[(size_t)(start + qrow) * qkv_dim + q_offset + hv + d] : 0.0f;
- qh[idx] = __float2half(qv);
- acc[idx] = 0.0f;
- }
- if (tid < QT) {
- row_m[tid] = -3.402823466e+38F;
- row_d[tid] = 0.0f;
- row_old_scale[tid] = 1.0f;
- }
- __syncthreads();
-
- for (int k0 = 0; k0 < L; k0 += KT) {
- for (int idx = tid; idx < HD * KT; idx += blockDim.x) {
- int d = idx / KT;
- int kk = idx - d * KT;
- int key = k0 + kk;
- float kv = key < L ? kexp[(size_t)(start + key) * q_dim + hv + d] : 0.0f;
- kh[idx] = __float2half(kv);
- }
- __syncthreads();
-
- if (tid < 32) {
- nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, 16, 16, 16, half, nvcuda::wmma::row_major>
- a_frag;
- nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, 16, 16, 16, half, nvcuda::wmma::row_major>
- b_frag;
- nvcuda::wmma::fragment<nvcuda::wmma::accumulator, 16, 16, 16, float> c_frag;
- nvcuda::wmma::fill_fragment(c_frag, 0.0f);
- for (int kd = 0; kd < HD; kd += 16) {
- nvcuda::wmma::load_matrix_sync(a_frag, qh + kd, HD);
- nvcuda::wmma::load_matrix_sync(b_frag, kh + kd * KT, KT);
- nvcuda::wmma::mma_sync(c_frag, a_frag, b_frag, c_frag);
- }
- nvcuda::wmma::store_matrix_sync(scores, c_frag, KT, nvcuda::wmma::mem_row_major);
- }
- __syncthreads();
-
- if (tid < QT) {
- int qrow = q_tile + tid;
- if (qrow < L) {
- float tile_m = -3.402823466e+38F;
- for (int kk = 0; kk < KT; kk++) {
- int key = k0 + kk;
- if (key < L && (!CAUSAL || key <= qrow))
- tile_m = fmaxf(tile_m, scores[tid * KT + kk] * scale);
- }
- float m_new = fmaxf(row_m[tid], tile_m);
- float old_scale = __expf(row_m[tid] - m_new);
- float tile_sum = 0.0f;
- for (int kk = 0; kk < KT; kk++) {
- int key = k0 + kk;
- float p = 0.0f;
- if (key < L && (!CAUSAL || key <= qrow))
- p = __expf(scores[tid * KT + kk] * scale - m_new);
- scores[tid * KT + kk] = p;
- tile_sum += p;
- }
- row_m[tid] = m_new;
- row_d[tid] = row_d[tid] * old_scale + tile_sum;
- row_old_scale[tid] = old_scale;
- } else {
- row_old_scale[tid] = 1.0f;
- for (int kk = 0; kk < KT; kk++)
- scores[tid * KT + kk] = 0.0f;
- }
- }
- __syncthreads();
-
- for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
- int qr = idx / HD;
- int qrow = q_tile + qr;
- if (qrow < L)
- acc[idx] *= row_old_scale[qr];
- }
- for (int idx = tid; idx < QT * KT; idx += blockDim.x)
- ph[idx] = __float2half(scores[idx]);
- for (int idx = tid; idx < KT * HD; idx += blockDim.x) {
- int kk = idx / HD;
- int d = idx - kk * HD;
- int key = k0 + kk;
- float vv = key < L ? load_attn_value(vexp, (size_t)(start + key) * q_dim + hv + d) : 0.0f;
- vh[idx] = __float2half(vv);
- }
- __syncthreads();
-
- int warp_id = tid >> 5;
- if (warp_id < DCHUNKS) {
- int d0 = warp_id * 16;
- nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, 16, 16, 16, half, nvcuda::wmma::row_major>
- p_frag;
- nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, 16, 16, 16, half, nvcuda::wmma::row_major>
- v_frag;
- nvcuda::wmma::fragment<nvcuda::wmma::accumulator, 16, 16, 16, float> o_frag;
- nvcuda::wmma::fill_fragment(o_frag, 0.0f);
- nvcuda::wmma::load_matrix_sync(p_frag, ph, KT);
- nvcuda::wmma::load_matrix_sync(v_frag, vh + d0, HD);
- nvcuda::wmma::mma_sync(o_frag, p_frag, v_frag, o_frag);
- nvcuda::wmma::store_matrix_sync(delta + warp_id * QT * 16, o_frag, 16,
- nvcuda::wmma::mem_row_major);
- }
- __syncthreads();
-
- for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
- int qr = idx / HD;
- int d = idx - qr * HD;
- int qrow = q_tile + qr;
- if (qrow < L) {
- int chunk = d >> 4;
- int col = d & 15;
- acc[idx] += delta[chunk * QT * 16 + qr * 16 + col];
- }
- }
- __syncthreads();
- }
-
- for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
- int qr = idx / HD;
- int d = idx - qr * HD;
- int qrow = q_tile + qr;
- if (qrow < L)
- store_act(out, (size_t)(start + qrow) * q_dim + hv + d, acc[idx] / row_d[qr]);
- }
-}
-
__device__ static __forceinline__ uint32_t bf16_pair_bits(__nv_bfloat16 lo, __nv_bfloat16 hi) {
return (uint32_t)__bfloat16_as_ushort(lo) | ((uint32_t)__bfloat16_as_ushort(hi) << 16);
}
@@ -1012,28 +728,6 @@ tc_load_b_qk_ldmatrix_swz_bf16(uint32_t b[4], const __nv_bfloat16 *ktsh, int d0)
}
__device__ static __forceinline__ void
-tc_load_a_q_bf16(uint32_t a[4], const __nv_bfloat16 *qsh, int d0) {
- int lane = threadIdx.x & 31;
-#pragma unroll
- for (int r = 0; r < 4; r++) {
- int row = ((r & 1) << 3) + (lane >> 2);
- int d = d0 + ((r >> 1) << 3) + ((lane & 3) << 1);
- a[r] = bf16_pair_bits(qsh[row * 128 + d], qsh[row * 128 + d + 1]);
- }
-}
-
-__device__ static __forceinline__ void
-tc_load_b_qk_bf16(uint32_t b[4], const __nv_bfloat16 *ksh, int d0) {
- int lane = threadIdx.x & 31;
-#pragma unroll
- for (int r = 0; r < 4; r++) {
- int key_col = ((r >> 1) << 3) + (lane >> 2);
- int d = d0 + ((r & 1) << 3) + ((lane & 3) << 1);
- b[r] = bf16_pair_bits(ksh[key_col * 128 + d], ksh[key_col * 128 + d + 1]);
- }
-}
-
-__device__ static __forceinline__ void
tc_load_b_pv_bf16(uint32_t b[4], const __nv_bfloat16 *vsh, int d0) {
int lane = threadIdx.x & 31;
#pragma unroll
@@ -1050,16 +744,6 @@ __device__ static __forceinline__ void tc_probs_to_a_bf16(uint32_t a[4], const f
a[r] = bf16_pair_bits(s[2 * r], s[2 * r + 1]);
}
-__device__ static __forceinline__ void tc_load_a_p_bf16(uint32_t a[4], const __nv_bfloat16 *psh) {
- int lane = threadIdx.x & 31;
-#pragma unroll
- for (int r = 0; r < 4; r++) {
- int row = ((r & 1) << 3) + (lane >> 2);
- int key = ((r >> 1) << 3) + ((lane & 3) << 1);
- a[r] = bf16_pair_bits(psh[row * 16 + key], psh[row * 16 + key + 1]);
- }
-}
-
// Head-dim-parametric Q and V fragment loaders for the flash kernels. The QK S
// tile and the PV O tile are 16x16 regardless of head_dim, so only the Q-tile
// swizzle stride (HD) and the V row stride (HD) depend on it; these are the
@@ -1088,676 +772,10 @@ tc_load_b_pv_hd(uint32_t b[4], const __nv_bfloat16 *vsh, int d0) {
}
}
-// BF16 tensor-core flash attention for the Qwen-family HD=128 BF16-output path.
-// One warp owns a 16-query tile for one (sequence, head). Q/K are rounded to
-// BF16 before QK, matching the reduced-precision materialized baseline; scores,
-// online softmax state, and the 16x128 output accumulator stay in registers.
-// K/V use shared memory only as the current 16-key staging tile.
-template <int CAUSAL>
-__global__ static void attn_stream_tc128_bf16_kernel(__nv_bfloat16 *out,
- const float *qkv,
- const __nv_bfloat16 *kexp,
- const __nv_bfloat16 *vexp,
- const int *offsets,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale) {
- enum { QT = 16, KT = 16, HD = 128, DCHUNKS = 8 };
- __shared__ __nv_bfloat16 qsh[QT * HD];
- __shared__ __nv_bfloat16 ksh[KT * HD];
- __shared__ __nv_bfloat16 vsh[KT * HD];
-
- int lane = threadIdx.x & 31;
- int q0 = blockIdx.x * QT;
- int head = blockIdx.y;
- int b = blockIdx.z;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
- if (q0 >= L)
- return;
- size_t hv = (size_t)head * HD;
-
- for (int idx = lane; idx < QT * HD; idx += 32) {
- int qr = idx / HD;
- int d = idx - qr * HD;
- int row = q0 + qr;
- float qv = row < L ? qkv[(size_t)(start + row) * qkv_dim + q_offset + hv + d] : 0.0f;
- qsh[idx] = __float2bfloat16(qv);
- }
- __syncwarp();
-
- float o[DCHUNKS][8];
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++)
-#pragma unroll
- for (int r = 0; r < 8; r++)
- o[dc][r] = 0.0f;
-
- float m[2] = {-3.402823466e+38F, -3.402823466e+38F};
- float denom[2] = {0.0f, 0.0f};
- int key_hi = L;
- if (CAUSAL) {
- int lim = q0 + QT;
- if (lim < key_hi)
- key_hi = lim;
- }
-
- for (int k0 = 0; k0 < key_hi; k0 += KT) {
- for (int idx = lane; idx < KT * HD; idx += 32) {
- int kk = idx / HD;
- int d = idx - kk * HD;
- int key = k0 + kk;
- size_t kb = (size_t)(start + key) * q_dim + hv + d;
- ksh[idx] = key < L ? kexp[kb] : __float2bfloat16(0.0f);
- vsh[idx] = key < L ? vexp[kb] : __float2bfloat16(0.0f);
- }
- __syncwarp();
-
- float s[8];
-#pragma unroll
- for (int r = 0; r < 8; r++)
- s[r] = 0.0f;
-#pragma unroll
- for (int d0 = 0; d0 < HD; d0 += 16) {
- uint32_t a[4], bb[4];
- tc_load_a_q_bf16(a, qsh, d0);
- tc_load_b_qk_bf16(bb, ksh, d0);
- mma_m16n16k16_bf16(s, a, bb);
- }
-
- const float neg_inf = -3.402823466e+38F;
-#pragma unroll
- for (int r = 0; r < 8; r++) {
- int qr = (lane >> 2) + (((r % 4) >> 1) << 3);
- int kc = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
- int qrow = q0 + qr;
- int key = k0 + kc;
- if (qrow < L && key < L && (!CAUSAL || key <= qrow))
- s[r] *= scale;
- else
- s[r] = neg_inf;
- }
-
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int r0 = j * 2;
- float tile_m = fmaxf(fmaxf(s[r0 + 0], s[r0 + 1]), fmaxf(s[r0 + 4], s[r0 + 5]));
- float m_new = fmaxf(m[j], warp_group4_max(tile_m));
- float corr = (m_new == neg_inf) ? 1.0f : __expf(m[j] - m_new);
- denom[j] *= corr;
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++) {
- o[dc][r0 + 0] *= corr;
- o[dc][r0 + 1] *= corr;
- o[dc][r0 + 4] *= corr;
- o[dc][r0 + 5] *= corr;
- }
-
- float part = 0.0f;
- float p0 = (s[r0 + 0] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 0] - m_new);
- float p1 = (s[r0 + 1] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 1] - m_new);
- float p4 = (s[r0 + 4] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 4] - m_new);
- float p5 = (s[r0 + 5] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 5] - m_new);
- s[r0 + 0] = p0;
- s[r0 + 1] = p1;
- s[r0 + 4] = p4;
- s[r0 + 5] = p5;
- part = p0 + p1 + p4 + p5;
- denom[j] += warp_group4_sum(part);
- m[j] = m_new;
- }
-
- uint32_t pfrag[4];
- tc_probs_to_a_bf16(pfrag, s);
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++) {
- uint32_t vb[4];
- tc_load_b_pv_bf16(vb, vsh, dc * 16);
- mma_m16n16k16_bf16(o[dc], pfrag, vb);
- }
- __syncwarp();
- }
-
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int qrow = q0 + (lane >> 2) + j * 8;
- if (qrow >= L)
- continue;
- float inv = denom[j] > 0.0f ? 1.0f / denom[j] : 0.0f;
- size_t ob = (size_t)(start + qrow) * q_dim + hv;
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++) {
- int col = dc * 16 + ((lane & 3) << 1);
- store_act(out, ob + col + 0, o[dc][j * 2 + 0] * inv);
- store_act(out, ob + col + 1, o[dc][j * 2 + 1] * inv);
- store_act(out, ob + col + 8, o[dc][j * 2 + 4] * inv);
- store_act(out, ob + col + 9, o[dc][j * 2 + 5] * inv);
- }
- }
-}
-
-// PV-parallel variant of the BF16 tensor-core tile. Warp 0 computes QK and the
-// online-softmax recurrence once per key tile, writing the 16x16 unnormalized
-// probability tile plus row rescale factors to shared memory. Warps 1..8 each
-// own one 16-column V/O chunk and keep their output accumulator in registers.
-// This preserves the register-resident output state while exposing the P*V MMA
-// work to eight tensor-core warps instead of serializing it in one warp.
-template <int CAUSAL>
-__global__ static void attn_stream_tc128_bf16_pv_kernel(__nv_bfloat16 *out,
- const float *qkv,
- const __nv_bfloat16 *kexp,
- const __nv_bfloat16 *vexp,
- const int *offsets,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale) {
- enum { QT = 16, KT = 16, HD = 128, DCHUNKS = 8, WARPS = 9 };
- __shared__ __nv_bfloat16 qsh[QT * HD];
- __shared__ __nv_bfloat16 ksh[KT * HD];
- __shared__ __nv_bfloat16 vsh[KT * HD];
- __shared__ __nv_bfloat16 psh[QT * KT];
- __shared__ float corr_sh[QT];
- __shared__ float denom_sh[QT];
-
- int lane = threadIdx.x & 31;
- int warp = threadIdx.y;
- int tid = warp * 32 + lane;
- int q0 = blockIdx.x * QT;
- int head = blockIdx.y;
- int b = blockIdx.z;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
- if (q0 >= L)
- return;
- size_t hv = (size_t)head * HD;
-
- for (int idx = tid; idx < QT * HD; idx += WARPS * 32) {
- int qr = idx / HD;
- int d = idx - qr * HD;
- int row = q0 + qr;
- float qv = row < L ? qkv[(size_t)(start + row) * qkv_dim + q_offset + hv + d] : 0.0f;
- qsh[idx] = __float2bfloat16(qv);
- }
- if (tid < QT) {
- corr_sh[tid] = 1.0f;
- denom_sh[tid] = 0.0f;
- }
- __syncthreads();
-
- float m[2] = {-3.402823466e+38F, -3.402823466e+38F};
- float denom[2] = {0.0f, 0.0f};
- float o[8];
-#pragma unroll
- for (int r = 0; r < 8; r++)
- o[r] = 0.0f;
-
- int key_hi = L;
- if (CAUSAL) {
- int lim = q0 + QT;
- if (lim < key_hi)
- key_hi = lim;
- }
-
- for (int k0 = 0; k0 < key_hi; k0 += KT) {
- for (int idx = tid; idx < KT * HD; idx += WARPS * 32) {
- int kk = idx / HD;
- int d = idx - kk * HD;
- int key = k0 + kk;
- size_t kb = (size_t)(start + key) * q_dim + hv + d;
- ksh[idx] = key < L ? kexp[kb] : __float2bfloat16(0.0f);
- vsh[idx] = key < L ? vexp[kb] : __float2bfloat16(0.0f);
- }
- __syncthreads();
-
- if (warp == 0) {
- float s[8];
-#pragma unroll
- for (int r = 0; r < 8; r++)
- s[r] = 0.0f;
-#pragma unroll
- for (int d0 = 0; d0 < HD; d0 += 16) {
- uint32_t a[4], bb[4];
- tc_load_a_q_bf16(a, qsh, d0);
- tc_load_b_qk_bf16(bb, ksh, d0);
- mma_m16n16k16_bf16(s, a, bb);
- }
-
- const float neg_inf = -3.402823466e+38F;
-#pragma unroll
- for (int r = 0; r < 8; r++) {
- int qr = (lane >> 2) + (((r % 4) >> 1) << 3);
- int kc = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
- int qrow = q0 + qr;
- int key = k0 + kc;
- if (qrow < L && key < L && (!CAUSAL || key <= qrow))
- s[r] *= scale;
- else
- s[r] = neg_inf;
- }
-
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int r0 = j * 2;
- int row = (lane >> 2) + j * 8;
- float tile_m = fmaxf(fmaxf(s[r0 + 0], s[r0 + 1]), fmaxf(s[r0 + 4], s[r0 + 5]));
- float m_new = fmaxf(m[j], warp_group4_max(tile_m));
- float corr = (m_new == neg_inf) ? 1.0f : __expf(m[j] - m_new);
- denom[j] *= corr;
-
- float p0 =
- (s[r0 + 0] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 0] - m_new);
- float p1 =
- (s[r0 + 1] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 1] - m_new);
- float p4 =
- (s[r0 + 4] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 4] - m_new);
- float p5 =
- (s[r0 + 5] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 5] - m_new);
- s[r0 + 0] = p0;
- s[r0 + 1] = p1;
- s[r0 + 4] = p4;
- s[r0 + 5] = p5;
- denom[j] += warp_group4_sum(p0 + p1 + p4 + p5);
- m[j] = m_new;
- if ((lane & 3) == 0) {
- corr_sh[row] = corr;
- denom_sh[row] = denom[j];
- }
- }
-
-#pragma unroll
- for (int r = 0; r < 8; r++) {
- int row = (lane >> 2) + (((r % 4) >> 1) << 3);
- int col = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
- psh[row * KT + col] = __float2bfloat16(s[r]);
- }
- }
- __syncthreads();
-
- if (warp > 0) {
- int dc = warp - 1;
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- float corr = corr_sh[(lane >> 2) + j * 8];
- int r0 = j * 2;
- o[r0 + 0] *= corr;
- o[r0 + 1] *= corr;
- o[r0 + 4] *= corr;
- o[r0 + 5] *= corr;
- }
- uint32_t pfrag[4], vb[4];
- tc_load_a_p_bf16(pfrag, psh);
- tc_load_b_pv_bf16(vb, vsh, dc * 16);
- mma_m16n16k16_bf16(o, pfrag, vb);
- }
- __syncthreads();
- }
-
- if (warp > 0) {
- int dc = warp - 1;
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int qrow = q0 + (lane >> 2) + j * 8;
- if (qrow >= L)
- continue;
- float d = denom_sh[(lane >> 2) + j * 8];
- float inv = d > 0.0f ? 1.0f / d : 0.0f;
- size_t ob = (size_t)(start + qrow) * q_dim + hv;
- int col = dc * 16 + ((lane & 3) << 1);
- store_act(out, ob + col + 0, o[j * 2 + 0] * inv);
- store_act(out, ob + col + 1, o[j * 2 + 1] * inv);
- store_act(out, ob + col + 8, o[j * 2 + 4] * inv);
- store_act(out, ob + col + 9, o[j * 2 + 5] * inv);
- }
- }
-}
-
-// K/V-sharing version of the PV-parallel tensor-core tile. Each query tile still
-// uses one QK/softmax warp plus eight P*V warps, but a block owns two query
-// tiles so both groups share the same staged K/V tile. This tests whether the
-// tc2 K/V-traffic win combines with the original tc PV parallelism without
-// changing either tile's math.
-template <int CAUSAL>
-__global__ static void attn_stream_tc128_bf16_pv_kv_kernel(__nv_bfloat16 *out,
- const float *qkv,
- const __nv_bfloat16 *kexp,
- const __nv_bfloat16 *vexp,
- const int *offsets,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale) {
- enum { QT = 16, KT = 16, HD = 128, DCHUNKS = 8, QGROUPS = 2, WARPS_PER_Q = 9, WARPS = 18 };
- __shared__ __nv_bfloat16 qsh[QGROUPS * QT * HD];
- __shared__ __nv_bfloat16 ksh[KT * HD];
- __shared__ __nv_bfloat16 vsh[KT * HD];
- __shared__ __nv_bfloat16 psh[QGROUPS * QT * KT];
- __shared__ float corr_sh[QGROUPS * QT];
- __shared__ float denom_sh[QGROUPS * QT];
-
- int lane = threadIdx.x & 31;
- int warp = threadIdx.y;
- int tid = warp * 32 + lane;
- int qg = warp / WARPS_PER_Q;
- int role = warp - qg * WARPS_PER_Q;
- int block_q0 = blockIdx.x * (QGROUPS * QT);
- int head = blockIdx.y;
- int b = blockIdx.z;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
- if (block_q0 >= L)
- return;
- int q0 = block_q0 + qg * QT;
- size_t hv = (size_t)head * HD;
-
- for (int idx = tid; idx < QGROUPS * QT * HD; idx += WARPS * 32) {
- int g = idx / (QT * HD);
- int rem = idx - g * QT * HD;
- int qr = rem / HD;
- int d = rem - qr * HD;
- int row = block_q0 + g * QT + qr;
- float qv = row < L ? qkv[(size_t)(start + row) * qkv_dim + q_offset + hv + d] : 0.0f;
- qsh[idx] = __float2bfloat16(qv);
- }
- if (tid < QGROUPS * QT) {
- corr_sh[tid] = 1.0f;
- denom_sh[tid] = 0.0f;
- }
- __syncthreads();
-
- float m[2] = {-3.402823466e+38F, -3.402823466e+38F};
- float denom[2] = {0.0f, 0.0f};
- float o[8];
-#pragma unroll
- for (int r = 0; r < 8; r++)
- o[r] = 0.0f;
-
- int key_hi = L;
- if (CAUSAL) {
- int lim = block_q0 + QGROUPS * QT;
- if (lim < key_hi)
- key_hi = lim;
- }
-
- __nv_bfloat16 *myq = qsh + qg * QT * HD;
- __nv_bfloat16 *mypsh = psh + qg * QT * KT;
- float *mycorr = corr_sh + qg * QT;
- float *mydenom = denom_sh + qg * QT;
-
- for (int k0 = 0; k0 < key_hi; k0 += KT) {
- for (int idx = tid; idx < KT * HD; idx += WARPS * 32) {
- int kk = idx / HD;
- int d = idx - kk * HD;
- int key = k0 + kk;
- size_t kb = (size_t)(start + key) * q_dim + hv + d;
- ksh[idx] = key < L ? kexp[kb] : __float2bfloat16(0.0f);
- vsh[idx] = key < L ? vexp[kb] : __float2bfloat16(0.0f);
- }
- __syncthreads();
-
- if (role == 0) {
- float s[8];
-#pragma unroll
- for (int r = 0; r < 8; r++)
- s[r] = 0.0f;
-#pragma unroll
- for (int d0 = 0; d0 < HD; d0 += 16) {
- uint32_t a[4], bb[4];
- tc_load_a_q_bf16(a, myq, d0);
- tc_load_b_qk_bf16(bb, ksh, d0);
- mma_m16n16k16_bf16(s, a, bb);
- }
-
- const float neg_inf = -3.402823466e+38F;
-#pragma unroll
- for (int r = 0; r < 8; r++) {
- int qr = (lane >> 2) + (((r % 4) >> 1) << 3);
- int kc = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
- int qrow = q0 + qr;
- int key = k0 + kc;
- if (qrow < L && key < L && (!CAUSAL || key <= qrow))
- s[r] *= scale;
- else
- s[r] = neg_inf;
- }
-
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int r0 = j * 2;
- int row = (lane >> 2) + j * 8;
- float tile_m = fmaxf(fmaxf(s[r0 + 0], s[r0 + 1]), fmaxf(s[r0 + 4], s[r0 + 5]));
- float m_new = fmaxf(m[j], warp_group4_max(tile_m));
- float corr = (m_new == neg_inf) ? 1.0f : __expf(m[j] - m_new);
- denom[j] *= corr;
-
- float p0 =
- (s[r0 + 0] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 0] - m_new);
- float p1 =
- (s[r0 + 1] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 1] - m_new);
- float p4 =
- (s[r0 + 4] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 4] - m_new);
- float p5 =
- (s[r0 + 5] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 5] - m_new);
- s[r0 + 0] = p0;
- s[r0 + 1] = p1;
- s[r0 + 4] = p4;
- s[r0 + 5] = p5;
- denom[j] += warp_group4_sum(p0 + p1 + p4 + p5);
- m[j] = m_new;
- if ((lane & 3) == 0) {
- mycorr[row] = corr;
- mydenom[row] = denom[j];
- }
- }
-
-#pragma unroll
- for (int r = 0; r < 8; r++) {
- int row = (lane >> 2) + (((r % 4) >> 1) << 3);
- int col = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
- mypsh[row * KT + col] = __float2bfloat16(s[r]);
- }
- }
- __syncthreads();
-
- if (role > 0) {
- int dc = role - 1;
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- float corr = mycorr[(lane >> 2) + j * 8];
- int r0 = j * 2;
- o[r0 + 0] *= corr;
- o[r0 + 1] *= corr;
- o[r0 + 4] *= corr;
- o[r0 + 5] *= corr;
- }
- uint32_t pfrag[4], vb[4];
- tc_load_a_p_bf16(pfrag, mypsh);
- tc_load_b_pv_bf16(vb, vsh, dc * 16);
- mma_m16n16k16_bf16(o, pfrag, vb);
- }
- __syncthreads();
- }
-
- if (role > 0) {
- int dc = role - 1;
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int qrow = q0 + (lane >> 2) + j * 8;
- if (qrow >= L)
- continue;
- float d = mydenom[(lane >> 2) + j * 8];
- float inv = d > 0.0f ? 1.0f / d : 0.0f;
- size_t ob = (size_t)(start + qrow) * q_dim + hv;
- int col = dc * 16 + ((lane & 3) << 1);
- store_act(out, ob + col + 0, o[j * 2 + 0] * inv);
- store_act(out, ob + col + 1, o[j * 2 + 1] * inv);
- store_act(out, ob + col + 8, o[j * 2 + 4] * inv);
- store_act(out, ob + col + 9, o[j * 2 + 5] * inv);
- }
- }
-}
-
-// K/V-sharing variant of the one-warp BF16 tensor-core tile. A block holds WARPS
-// warps; warp w owns the 16-query tile at block_q0 + w*16 of one (sequence,
-// head). All warps share ONE 16-key K/V staging tile in shared memory, so each
-// global K/V tile is read once per WARPS query tiles instead of once per query
-// tile. At long context the materialized and tc/tc1 paths re-read all of K/V for
-// every 16-query tile (O(N^2) K/V traffic); this cuts that by WARPS. Per-warp QK,
-// online softmax, and the 16x128 register output accumulator are unchanged from
-// attn_stream_tc128_bf16_kernel.
-template <int CAUSAL, int WARPS>
-__global__ static void attn_stream_tc128_bf16_kv_kernel(__nv_bfloat16 *out,
- const float *qkv,
- const __nv_bfloat16 *kexp,
- const __nv_bfloat16 *vexp,
- const int *offsets,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale) {
- enum { QT = 16, KT = 16, HD = 128, DCHUNKS = 8 };
- __shared__ __nv_bfloat16 qsh[WARPS * QT * HD];
- __shared__ __nv_bfloat16 ksh[KT * HD];
- __shared__ __nv_bfloat16 vsh[KT * HD];
-
- int lane = threadIdx.x & 31;
- int warp = threadIdx.y;
- int tid = warp * 32 + lane;
- int nthreads = WARPS * 32;
- int block_q0 = blockIdx.x * (WARPS * QT);
- int head = blockIdx.y;
- int b = blockIdx.z;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
- if (block_q0 >= L) // no valid query row in this block (variable-length tail)
- return;
- int q0 = block_q0 + warp * QT; // this warp's query tile
- size_t hv = (size_t)head * HD;
- __nv_bfloat16 *myq = qsh + warp * QT * HD;
-
- for (int idx = lane; idx < QT * HD; idx += 32) {
- int qr = idx / HD;
- int d = idx - qr * HD;
- int row = q0 + qr;
- float qv = row < L ? qkv[(size_t)(start + row) * qkv_dim + q_offset + hv + d] : 0.0f;
- myq[idx] = __float2bfloat16(qv);
- }
-
- float o[DCHUNKS][8];
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++)
-#pragma unroll
- for (int r = 0; r < 8; r++)
- o[dc][r] = 0.0f;
- float m[2] = {-3.402823466e+38F, -3.402823466e+38F};
- float denom[2] = {0.0f, 0.0f};
-
- // Block-level key bound: all warps load the same tiles; per-warp causality is
- // applied in the score mask below. The block's last query row sets the bound.
- int key_hi = L;
- if (CAUSAL) {
- int lim = block_q0 + WARPS * QT;
- if (lim < key_hi)
- key_hi = lim;
- }
-
- for (int k0 = 0; k0 < key_hi; k0 += KT) {
- for (int idx = tid; idx < KT * HD; idx += nthreads) {
- int kk = idx / HD;
- int d = idx - kk * HD;
- int key = k0 + kk;
- size_t kb = (size_t)(start + key) * q_dim + hv + d;
- ksh[idx] = key < L ? kexp[kb] : __float2bfloat16(0.0f);
- vsh[idx] = key < L ? vexp[kb] : __float2bfloat16(0.0f);
- }
- __syncthreads();
-
- float s[8];
-#pragma unroll
- for (int r = 0; r < 8; r++)
- s[r] = 0.0f;
-#pragma unroll
- for (int d0 = 0; d0 < HD; d0 += 16) {
- uint32_t a[4], bb[4];
- tc_load_a_q_bf16(a, myq, d0);
- tc_load_b_qk_bf16(bb, ksh, d0);
- mma_m16n16k16_bf16(s, a, bb);
- }
-
- const float neg_inf = -3.402823466e+38F;
-#pragma unroll
- for (int r = 0; r < 8; r++) {
- int qr = (lane >> 2) + (((r % 4) >> 1) << 3);
- int kc = ((lane & 3) << 1) + ((r >> 2) << 3) + (r & 1);
- int qrow = q0 + qr;
- int key = k0 + kc;
- if (qrow < L && key < L && (!CAUSAL || key <= qrow))
- s[r] *= scale;
- else
- s[r] = neg_inf;
- }
-
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int r0 = j * 2;
- float tile_m = fmaxf(fmaxf(s[r0 + 0], s[r0 + 1]), fmaxf(s[r0 + 4], s[r0 + 5]));
- float m_new = fmaxf(m[j], warp_group4_max(tile_m));
- float corr = (m_new == neg_inf) ? 1.0f : __expf(m[j] - m_new);
- denom[j] *= corr;
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++) {
- o[dc][r0 + 0] *= corr;
- o[dc][r0 + 1] *= corr;
- o[dc][r0 + 4] *= corr;
- o[dc][r0 + 5] *= corr;
- }
- float p0 = (s[r0 + 0] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 0] - m_new);
- float p1 = (s[r0 + 1] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 1] - m_new);
- float p4 = (s[r0 + 4] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 4] - m_new);
- float p5 = (s[r0 + 5] == neg_inf || m_new == neg_inf) ? 0.0f : __expf(s[r0 + 5] - m_new);
- s[r0 + 0] = p0;
- s[r0 + 1] = p1;
- s[r0 + 4] = p4;
- s[r0 + 5] = p5;
- denom[j] += warp_group4_sum(p0 + p1 + p4 + p5);
- m[j] = m_new;
- }
-
- uint32_t pfrag[4];
- tc_probs_to_a_bf16(pfrag, s);
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++) {
- uint32_t vb[4];
- tc_load_b_pv_bf16(vb, vsh, dc * 16);
- mma_m16n16k16_bf16(o[dc], pfrag, vb);
- }
- __syncthreads();
- }
-
-#pragma unroll
- for (int j = 0; j < 2; j++) {
- int qrow = q0 + (lane >> 2) + j * 8;
- if (qrow >= L)
- continue;
- float inv = denom[j] > 0.0f ? 1.0f / denom[j] : 0.0f;
- size_t ob = (size_t)(start + qrow) * q_dim + hv;
-#pragma unroll
- for (int dc = 0; dc < DCHUNKS; dc++) {
- int col = dc * 16 + ((lane & 3) << 1);
- store_act(out, ob + col + 0, o[dc][j * 2 + 0] * inv);
- store_act(out, ob + col + 1, o[dc][j * 2 + 1] * inv);
- store_act(out, ob + col + 8, o[dc][j * 2 + 4] * inv);
- store_act(out, ob + col + 9, o[dc][j * 2 + 5] * inv);
- }
- }
-}
-
-// tc2 with ldmatrix QK fragment assembly. Q keeps the same logical row-major
-// tile but is stored with a grain-level XOR swizzle; K is staged
-// transposed+swizzled so ldmatrix.x4.trans feeds B fragments with the same
-// element order as the scalar tc_load_b_qk_bf16 path. P*V stays on the validated
-// scalar fragment loader; numerical math is otherwise identical to tc2.
+// Qwen BF16 tensor-core flash attention. One block owns WARPS 16-query tiles
+// that share a 16-key K/V staging tile. Q is stored with a grain-level XOR
+// swizzle and K is staged transposed+swizzled so ldmatrix.x4.trans feeds the QK
+// MMA fragments directly; P*V uses the validated BF16 pair loader.
template <int CAUSAL, int WARPS>
__global__ static void attn_stream_tc128_bf16_kv_ldm_kernel(__nv_bfloat16 *out,
const float *qkv,
@@ -2053,137 +1071,6 @@ __global__ static void attn_stream_bert_flash_kernel(__nv_bfloat16 *out,
}
}
-// Register-resident streaming (online-softmax) attention: the FlashAttention-2
-// tile for our packed layout. One warp owns RQ query rows of one (sequence,
-// head); its 32 lanes split head_dim into DPL = HD/32 lane-local elements. The
-// QK dot is then DPL per-lane FMAs plus a single warp_sum (one shuffle
-// butterfly), and the P*V accumulation is lane-local with no reduction. The
-// running max m, denom l, the query rows, and the output accumulator all stay in
-// registers; only the streaming K/V tile passes through shared memory, loaded
-// once per BC keys (not once per key, unlike attn_stream_bert64_kernel). No LxL
-// score matrix is written to global memory; HBM cost is O(N*d), not O(H*N^2).
-// V_T/OUT_T are __nv_bfloat16 when the BF16-weight Qwen path feeds the wo GEMM
-// directly. CAUSAL masks key > query and skips key tiles past the block.
-//
-// Output is bit-exact with the materialized path, but this stays behind
-// FFWD_CUDA_STREAMING_ATTN because cuBLAS attention is faster at F32 for our
-// sizes even at long context: the per-length cost there is dominated by the
-// O(N^2) attention *compute*, which cuBLAS does better than scalar FMA, so
-// removing the score-matrix HBM traffic does not pay off. A streaming win needs
-// tensor-core (fp16/bf16) MMAs, not F32 FMA; this tile is the correct scaffold
-// for that, with the softmax/accumulator state already register-resident.
-template <int HD, int RQ, int BC, typename V_T, typename OUT_T, int CAUSAL>
-__global__ static void attn_stream_reg_kernel(OUT_T *out,
- const float *qkv,
- const float *kexp,
- const V_T *vexp,
- const int *offsets,
- int q_dim,
- int qkv_dim,
- int q_offset,
- float scale) {
- enum { DPL = HD / 32 };
- __shared__ float ksh[BC * HD];
- __shared__ float vsh[BC * HD];
-
- int lane = threadIdx.x;
- int warp = threadIdx.y;
- int nthreads = blockDim.y * 32;
- int tid = warp * 32 + lane;
- int head = blockIdx.y;
- int b = blockIdx.z;
- int start = offsets[b];
- int L = offsets[b + 1] - start;
-
- int block_q0 = blockIdx.x * (blockDim.y * RQ);
- if (block_q0 >= L) // this block owns no valid query row (variable length tail)
- return;
- int warp_row0 = block_q0 + warp * RQ;
- size_t hv = (size_t)head * HD;
-
- // Each warp loads its RQ query rows into registers, pre-scaled by `scale` so
- // the inner product is already the scaled score. Invalid rows keep q=0.
- float q[RQ][DPL], acc[RQ][DPL], m[RQ], l[RQ];
-#pragma unroll
- for (int i = 0; i < RQ; i++) {
- int row = warp_row0 + i;
- size_t qb = (size_t)(start + row) * qkv_dim + q_offset + hv;
-#pragma unroll
- for (int e = 0; e < DPL; e++) {
- q[i][e] = row < L ? qkv[qb + lane + 32 * e] * scale : 0.0f;
- acc[i][e] = 0.0f;
- }
- m[i] = -3.402823466e+38F;
- l[i] = 0.0f;
- }
-
- // Non-causal scans all keys; causal stops at the block's last query row.
- int key_hi = L;
- if (CAUSAL) {
- int lim = block_q0 + (int)blockDim.y * RQ;
- if (lim < key_hi)
- key_hi = lim;
- }
-
- for (int k0 = 0; k0 < key_hi; k0 += BC) {
- // Cooperative tile load: BC keys x HD floats, contiguous writes are
- // conflict-free. Out-of-range keys are zeroed (and masked out below).
- for (int idx = tid; idx < BC * HD; idx += nthreads) {
- int j = idx / HD;
- int key = k0 + j;
- size_t kb = (size_t)(start + key) * q_dim + hv + (idx - j * HD);
- ksh[idx] = key < L ? kexp[kb] : 0.0f;
- vsh[idx] = key < L ? load_attn_value(vexp, kb) : 0.0f;
- }
- __syncthreads();
-
- int tile = key_hi - k0;
- if (tile > BC)
- tile = BC;
- for (int j = 0; j < tile; j++) {
- int key = k0 + j;
- float kv[DPL], vv[DPL];
-#pragma unroll
- for (int e = 0; e < DPL; e++) {
- kv[e] = ksh[j * HD + lane + 32 * e];
- vv[e] = vsh[j * HD + lane + 32 * e];
- }
-#pragma unroll
- for (int i = 0; i < RQ; i++) {
- float part = 0.0f;
-#pragma unroll
- for (int e = 0; e < DPL; e++)
- part += q[i][e] * kv[e];
- float score = warp_sum(part); // full scaled dot, broadcast to all lanes
- int row = warp_row0 + i;
- if (row >= L || (CAUSAL && key > row))
- continue;
- float m_new = fmaxf(m[i], score);
- float corr = __expf(m[i] - m_new);
- float p = __expf(score - m_new);
- l[i] = l[i] * corr + p;
-#pragma unroll
- for (int e = 0; e < DPL; e++)
- acc[i][e] = acc[i][e] * corr + p * vv[e];
- m[i] = m_new;
- }
- }
- __syncthreads();
- }
-
-#pragma unroll
- for (int i = 0; i < RQ; i++) {
- int row = warp_row0 + i;
- if (row >= L)
- continue;
- float inv = 1.0f / l[i];
- size_t ob = (size_t)(start + row) * q_dim + hv;
-#pragma unroll
- for (int e = 0; e < DPL; e++)
- store_act(out, ob + lane + 32 * e, acc[i][e] * inv);
- }
-}
-
__global__ static void
mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
int b = blockIdx.x;
@@ -2398,42 +1285,20 @@ static cublasComputeType_t gemm_compute(void) {
return g_gemm_compute;
}
-// Streaming attention selector. The DEFAULT (no env) runs the tc3 tensor-core
-// flash kernel for the Qwen BF16 HD=128 path (promoted; see use_streaming_attention)
-// and the materialized cuBLAS path for everything else. Overrides via
-// FFWD_CUDA_STREAMING_ATTN: =mat/=off forces the materialized path even for Qwen
-// BF16 (escape hatch for debug / A-B); =tc the PV-parallel BF16 tile, =tc1 the
-// one-warp fallback, =tc2 the K/V-sharing multi-warp tile, =tc3 the tc2
-// ldmatrix/swizzled-load variant (same kernel the default uses), =tc4 the
-// PV-parallel K/V-sharing experiment, =reg the F32 register-resident tile, =bert
-// the scalar BERT tile, =wmma the shared-memory WMMA prototype, =1/=all the
-// generic row kernel for every head_dim<=128 call.
+// Streaming attention selector. The default path runs the promoted tc3
+// tensor-core flash kernel for Qwen BF16 HD=128 and the materialized cuBLAS path
+// for everything else. FFWD_CUDA_STREAMING_ATTN=mat/off/materialized forces the
+// Qwen BF16 path back to materialized for A/B; =tc3/mma_kv_ldm/mma_ldmatrix
+// explicitly selects the same promoted kernel.
static int streaming_attention_mode(void) {
static int init = 0;
- static int mode =
- 0; // -1 force-mat, 0 default, 1 all,2 BERT,3 WMMA,4 reg,5 TC,6 TC1,7 TC2,8 TC3,9 TC4
+ static int mode = 0; // -1 force materialized, 0 default, 8 explicit tc3
if (!init) {
const char *e = getenv("FFWD_CUDA_STREAMING_ATTN");
- if (e && (!strcmp(e, "1") || !strcmp(e, "all")))
- mode = 1;
- else if (e && !strcmp(e, "bert"))
- mode = 2;
- else if (e && (!strcmp(e, "wmma") || !strcmp(e, "bert_wmma")))
- mode = 3;
- else if (e && (!strcmp(e, "reg") || !strcmp(e, "flash")))
- mode = 4;
- else if (e && (!strcmp(e, "tc") || !strcmp(e, "mma") || !strcmp(e, "mma_bf16")))
- mode = 5;
- else if (e && (!strcmp(e, "tc1") || !strcmp(e, "mma1") || !strcmp(e, "mma_bf16_1warp")))
- mode = 6;
- else if (e && (!strcmp(e, "tc2") || !strcmp(e, "mma_kv")))
- mode = 7;
- else if (e && (!strcmp(e, "tc3") || !strcmp(e, "mma_kv_ldm") || !strcmp(e, "mma_ldmatrix")))
+ if (e && (!strcmp(e, "tc3") || !strcmp(e, "mma_kv_ldm") || !strcmp(e, "mma_ldmatrix")))
mode = 8;
- else if (e && (!strcmp(e, "tc4") || !strcmp(e, "tc_pv_kv") || !strcmp(e, "mma_pv_kv")))
- mode = 9;
else if (e && (!strcmp(e, "mat") || !strcmp(e, "off") || !strcmp(e, "materialized")))
- mode = -1; // escape hatch: force the materialized path (debug / A-B vs the tc3 default)
+ mode = -1;
init = 1;
}
return mode;
@@ -2462,36 +1327,15 @@ static int tc_kv_warps_or(int default_w) {
static int use_streaming_attention(const ffwd_config_t *c, int out_bf16, int head_dim) {
int mode = streaming_attention_mode();
- if (head_dim > 128)
- return 0;
if (mode < 0) // FFWD_CUDA_STREAMING_ATTN=mat/off: force the materialized path
return 0;
// Promoted default (no FFWD_CUDA_STREAMING_ATTN set): the Qwen BF16 HD=128
// path runs the tc3 tensor-core flash kernel - faster than materialized BF16
// attention (1.2-2.4x, growing with length) at parity cosine ~0.99997. Every
// other default case (F32, BERT, non-128 head_dim) keeps the materialized
- // path. An explicit mode overrides this below.
- if (!mode)
- return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
- if (mode == 2)
- return c->family == FFWD_FAMILY_BERT && !out_bf16;
- if (mode == 3)
- return (c->family == FFWD_FAMILY_BERT && !out_bf16 && head_dim == 64) ||
- (c->family == FFWD_FAMILY_QWEN3 && head_dim == 128);
- // Register tile handles both shapes: head_dim 64 (BERT/BGE) and 128 (Qwen).
- if (mode == 4)
- return head_dim == 64 || head_dim == 128;
- if (mode == 5)
- return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
- if (mode == 6)
- return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
- if (mode == 7)
- return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
- if (mode == 8)
- return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
- if (mode == 9)
- return c->family == FFWD_FAMILY_QWEN3 && out_bf16 && head_dim == 128;
- return 1;
+ // path. Explicit tc3 uses the same predicate.
+ return (mode == 0 || mode == 8) && c->family == FFWD_FAMILY_QWEN3 && out_bf16 &&
+ head_dim == 128;
}
// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only
@@ -2788,7 +1632,7 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
int stream_mode = streaming_attention_mode();
int use_stream = use_streaming_attention(c, out_bf16 != NULL, hd);
int use_tc_kexp_bf16 = use_stream && out_bf16 && c->family == FFWD_FAMILY_QWEN3 && hd == 128 &&
- (stream_mode == 0 || (stream_mode >= 5 && stream_mode <= 9));
+ (stream_mode == 0 || stream_mode == 8);
// BERT bf16 flash attention: head_dim 32/64, bidirectional, K/V expanded BF16.
int bert_flash = bert_flash_mode();
int use_bert_flash = out_bf16 && c->family == FFWD_FAMILY_BERT &&
@@ -2843,73 +1687,12 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
}
if (use_stream) {
- int max_L = 0;
- for (int b = 0; b < batch; b++) {
- int L = h_offsets[b + 1] - h_offsets[b];
- if (L > max_L)
- max_L = L;
- }
- if (stream_mode == 5 && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && hd == 128) {
- dim3 grid((max_L + 15) / 16, H, batch);
- dim3 block(32, 9);
- if (causal)
- attn_stream_tc128_bf16_pv_kernel<1><<<grid, block, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16,
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- else
- attn_stream_tc128_bf16_pv_kernel<0><<<grid, block, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16,
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- return launch_check();
- }
- if (stream_mode == 6 && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && hd == 128) {
- dim3 grid((max_L + 15) / 16, H, batch);
- if (causal)
- attn_stream_tc128_bf16_kernel<1><<<grid, 32, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16,
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- else
- attn_stream_tc128_bf16_kernel<0><<<grid, 32, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16,
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- return launch_check();
- }
- if (stream_mode == 7 && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && hd == 128) {
- // WARPS query tiles per block share one K/V staging tile, so each
- // global K/V tile is read once per WARPS query tiles (less re-read at
- // long context). WARPS is tunable via FFWD_TC_WARPS (2/4/8); 4 by
- // default. More warps = more reuse but more shared memory / lower
- // occupancy, so the best value is length-dependent.
- int w = tc_kv_warps_or(4);
-#define TC2_LAUNCH(W) \
- do { \
- dim3 grid((max_L + (W) * 16 - 1) / ((W) * 16), H, batch); \
- dim3 block(32, (W)); \
- if (causal) \
- attn_stream_tc128_bf16_kv_kernel<1, (W)><<<grid, block, 0, ctx->stream>>>( \
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16, \
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale); \
- else \
- attn_stream_tc128_bf16_kv_kernel<0, (W)><<<grid, block, 0, ctx->stream>>>( \
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16, \
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale); \
- } while (0)
- if (w == 2)
- TC2_LAUNCH(2);
- else if (w == 8)
- TC2_LAUNCH(8);
- else
- TC2_LAUNCH(4);
-#undef TC2_LAUNCH
- return launch_check();
- }
// stream_mode 0 is the promoted default (use_streaming_attention only
// returns true for mode 0 on this Qwen BF16 HD=128 case); mode 8 is the
// explicit `tc3` override. Both run the same kernel.
if ((stream_mode == 8 || stream_mode == 0) && c->family == FFWD_FAMILY_QWEN3 && out_bf16 &&
hd == 128) {
- // tc2 plus ldmatrix/swizzled QK fragment loads. Uses the same WARPS
- // knob so it can be compared directly with tc2.
+ // Promoted Qwen BF16 tensor-core flash attention.
int w = tc_kv_warps_or(8);
#define TC3_LAUNCH(W) \
do { \
@@ -2933,102 +1716,6 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
#undef TC3_LAUNCH
return launch_check();
}
- if (stream_mode == 9 && c->family == FFWD_FAMILY_QWEN3 && out_bf16 && hd == 128) {
- // Two PV-parallel query tiles per block share one K/V staging tile.
- dim3 grid((max_L + 31) / 32, H, batch);
- dim3 block(32, 18);
- if (causal)
- attn_stream_tc128_bf16_pv_kv_kernel<1><<<grid, block, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16,
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- else
- attn_stream_tc128_bf16_pv_kv_kernel<0><<<grid, block, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, (const __nv_bfloat16 *)ctx->kexp_bf16,
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- return launch_check();
- }
- if (stream_mode == 4) {
- // Register-resident tile: one warp owns RQ query rows, a block owns
- // blockDim.y*RQ = 32 rows. hd=64: 4 warps x RQ=8, BC=16 keys/tile;
- // hd=128: 8 warps x RQ=4, BC=32. Q/m/l/output accumulator in regs.
- dim3 grid((max_L + 31) / 32, H, batch);
- dim3 block(32, hd == 64 ? 4 : 8);
-#define REG_ATTN(HD, RQ, BC, CZ) \
- do { \
- if (out_bf16) \
- attn_stream_reg_kernel<HD, RQ, BC, __nv_bfloat16, __nv_bfloat16, CZ> \
- <<<grid, block, 0, ctx->stream>>>(out_bf16, ctx->qkv, ctx->kexp, \
- (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, \
- q_dim, qkv_dim, q_offset, scale); \
- else \
- attn_stream_reg_kernel<HD, RQ, BC, float, float, CZ> \
- <<<grid, block, 0, ctx->stream>>>(ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp, \
- ctx->offsets, q_dim, qkv_dim, q_offset, scale); \
- } while (0)
- if (hd == 64) {
- if (causal)
- REG_ATTN(64, 8, 16, 1);
- else
- REG_ATTN(64, 8, 16, 0);
- } else {
- if (causal)
- REG_ATTN(128, 4, 32, 1);
- else
- REG_ATTN(128, 4, 32, 0);
- }
-#undef REG_ATTN
- return launch_check();
- }
- if (stream_mode == 3 && c->family == FFWD_FAMILY_BERT && !out_bf16 && !causal && hd == 64) {
- dim3 grid((max_L + 15) / 16, H, batch);
- attn_stream_wmma_kernel<64, float, float, 0>
- <<<grid, 256, 0, ctx->stream>>>(ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp,
- ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- return launch_check();
- }
- if (stream_mode == 3 && hd == 128) {
- dim3 grid((max_L + 15) / 16, H, batch);
- if (out_bf16) {
- if (causal)
- attn_stream_wmma_kernel<128, __nv_bfloat16, __nv_bfloat16, 1>
- <<<grid, 256, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp,
- ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- else
- attn_stream_wmma_kernel<128, __nv_bfloat16, __nv_bfloat16, 0>
- <<<grid, 256, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp,
- ctx->offsets, q_dim, qkv_dim, q_offset, scale);
- } else {
- if (causal)
- attn_stream_wmma_kernel<128, float, float, 1><<<grid, 256, 0, ctx->stream>>>(
- ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp, ctx->offsets, q_dim, qkv_dim,
- q_offset, scale);
- else
- attn_stream_wmma_kernel<128, float, float, 0><<<grid, 256, 0, ctx->stream>>>(
- ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp, ctx->offsets, q_dim, qkv_dim,
- q_offset, scale);
- }
- return launch_check();
- }
- if (stream_mode == 2 && c->family == FFWD_FAMILY_BERT && !out_bf16 && !causal && hd == 64) {
- dim3 grid((max_L + 15) / 16, H, batch);
- attn_stream_bert64_kernel<<<grid, 512, 0, ctx->stream>>>(
- ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset,
- scale);
- return launch_check();
- }
- dim3 grid(max_L, H, batch);
- if (out_bf16) {
- attn_stream_row_kernel<<<grid, 128, 0, ctx->stream>>>(
- out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, hd,
- q_dim, qkv_dim, q_offset, scale, causal);
- } else {
- attn_stream_row_kernel<<<grid, 128, 0, ctx->stream>>>(ctx->attn_out, ctx->qkv, ctx->kexp,
- ctx->vexp, ctx->offsets, hd, q_dim,
- qkv_dim, q_offset, scale, causal);
- }
- return launch_check();
}
if (ctx->attn_G >= 2) {