aboutsummaryrefslogtreecommitdiff
path: root/libffwd
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-21 18:37:03 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 18:37:03 +0200
commitce3ad004eccf0799e6d3dd9d7378ac7c335477b7 (patch)
tree897ffe65e934240bf041e325771409384cd85743 /libffwd
parent63b7c693a1249512161c917cd6cfc81108160053 (diff)
downloadffwd-ce3ad004eccf0799e6d3dd9d7378ac7c335477b7.tar.gz
ffwd-ce3ad004eccf0799e6d3dd9d7378ac7c335477b7.zip
cuda: add gated streaming attention prototype
Diffstat (limited to 'libffwd')
-rw-r--r--libffwd/cuda.cu383
1 files changed, 383 insertions, 0 deletions
diff --git a/libffwd/cuda.cu b/libffwd/cuda.cu
index 996094b..b030e74 100644
--- a/libffwd/cuda.cu
+++ b/libffwd/cuda.cu
@@ -5,7 +5,9 @@ extern "C" {
#include <cuda_runtime.h>
#include <cuda_bf16.h>
+#include <cuda_fp16.h>
#include <cublas_v2.h>
+#include <mma.h>
#include <limits.h>
#include <math.h>
@@ -397,6 +399,12 @@ __device__ static float block_sum(float v) {
return total;
}
+__device__ static float warp_sum(float v) {
+ for (int offset = 16; offset > 0; offset >>= 1)
+ v += __shfl_down_sync(0xffffffff, v, offset);
+ return __shfl_sync(0xffffffff, v, 0);
+}
+
__device__ static inline void store_act(float *out, size_t i, float v) { out[i] = v; }
__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) {
@@ -407,6 +415,12 @@ __device__ static inline float load_embedding_value(const void *emb, int emb_bf1
return emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[i]) : ((const float *)emb)[i];
}
+__device__ static inline float load_attn_value(const float *v, size_t i) { return v[i]; }
+
+__device__ static inline float load_attn_value(const __nv_bfloat16 *v, size_t i) {
+ return __bfloat162float(v[i]);
+}
+
// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight
// projection GEMM directly (same rounding as a separate cast kernel).
template <typename OUT_T>
@@ -621,6 +635,283 @@ __global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, int
store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f);
}
+// One block computes one (sequence, head, query) row using the online-softmax
+// recurrence. Scores stay in registers and the output accumulator; no LxL score
+// matrix is written to global memory. This first version is intentionally
+// simple and handles the head_dim <= blockDim.x shapes used by BERT/BGE and the
+// current Qwen blocks.
+template <typename V_T, typename OUT_T>
+__global__ static void attn_stream_row_kernel(OUT_T *out,
+ const float *qkv,
+ const float *kexp,
+ const V_T *vexp,
+ const int *offsets,
+ int head_dim,
+ int q_dim,
+ int qkv_dim,
+ int q_offset,
+ float scale,
+ int causal) {
+ int qrow = blockIdx.x;
+ int head = blockIdx.y;
+ int b = blockIdx.z;
+ int start = offsets[b];
+ int L = offsets[b + 1] - start;
+ if (qrow >= L || head_dim > blockDim.x)
+ return;
+
+ int tid = threadIdx.x;
+ size_t q_base = (size_t)(start + qrow) * qkv_dim + q_offset + (size_t)head * head_dim;
+ size_t hv = (size_t)head * head_dim;
+ float qv = tid < head_dim ? qkv[q_base + tid] : 0.0f;
+ float acc = 0.0f;
+ float m = -3.402823466e+38F;
+ float denom = 0.0f;
+ int keys = causal ? qrow + 1 : L;
+
+ for (int j = 0; j < keys; j++) {
+ size_t k_base = (size_t)(start + j) * q_dim + hv;
+ float dot_part = tid < head_dim ? qv * kexp[k_base + tid] : 0.0f;
+ float score = block_sum(dot_part) * scale;
+ float m_new = fmaxf(m, score);
+ float old_scale = __expf(m - m_new);
+ float p = __expf(score - m_new);
+ denom = denom * old_scale + p;
+ if (tid < head_dim) {
+ float vv = load_attn_value(vexp, k_base + tid);
+ acc = acc * old_scale + p * vv;
+ }
+ m = m_new;
+ }
+
+ if (tid < head_dim)
+ store_act(out, (size_t)(start + qrow) * q_dim + hv + tid, acc / denom);
+}
+
+// BERT/BGE non-causal head_dim=64 specialization. One block owns one query
+// tile for one (sequence, head); each warp accumulates one query row while the
+// block loads a K/V row into shared memory once and reuses it across the query
+// warps.
+__global__ static void attn_stream_bert64_kernel(float *out,
+ const float *qkv,
+ const float *kexp,
+ const float *vexp,
+ const int *offsets,
+ int q_dim,
+ int qkv_dim,
+ int q_offset,
+ float scale) {
+ __shared__ float kbuf[64];
+ __shared__ float vbuf[64];
+ int q_per_block = blockDim.x >> 5;
+ int q_tile = blockIdx.x * q_per_block;
+ int head = blockIdx.y;
+ int b = blockIdx.z;
+ int lane = threadIdx.x & 31;
+ int warp = threadIdx.x >> 5;
+ int qrow = q_tile + warp;
+ int start = offsets[b];
+ int L = offsets[b + 1] - start;
+ int active = qrow < L;
+ size_t hv = (size_t)head * 64;
+ size_t q_base = (size_t)(start + qrow) * qkv_dim + q_offset + hv;
+ float q0 = active ? qkv[q_base + lane] : 0.0f;
+ float q1 = active ? qkv[q_base + lane + 32] : 0.0f;
+ float acc0 = 0.0f;
+ float acc1 = 0.0f;
+ float m = -3.402823466e+38F;
+ float denom = 0.0f;
+
+ for (int j = 0; j < L; j++) {
+ size_t kv_base = (size_t)(start + j) * q_dim + hv;
+ if (threadIdx.x < 64) {
+ kbuf[threadIdx.x] = kexp[kv_base + threadIdx.x];
+ vbuf[threadIdx.x] = vexp[kv_base + threadIdx.x];
+ }
+ __syncthreads();
+
+ float dot = q0 * kbuf[lane] + q1 * kbuf[lane + 32];
+ float score = warp_sum(dot) * scale;
+ if (active) {
+ float m_new = fmaxf(m, score);
+ float old_scale = __expf(m - m_new);
+ float p = __expf(score - m_new);
+ denom = denom * old_scale + p;
+ acc0 = acc0 * old_scale + p * vbuf[lane];
+ acc1 = acc1 * old_scale + p * vbuf[lane + 32];
+ m = m_new;
+ }
+ __syncthreads();
+ }
+
+ if (active) {
+ size_t o_base = (size_t)(start + qrow) * q_dim + hv;
+ out[o_base + lane] = acc0 / denom;
+ out[o_base + lane + 32] = acc1 / denom;
+ }
+}
+
+// Tensor-core prototype for 16-row query tiles. It keeps the online-softmax
+// state in shared memory and computes each 16x16 QK/PV tile with WMMA using
+// half-rounded Q/K/P/V operands. The output accumulator stays F32.
+template <int HD, typename V_T, typename OUT_T, int CAUSAL>
+__global__ static void attn_stream_wmma_kernel(OUT_T *out,
+ const float *qkv,
+ const float *kexp,
+ const V_T *vexp,
+ const int *offsets,
+ int q_dim,
+ int qkv_dim,
+ int q_offset,
+ float scale) {
+ enum { QT = 16, KT = 16, DCHUNKS = HD / 16 };
+ __shared__ half qh[QT * HD];
+ __shared__ half kh[HD * KT];
+ __shared__ half ph[QT * KT];
+ __shared__ half vh[KT * HD];
+ __shared__ float scores[QT * KT];
+ __shared__ float acc[QT * HD];
+ __shared__ float delta[DCHUNKS * QT * 16];
+ __shared__ float row_m[QT];
+ __shared__ float row_d[QT];
+ __shared__ float row_old_scale[QT];
+
+ int tid = threadIdx.x;
+ int q_tile = blockIdx.x * QT;
+ int head = blockIdx.y;
+ int b = blockIdx.z;
+ int start = offsets[b];
+ int L = offsets[b + 1] - start;
+ size_t hv = (size_t)head * HD;
+
+ for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
+ int qr = idx / HD;
+ int d = idx - qr * HD;
+ int qrow = q_tile + qr;
+ float qv = qrow < L ? qkv[(size_t)(start + qrow) * qkv_dim + q_offset + hv + d] : 0.0f;
+ qh[idx] = __float2half(qv);
+ acc[idx] = 0.0f;
+ }
+ if (tid < QT) {
+ row_m[tid] = -3.402823466e+38F;
+ row_d[tid] = 0.0f;
+ row_old_scale[tid] = 1.0f;
+ }
+ __syncthreads();
+
+ for (int k0 = 0; k0 < L; k0 += KT) {
+ for (int idx = tid; idx < HD * KT; idx += blockDim.x) {
+ int d = idx / KT;
+ int kk = idx - d * KT;
+ int key = k0 + kk;
+ float kv = key < L ? kexp[(size_t)(start + key) * q_dim + hv + d] : 0.0f;
+ kh[idx] = __float2half(kv);
+ }
+ __syncthreads();
+
+ if (tid < 32) {
+ nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, 16, 16, 16, half, nvcuda::wmma::row_major>
+ a_frag;
+ nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, 16, 16, 16, half, nvcuda::wmma::row_major>
+ b_frag;
+ nvcuda::wmma::fragment<nvcuda::wmma::accumulator, 16, 16, 16, float> c_frag;
+ nvcuda::wmma::fill_fragment(c_frag, 0.0f);
+ for (int kd = 0; kd < HD; kd += 16) {
+ nvcuda::wmma::load_matrix_sync(a_frag, qh + kd, HD);
+ nvcuda::wmma::load_matrix_sync(b_frag, kh + kd * KT, KT);
+ nvcuda::wmma::mma_sync(c_frag, a_frag, b_frag, c_frag);
+ }
+ nvcuda::wmma::store_matrix_sync(scores, c_frag, KT, nvcuda::wmma::mem_row_major);
+ }
+ __syncthreads();
+
+ if (tid < QT) {
+ int qrow = q_tile + tid;
+ if (qrow < L) {
+ float tile_m = -3.402823466e+38F;
+ for (int kk = 0; kk < KT; kk++) {
+ int key = k0 + kk;
+ if (key < L && (!CAUSAL || key <= qrow))
+ tile_m = fmaxf(tile_m, scores[tid * KT + kk] * scale);
+ }
+ float m_new = fmaxf(row_m[tid], tile_m);
+ float old_scale = __expf(row_m[tid] - m_new);
+ float tile_sum = 0.0f;
+ for (int kk = 0; kk < KT; kk++) {
+ int key = k0 + kk;
+ float p = 0.0f;
+ if (key < L && (!CAUSAL || key <= qrow))
+ p = __expf(scores[tid * KT + kk] * scale - m_new);
+ scores[tid * KT + kk] = p;
+ tile_sum += p;
+ }
+ row_m[tid] = m_new;
+ row_d[tid] = row_d[tid] * old_scale + tile_sum;
+ row_old_scale[tid] = old_scale;
+ } else {
+ row_old_scale[tid] = 1.0f;
+ for (int kk = 0; kk < KT; kk++)
+ scores[tid * KT + kk] = 0.0f;
+ }
+ }
+ __syncthreads();
+
+ for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
+ int qr = idx / HD;
+ int qrow = q_tile + qr;
+ if (qrow < L)
+ acc[idx] *= row_old_scale[qr];
+ }
+ for (int idx = tid; idx < QT * KT; idx += blockDim.x)
+ ph[idx] = __float2half(scores[idx]);
+ for (int idx = tid; idx < KT * HD; idx += blockDim.x) {
+ int kk = idx / HD;
+ int d = idx - kk * HD;
+ int key = k0 + kk;
+ float vv = key < L ? load_attn_value(vexp, (size_t)(start + key) * q_dim + hv + d) : 0.0f;
+ vh[idx] = __float2half(vv);
+ }
+ __syncthreads();
+
+ int warp_id = tid >> 5;
+ if (warp_id < DCHUNKS) {
+ int d0 = warp_id * 16;
+ nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, 16, 16, 16, half, nvcuda::wmma::row_major>
+ p_frag;
+ nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, 16, 16, 16, half, nvcuda::wmma::row_major>
+ v_frag;
+ nvcuda::wmma::fragment<nvcuda::wmma::accumulator, 16, 16, 16, float> o_frag;
+ nvcuda::wmma::fill_fragment(o_frag, 0.0f);
+ nvcuda::wmma::load_matrix_sync(p_frag, ph, KT);
+ nvcuda::wmma::load_matrix_sync(v_frag, vh + d0, HD);
+ nvcuda::wmma::mma_sync(o_frag, p_frag, v_frag, o_frag);
+ nvcuda::wmma::store_matrix_sync(delta + warp_id * QT * 16, o_frag, 16,
+ nvcuda::wmma::mem_row_major);
+ }
+ __syncthreads();
+
+ for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
+ int qr = idx / HD;
+ int d = idx - qr * HD;
+ int qrow = q_tile + qr;
+ if (qrow < L) {
+ int chunk = d >> 4;
+ int col = d & 15;
+ acc[idx] += delta[chunk * QT * 16 + qr * 16 + col];
+ }
+ }
+ __syncthreads();
+ }
+
+ for (int idx = tid; idx < QT * HD; idx += blockDim.x) {
+ int qr = idx / HD;
+ int d = idx - qr * HD;
+ int qrow = q_tile + qr;
+ if (qrow < L)
+ store_act(out, (size_t)(start + qrow) * q_dim + hv + d, acc[idx] / row_d[qr]);
+ }
+}
+
__global__ static void
mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
int b = blockIdx.x;
@@ -793,6 +1084,38 @@ static cublasComputeType_t gemm_compute(void) {
return g_gemm_compute;
}
+// Experimental streaming attention selector. Off by default; use
+// FFWD_CUDA_STREAMING_ATTN=bert for the scalar BERT tile, or =wmma for the
+// half-rounded tensor-core prototype. =1/=all forces the generic row kernel for
+// every head_dim<=128 attention call.
+static int streaming_attention_mode(void) {
+ static int init = 0;
+ static int mode = 0; // 0 off, 1 all, 2 BERT scalar tile, 3 WMMA tile
+ if (!init) {
+ const char *e = getenv("FFWD_CUDA_STREAMING_ATTN");
+ if (e && (!strcmp(e, "1") || !strcmp(e, "all")))
+ mode = 1;
+ else if (e && !strcmp(e, "bert"))
+ mode = 2;
+ else if (e && (!strcmp(e, "wmma") || !strcmp(e, "bert_wmma")))
+ mode = 3;
+ init = 1;
+ }
+ return mode;
+}
+
+static int use_streaming_attention(const ffwd_config_t *c, int out_bf16, int head_dim) {
+ int mode = streaming_attention_mode();
+ if (!mode || head_dim > 128)
+ return 0;
+ if (mode == 2)
+ return c->family == FFWD_FAMILY_BERT && !out_bf16;
+ if (mode == 3)
+ return (c->family == FFWD_FAMILY_BERT && !out_bf16 && head_dim == 64) ||
+ (c->family == FFWD_FAMILY_QWEN3 && head_dim == 128);
+ return 1;
+}
+
// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only
// BF16 path materializes one F32 weight at a time into this buffer.
static int ensure_weight_f32(ffwd_cuda_ctx_t *ctx, size_t elems) {
@@ -1062,6 +1385,66 @@ static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
if (launch_check() != 0)
return -1;
+ if (use_streaming_attention(c, out_bf16 != NULL, hd)) {
+ int stream_mode = streaming_attention_mode();
+ int max_L = 0;
+ for (int b = 0; b < batch; b++) {
+ int L = h_offsets[b + 1] - h_offsets[b];
+ if (L > max_L)
+ max_L = L;
+ }
+ if (stream_mode == 3 && c->family == FFWD_FAMILY_BERT && !out_bf16 && !causal && hd == 64) {
+ dim3 grid((max_L + 15) / 16, H, batch);
+ attn_stream_wmma_kernel<64, float, float, 0>
+ <<<grid, 256, 0, ctx->stream>>>(ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp,
+ ctx->offsets, q_dim, qkv_dim, q_offset, scale);
+ return launch_check();
+ }
+ if (stream_mode == 3 && hd == 128) {
+ dim3 grid((max_L + 15) / 16, H, batch);
+ if (out_bf16) {
+ if (causal)
+ attn_stream_wmma_kernel<128, __nv_bfloat16, __nv_bfloat16, 1>
+ <<<grid, 256, 0, ctx->stream>>>(
+ out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp,
+ ctx->offsets, q_dim, qkv_dim, q_offset, scale);
+ else
+ attn_stream_wmma_kernel<128, __nv_bfloat16, __nv_bfloat16, 0>
+ <<<grid, 256, 0, ctx->stream>>>(
+ out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp,
+ ctx->offsets, q_dim, qkv_dim, q_offset, scale);
+ } else {
+ if (causal)
+ attn_stream_wmma_kernel<128, float, float, 1><<<grid, 256, 0, ctx->stream>>>(
+ ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp, ctx->offsets, q_dim, qkv_dim,
+ q_offset, scale);
+ else
+ attn_stream_wmma_kernel<128, float, float, 0><<<grid, 256, 0, ctx->stream>>>(
+ ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp, ctx->offsets, q_dim, qkv_dim,
+ q_offset, scale);
+ }
+ return launch_check();
+ }
+ if (stream_mode == 2 && c->family == FFWD_FAMILY_BERT && !out_bf16 && !causal && hd == 64) {
+ dim3 grid((max_L + 15) / 16, H, batch);
+ attn_stream_bert64_kernel<<<grid, 512, 0, ctx->stream>>>(
+ ctx->attn_out, ctx->qkv, ctx->kexp, ctx->vexp, ctx->offsets, q_dim, qkv_dim, q_offset,
+ scale);
+ return launch_check();
+ }
+ dim3 grid(max_L, H, batch);
+ if (out_bf16) {
+ attn_stream_row_kernel<<<grid, 128, 0, ctx->stream>>>(
+ out_bf16, ctx->qkv, ctx->kexp, (const __nv_bfloat16 *)ctx->vexp, ctx->offsets, hd,
+ q_dim, qkv_dim, q_offset, scale, causal);
+ } else {
+ attn_stream_row_kernel<<<grid, 128, 0, ctx->stream>>>(ctx->attn_out, ctx->qkv, ctx->kexp,
+ ctx->vexp, ctx->offsets, hd, q_dim,
+ qkv_dim, q_offset, scale, causal);
+ }
+ return launch_check();
+ }
+
if (ctx->attn_G >= 2) {
int G = ctx->attn_G, L = ctx->attn_L;
const void *const *K = (const void *const *)ctx->attn_ptrs;