aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-08 19:00:38 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitcdb1f4c2ad4d05ebee645761d81daecad067f92a (patch)
tree1f17238208b851ee49f7713f49a7dc278edb79ee
parent294bacb93bcf8a1121e4934c0b238e0d238b1f61 (diff)
downloadffwd-cdb1f4c2ad4d05ebee645761d81daecad067f92a.tar.gz
ffwd-cdb1f4c2ad4d05ebee645761d81daecad067f92a.zip
cuda: extend tiled attention to ragged micro-batches
-rw-r--r--pplx_embed_cuda.cu108
1 files changed, 105 insertions, 3 deletions
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu
index 3cf8eb5..502b465 100644
--- a/pplx_embed_cuda.cu
+++ b/pplx_embed_cuda.cu
@@ -49,6 +49,7 @@ struct pplx_cuda_ctx {
float *rope_sin;
int *token_ids;
int *offsets;
+ int *attn_tiles;
int *positions;
int *span_starts;
int *span_lens;
@@ -394,6 +395,9 @@ __global__ static void gqa_attention_kernel(float *out, const float *qkv,
out[((size_t)row * n_heads + head) * head_dim + tid] = acc / sum_s;
}
+// Single-sequence tile-4 attention (batch == 1): the whole packed buffer is one
+// sequence [0, total), so bounds are compile-time and no tile descriptor or
+// extra host copy is needed. Each block reuses each K/V row across 4 query rows.
__global__ static void gqa_attention_tile4_single_kernel(
float *out, const float *qkv, int total, int n_heads, int n_kv_heads,
int head_dim, int qkv_dim, int q_offset, int k_offset, int v_offset,
@@ -447,6 +451,69 @@ __global__ static void gqa_attention_tile4_single_kernel(
}
}
+// Sequence-aware tile-4 attention for micro-batches (batch > 1): each block owns
+// up to 4 consecutive query rows within one sequence (tile descriptor =
+// (seq, row0)) and reuses each K/V row of that sequence across the 4 query rows.
+// block_sum is called for all 4 q before the validity guard so __syncthreads
+// stays block-uniform on partial tiles. Assumes head_dim <= 128 (the launch
+// block size); shared with gqa_attention_kernel.
+__global__ static void gqa_attention_tile4_ragged_kernel(
+ float *out, const float *qkv, const int *offsets, const int *tiles,
+ int n_heads, int n_kv_heads, int head_dim, int qkv_dim,
+ int q_offset, int k_offset, int v_offset, float scale)
+{
+ int tile = blockIdx.x;
+ int head = blockIdx.y;
+ int tid = threadIdx.x;
+ int seq = tiles[tile * 2 + 0];
+ int row0 = tiles[tile * 2 + 1];
+ int start = offsets[seq];
+ int end = offsets[seq + 1];
+ int kv_head = head / (n_heads / n_kv_heads);
+
+ float acc[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+ float max_s[4] = {
+ -3.402823466e+38F, -3.402823466e+38F,
+ -3.402823466e+38F, -3.402823466e+38F
+ };
+ float sum_s[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+
+ for (int j = start; j < end; j++) {
+ const float *kk = qkv + (size_t)j * qkv_dim +
+ k_offset + kv_head * head_dim;
+ const float *vv = qkv + (size_t)j * qkv_dim +
+ v_offset + kv_head * head_dim;
+ float k = tid < head_dim ? kk[tid] : 0.0f;
+ float v = tid < head_dim ? vv[tid] : 0.0f;
+
+ for (int q = 0; q < 4; q++) {
+ int row = row0 + q;
+ int valid = row < end;
+ const float *qv = qkv + (size_t)row * qkv_dim +
+ q_offset + head * head_dim;
+ float part = valid && tid < head_dim ? qv[tid] * k : 0.0f;
+ float dot = block_sum(part);
+ if (valid) {
+ float score = dot * scale;
+ float next_max = fmaxf(max_s[q], score);
+ float old_scale = expf(max_s[q] - next_max);
+ float weight = expf(score - next_max);
+ if (tid < head_dim)
+ acc[q] = acc[q] * old_scale + weight * v;
+ sum_s[q] = sum_s[q] * old_scale + weight;
+ max_s[q] = next_max;
+ }
+ }
+ }
+
+ for (int q = 0; q < 4; q++) {
+ int row = row0 + q;
+ if (row < end && tid < head_dim)
+ out[((size_t)row * n_heads + head) * head_dim + tid] =
+ acc[q] / sum_s[q];
+ }
+}
+
__global__ static void mean_pool_kernel(float *out, const float *x,
const int *offsets, int batch,
int hidden)
@@ -538,10 +605,11 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se
cudaFree(ctx->attn_out);
cudaFree(ctx->ffn_gate_up);
cudaFree(ctx->token_ids);
+ cudaFree(ctx->attn_tiles);
cudaFree(ctx->positions);
ctx->x = ctx->x_norm = ctx->qkv = NULL;
ctx->attn_out = ctx->ffn_gate_up = NULL;
- ctx->token_ids = ctx->positions = NULL;
+ ctx->token_ids = ctx->attn_tiles = ctx->positions = NULL;
CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float)));
@@ -549,6 +617,7 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se
CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up,
(size_t)total * 2 * c->intermediate_size * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_tiles, (size_t)total * 2 * sizeof(int)));
CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int)));
ctx->seq_cap = total;
}
@@ -715,6 +784,7 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx)
cudaFree(ctx->rope_sin);
cudaFree(ctx->token_ids);
cudaFree(ctx->offsets);
+ cudaFree(ctx->attn_tiles);
cudaFree(ctx->positions);
cudaFree(ctx->span_starts);
cudaFree(ctx->span_lens);
@@ -753,6 +823,28 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
free(h_offsets); free(h_ids); free(h_pos);
return -1;
}
+
+ // Tile descriptors for the ragged tile-4 attention path. One (seq, row0)
+ // entry per quad of query rows within each sequence; only built when the
+ // dispatch below will use it. batch == 1 uses the single-sequence tile
+ // kernel instead (compile-time bounds, no descriptor or extra host copy).
+ int use_tiles = batch > 1 && max_seq >= 64;
+ int n_tiles = 0;
+ int *h_tiles = NULL;
+ if (use_tiles) {
+ h_tiles = (int *)malloc((size_t)total * 2 * sizeof(int));
+ if (!h_tiles) {
+ free(h_offsets); free(h_ids); free(h_pos);
+ return -1;
+ }
+ for (int b = 0; b < batch; b++) {
+ for (int r = h_offsets[b]; r < h_offsets[b + 1]; r += 4) {
+ h_tiles[n_tiles * 2 + 0] = b;
+ h_tiles[n_tiles * 2 + 1] = r;
+ n_tiles++;
+ }
+ }
+ }
for (int b = 0; b < batch; b++) {
int off = h_offsets[b];
memcpy(h_ids + off, inputs[b].ids,
@@ -775,7 +867,11 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
ce = cudaMemcpyAsync(ctx->offsets, h_offsets,
(size_t)(batch + 1) * sizeof(int),
cudaMemcpyHostToDevice, ctx->stream);
- free(h_offsets); free(h_ids); free(h_pos);
+ if (ce == cudaSuccess && use_tiles)
+ ce = cudaMemcpyAsync(ctx->attn_tiles, h_tiles,
+ (size_t)n_tiles * 2 * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ free(h_offsets); free(h_ids); free(h_pos); free(h_tiles);
if (ce != cudaSuccess) {
fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce));
return -1;
@@ -814,7 +910,13 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
c->rms_norm_eps);
if (launch_check() != 0) return -1;
- if (batch == 1 && total >= 64) {
+ if (use_tiles) {
+ dim3 grid(n_tiles, c->n_heads);
+ gqa_attention_tile4_ragged_kernel<<<grid, 128, 0, ctx->stream>>>(
+ ctx->attn_out, ctx->qkv, ctx->offsets, ctx->attn_tiles,
+ c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim,
+ q_offset, k_offset, v_offset, scale);
+ } else if (batch == 1 && total >= 64) {
dim3 grid((total + 3) / 4, c->n_heads);
gqa_attention_tile4_single_kernel<<<grid, 128, 0, ctx->stream>>>(
ctx->attn_out, ctx->qkv, total, c->n_heads, c->n_kv_heads,