diff options
| -rw-r--r-- | pplx_embed_cuda.cu | 108 |
1 files changed, 105 insertions, 3 deletions
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu index 3cf8eb5..502b465 100644 --- a/pplx_embed_cuda.cu +++ b/pplx_embed_cuda.cu @@ -49,6 +49,7 @@ struct pplx_cuda_ctx { float *rope_sin; int *token_ids; int *offsets; + int *attn_tiles; int *positions; int *span_starts; int *span_lens; @@ -394,6 +395,9 @@ __global__ static void gqa_attention_kernel(float *out, const float *qkv, out[((size_t)row * n_heads + head) * head_dim + tid] = acc / sum_s; } +// Single-sequence tile-4 attention (batch == 1): the whole packed buffer is one +// sequence [0, total), so bounds are compile-time and no tile descriptor or +// extra host copy is needed. Each block reuses each K/V row across 4 query rows. __global__ static void gqa_attention_tile4_single_kernel( float *out, const float *qkv, int total, int n_heads, int n_kv_heads, int head_dim, int qkv_dim, int q_offset, int k_offset, int v_offset, @@ -447,6 +451,69 @@ __global__ static void gqa_attention_tile4_single_kernel( } } +// Sequence-aware tile-4 attention for micro-batches (batch > 1): each block owns +// up to 4 consecutive query rows within one sequence (tile descriptor = +// (seq, row0)) and reuses each K/V row of that sequence across the 4 query rows. +// block_sum is called for all 4 q before the validity guard so __syncthreads +// stays block-uniform on partial tiles. Assumes head_dim <= 128 (the launch +// block size); shared with gqa_attention_kernel. +__global__ static void gqa_attention_tile4_ragged_kernel( + float *out, const float *qkv, const int *offsets, const int *tiles, + int n_heads, int n_kv_heads, int head_dim, int qkv_dim, + int q_offset, int k_offset, int v_offset, float scale) +{ + int tile = blockIdx.x; + int head = blockIdx.y; + int tid = threadIdx.x; + int seq = tiles[tile * 2 + 0]; + int row0 = tiles[tile * 2 + 1]; + int start = offsets[seq]; + int end = offsets[seq + 1]; + int kv_head = head / (n_heads / n_kv_heads); + + float acc[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + float max_s[4] = { + -3.402823466e+38F, -3.402823466e+38F, + -3.402823466e+38F, -3.402823466e+38F + }; + float sum_s[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + + for (int j = start; j < end; j++) { + const float *kk = qkv + (size_t)j * qkv_dim + + k_offset + kv_head * head_dim; + const float *vv = qkv + (size_t)j * qkv_dim + + v_offset + kv_head * head_dim; + float k = tid < head_dim ? kk[tid] : 0.0f; + float v = tid < head_dim ? vv[tid] : 0.0f; + + for (int q = 0; q < 4; q++) { + int row = row0 + q; + int valid = row < end; + const float *qv = qkv + (size_t)row * qkv_dim + + q_offset + head * head_dim; + float part = valid && tid < head_dim ? qv[tid] * k : 0.0f; + float dot = block_sum(part); + if (valid) { + float score = dot * scale; + float next_max = fmaxf(max_s[q], score); + float old_scale = expf(max_s[q] - next_max); + float weight = expf(score - next_max); + if (tid < head_dim) + acc[q] = acc[q] * old_scale + weight * v; + sum_s[q] = sum_s[q] * old_scale + weight; + max_s[q] = next_max; + } + } + } + + for (int q = 0; q < 4; q++) { + int row = row0 + q; + if (row < end && tid < head_dim) + out[((size_t)row * n_heads + head) * head_dim + tid] = + acc[q] / sum_s[q]; + } +} + __global__ static void mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) @@ -538,10 +605,11 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se cudaFree(ctx->attn_out); cudaFree(ctx->ffn_gate_up); cudaFree(ctx->token_ids); + cudaFree(ctx->attn_tiles); cudaFree(ctx->positions); ctx->x = ctx->x_norm = ctx->qkv = NULL; ctx->attn_out = ctx->ffn_gate_up = NULL; - ctx->token_ids = ctx->positions = NULL; + ctx->token_ids = ctx->attn_tiles = ctx->positions = NULL; CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); @@ -549,6 +617,7 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up, (size_t)total * 2 * c->intermediate_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_tiles, (size_t)total * 2 * sizeof(int))); CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int))); ctx->seq_cap = total; } @@ -715,6 +784,7 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx) cudaFree(ctx->rope_sin); cudaFree(ctx->token_ids); cudaFree(ctx->offsets); + cudaFree(ctx->attn_tiles); cudaFree(ctx->positions); cudaFree(ctx->span_starts); cudaFree(ctx->span_lens); @@ -753,6 +823,28 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, free(h_offsets); free(h_ids); free(h_pos); return -1; } + + // Tile descriptors for the ragged tile-4 attention path. One (seq, row0) + // entry per quad of query rows within each sequence; only built when the + // dispatch below will use it. batch == 1 uses the single-sequence tile + // kernel instead (compile-time bounds, no descriptor or extra host copy). + int use_tiles = batch > 1 && max_seq >= 64; + int n_tiles = 0; + int *h_tiles = NULL; + if (use_tiles) { + h_tiles = (int *)malloc((size_t)total * 2 * sizeof(int)); + if (!h_tiles) { + free(h_offsets); free(h_ids); free(h_pos); + return -1; + } + for (int b = 0; b < batch; b++) { + for (int r = h_offsets[b]; r < h_offsets[b + 1]; r += 4) { + h_tiles[n_tiles * 2 + 0] = b; + h_tiles[n_tiles * 2 + 1] = r; + n_tiles++; + } + } + } for (int b = 0; b < batch; b++) { int off = h_offsets[b]; memcpy(h_ids + off, inputs[b].ids, @@ -775,7 +867,11 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int), cudaMemcpyHostToDevice, ctx->stream); - free(h_offsets); free(h_ids); free(h_pos); + if (ce == cudaSuccess && use_tiles) + ce = cudaMemcpyAsync(ctx->attn_tiles, h_tiles, + (size_t)n_tiles * 2 * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + free(h_offsets); free(h_ids); free(h_pos); free(h_tiles); if (ce != cudaSuccess) { fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce)); return -1; @@ -814,7 +910,13 @@ static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, c->rms_norm_eps); if (launch_check() != 0) return -1; - if (batch == 1 && total >= 64) { + if (use_tiles) { + dim3 grid(n_tiles, c->n_heads); + gqa_attention_tile4_ragged_kernel<<<grid, 128, 0, ctx->stream>>>( + ctx->attn_out, ctx->qkv, ctx->offsets, ctx->attn_tiles, + c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, + q_offset, k_offset, v_offset, scale); + } else if (batch == 1 && total >= 64) { dim3 grid((total + 3) / 4, c->n_heads); gqa_attention_tile4_single_kernel<<<grid, 128, 0, ctx->stream>>>( ctx->attn_out, ctx->qkv, total, c->n_heads, c->n_kv_heads, |
