diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-08 05:33:08 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 2c3670e61b43453454bff78a841e9e95919d0e83 (patch) | |
| tree | 959a2bb98fc4c03ff9becdaec7777607e730ae15 | |
| parent | 7e12ee9a33faf7dc697c938f69a8cf51eee86cc6 (diff) | |
| download | ffwd-2c3670e61b43453454bff78a841e9e95919d0e83.tar.gz ffwd-2c3670e61b43453454bff78a841e9e95919d0e83.zip | |
cuda: accumulate residual projections
| -rw-r--r-- | pplx_embed_cuda.cu | 39 |
1 files changed, 18 insertions, 21 deletions
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu index 5160e77..883bbd5 100644 --- a/pplx_embed_cuda.cu +++ b/pplx_embed_cuda.cu @@ -44,7 +44,6 @@ struct pplx_cuda_ctx { float *k; float *v; float *attn_out; - float *proj_out; float *ffn_gate; float *ffn_up; float *rope_cos; @@ -239,12 +238,6 @@ __global__ static void rope_kernel(float *x, const int *positions, x[idx] = a * c + sign * b * s; } -__global__ static void add_kernel(float *x, const float *y, int n) -{ - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i < n) x[i] += y[i]; -} - __global__ static void silu_mul_kernel(float *gate, const float *up, int n) { int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -366,6 +359,19 @@ static int linear(cublasHandle_t blas, const cuda_matrix_t *w, return 0; } +static int linear_accum(cublasHandle_t blas, const cuda_matrix_t *w, + const float *x, float *y, + int rows, int in_dim, int out_dim) +{ + const float alpha = 1.0f; + const float beta = 1.0f; + CUBLAS_CHECK(cublasSgemm(blas, CUBLAS_OP_T, CUBLAS_OP_N, + out_dim, rows, in_dim, + &alpha, w->d, in_dim, x, in_dim, + &beta, y, out_dim)); + return 0; +} + static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_seq) { const pplx_config_t *c = &ctx->config; @@ -376,13 +382,12 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se cudaFree(ctx->k); cudaFree(ctx->v); cudaFree(ctx->attn_out); - cudaFree(ctx->proj_out); cudaFree(ctx->ffn_gate); cudaFree(ctx->ffn_up); cudaFree(ctx->token_ids); cudaFree(ctx->positions); ctx->x = ctx->x_norm = ctx->q = ctx->k = ctx->v = NULL; - ctx->attn_out = ctx->proj_out = ctx->ffn_gate = ctx->ffn_up = NULL; + ctx->attn_out = ctx->ffn_gate = ctx->ffn_up = NULL; ctx->token_ids = ctx->positions = NULL; CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); @@ -390,7 +395,6 @@ static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_se CUDA_CHECK(cudaMalloc((void **)&ctx->k, (size_t)total * c->kv_dim * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->v, (size_t)total * c->kv_dim * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->proj_out, (size_t)total * c->hidden_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate, (size_t)total * c->intermediate_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_up, (size_t)total * c->intermediate_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int))); @@ -537,7 +541,6 @@ void pplx_cuda_free(pplx_cuda_ctx_t *ctx) cudaFree(ctx->k); cudaFree(ctx->v); cudaFree(ctx->attn_out); - cudaFree(ctx->proj_out); cudaFree(ctx->ffn_gate); cudaFree(ctx->ffn_up); cudaFree(ctx->rope_cos); @@ -653,12 +656,9 @@ int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, total, c->n_heads, c->n_kv_heads, c->head_dim, scale); if (launch_check() != 0) return -1; - if (linear(ctx->blas, &l->wo, ctx->attn_out, ctx->proj_out, - total, c->q_dim, c->hidden_size) != 0) + if (linear_accum(ctx->blas, &l->wo, ctx->attn_out, ctx->x, + total, c->q_dim, c->hidden_size) != 0) return -1; - add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( - ctx->x, ctx->proj_out, total * c->hidden_size); - if (launch_check() != 0) return -1; rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>( ctx->x_norm, ctx->x, l->post_attn_norm, total, @@ -674,12 +674,9 @@ int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, silu_mul_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(ctx->ffn_gate, ctx->ffn_up, inter_count); if (launch_check() != 0) return -1; - if (linear(ctx->blas, &l->down_proj, ctx->ffn_gate, ctx->proj_out, - total, c->intermediate_size, c->hidden_size) != 0) + if (linear_accum(ctx->blas, &l->down_proj, ctx->ffn_gate, ctx->x, + total, c->intermediate_size, c->hidden_size) != 0) return -1; - add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( - ctx->x, ctx->proj_out, total * c->hidden_size); - if (launch_check() != 0) return -1; } rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>( |
