diff options
| -rw-r--r-- | Makefile | 22 | ||||
| -rw-r--r-- | main.c | 80 | ||||
| -rw-r--r-- | pplx_embed.c | 58 | ||||
| -rw-r--r-- | pplx_embed_cuda.cu | 731 | ||||
| -rw-r--r-- | pplx_embed_cuda.h | 24 | ||||
| -rw-r--r-- | pplx_embed_internal.h | 64 |
6 files changed, 911 insertions, 68 deletions
@@ -26,12 +26,13 @@ SRCS = pplx_embed.c \ deps/ae/monotonic.c MLX_SRCS = pplx_embed_mlx.c +CUDA_SRCS = pplx_embed_cuda.cu OBJS = $(SRCS:.c=.o) MLX_OBJS = $(MLX_SRCS:.c=.o) TARGET = pplx_embed -.PHONY: all blas mlx debug clean help +.PHONY: all blas mlx cuda debug clean help all: help @@ -41,6 +42,7 @@ help: @echo "Build targets:" @echo " make blas Build with BLAS acceleration (CPU)" @echo " make mlx Build with Apple MLX GPU backend (recommended on Apple Silicon)" + @echo " make cuda Build with CUDA/cuBLAS backend (Linux/NVIDIA)" @echo " make debug Debug build with AddressSanitizer" @echo " make clean Remove build artifacts" @echo "" @@ -90,6 +92,22 @@ mlx: $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o" # ============================================================================= +# CUDA build (Linux NVIDIA GPU - uses CUDA Runtime + cuBLAS) +# ============================================================================= +CUDA_HOME ?= /usr/local/cuda +NVCC ?= $(CUDA_HOME)/bin/nvcc + +cuda: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include +cuda: LDFLAGS += -L$(CUDA_HOME)/lib64 -lopenblas -lcudart -lcublas +cuda: + $(MAKE) clean + $(MAKE) pplx_embed_cuda.o + $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o" + +pplx_embed_cuda.o: pplx_embed_cuda.cu pplx_embed_cuda.h pplx_embed_internal.h pplx_embed.h + $(NVCC) -O3 -std=c++17 -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $< + +# ============================================================================= # Debug build # ============================================================================= SANITIZE ?= address,undefined @@ -147,4 +165,4 @@ deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h # ============================================================================= clean: - rm -f $(OBJS) pplx_embed_mlx.o main.o $(TARGET) + rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o $(TARGET) @@ -9,6 +9,9 @@ #ifdef USE_MLX #include "pplx_embed_mlx.h" #endif +#ifdef USE_CUDA +#include "pplx_embed_cuda.h" +#endif #include <stdio.h> #include <stdlib.h> @@ -31,6 +34,7 @@ static void print_usage(const char *prog) "Options:\n" " -d <dir> Model directory (required)\n" " --mlx Use Apple MLX GPU backend\n" + " --cuda Use CUDA/cuBLAS GPU backend\n" " --stdin Read lines from stdin, write JSON embeddings to stdout\n" " --serve Serve the Perplexity-compatible HTTP API\n" " --dist-worker\n" @@ -42,7 +46,7 @@ static void print_usage(const char *prog) " Distributed activation transport width: 32 or 16 (default: 32)\n" " --model ID=DIR\n" " Load model ID from directory for --serve (repeatable)\n" - " --backend cpu|mlx\n" + " --backend cpu|mlx|cuda\n" " Backend for --serve (default: cpu)\n" " --mlx-quantize-bits N\n" " Quantize MLX linear weights to 8 bits at load time\n" @@ -92,6 +96,9 @@ typedef struct { #ifdef USE_MLX pplx_mlx_ctx_t *mlx_ctx; #endif +#ifdef USE_CUDA + pplx_cuda_ctx_t *cuda_ctx; +#endif qwen_tokenizer_t *tok; qwen_tokenizer_workspace_t *tok_ws; int dim; @@ -117,10 +124,13 @@ static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, #ifdef USE_MLX if (e->mlx_ctx) return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings); - else #endif - return pplx_model_embed_batch(e->model, e->workspace, - inputs, batch, out_embeddings); +#ifdef USE_CUDA + if (e->cuda_ctx) + return pplx_cuda_embed_batch(e->cuda_ctx, inputs, batch, out_embeddings); +#endif + return pplx_model_embed_batch(e->model, e->workspace, + inputs, batch, out_embeddings); } static int parse_layers(const char *arg, int *start, int *end) @@ -514,6 +524,7 @@ int main(int argc, char *argv[]) int print_embs = 0; int verbose = 0; int use_mlx = 0; + int use_cuda = 0; int stdin_mode = 0; int serve_mode = 0; int dist_worker_mode = 0; @@ -568,13 +579,24 @@ int main(int argc, char *argv[]) if (!strcmp(backend, "mlx")) { #ifdef USE_MLX use_mlx = 1; + use_cuda = 0; #else fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); free_model_specs(&model_specs); return 1; #endif + } else if (!strcmp(backend, "cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); + free_model_specs(&model_specs); + return 1; +#endif } else if (!strcmp(backend, "cpu")) { use_mlx = 0; + use_cuda = 0; } else { fprintf(stderr, "invalid --backend: %s\n", backend); free_model_specs(&model_specs); @@ -629,12 +651,23 @@ int main(int argc, char *argv[]) else if (!strcmp(f, "--mlx")) { #ifdef USE_MLX use_mlx = 1; + use_cuda = 0; #else fprintf(stderr, "--mlx not available (build with: make mlx)\n"); free_model_specs(&model_specs); return 1; #endif } + else if (!strcmp(f, "--cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "--cuda not available (build with: make cuda)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(argv[0]); free_model_specs(&model_specs); @@ -664,12 +697,17 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } + if (use_cuda) { + fprintf(stderr, "--backend cuda is not available in --serve mode yet\n"); + free_model_specs(&model_specs); + return 1; + } if (mlx_quantize_bits && !use_mlx) { fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n"); free_model_specs(&model_specs); return 1; } - if (!use_mlx) { + if (!use_mlx && !use_cuda) { if (n_threads <= 0) n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); @@ -726,14 +764,19 @@ int main(int argc, char *argv[]) fprintf(stderr, "--dist-remote coordinator layers must start at zero\n"); return 1; } + if ((dist_worker_mode || dist_remote) && use_cuda) { + fprintf(stderr, "distributed mode is not available with --cuda yet\n"); + return 1; + } /* Threads (CPU backend) */ - if (!use_mlx) { + if (!use_mlx && !use_cuda) { if (n_threads <= 0) n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } else { - if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); + if (verbose >= 1) + fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); } if (mlx_quantize_bits && !use_mlx) { @@ -798,6 +841,17 @@ int main(int argc, char *argv[]) e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size; } else #endif +#ifdef USE_CUDA + if (use_cuda) { + e.cuda_ctx = pplx_cuda_load(model_dir); + if (!e.cuda_ctx) { + fprintf(stderr, "failed to load CUDA model\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.dim = pplx_cuda_config(e.cuda_ctx)->hidden_size; + } else +#endif { if (dist_remote) { char remote_host[1024]; @@ -835,8 +889,10 @@ int main(int argc, char *argv[]) } } if (verbose >= 1) - fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", - e.dim, now_ms() - t0, use_mlx ? " (MLX)" : ""); + fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n", + e.dim, now_ms() - t0, + use_mlx ? " (MLX)" : "", + use_cuda ? " (CUDA)" : ""); e.tok_ws = qwen_tokenizer_workspace_new(); if (!e.tok_ws) { @@ -847,6 +903,9 @@ int main(int argc, char *argv[]) #ifdef USE_MLX if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); #endif +#ifdef USE_CUDA + if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); +#endif qwen_tokenizer_free(tok); free_model_specs(&model_specs); return 1; @@ -866,6 +925,9 @@ int main(int argc, char *argv[]) #ifdef USE_MLX if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); #endif +#ifdef USE_CUDA + if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); +#endif qwen_tokenizer_workspace_free(e.tok_ws); qwen_tokenizer_free(tok); free_model_specs(&model_specs); diff --git a/pplx_embed.c b/pplx_embed.c index 2c0ade6..4dc358e 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -1,4 +1,5 @@ #include "pplx_embed.h" +#include "pplx_embed_internal.h" #include "qwen_asr_kernels.h" #include "qwen_asr_safetensors.h" @@ -9,63 +10,6 @@ #include <limits.h> #include <float.h> -typedef struct { - safetensor_dtype_t dtype; - const void *data; -} pplx_weight_ref_t; - -typedef struct { - pplx_weight_ref_t wq; /* [q_dim, hidden] */ - pplx_weight_ref_t wk; /* [kv_dim, hidden] */ - pplx_weight_ref_t wv; /* [kv_dim, hidden] */ - pplx_weight_ref_t wo; /* [hidden, q_dim] */ - const float *q_norm; /* [head_dim] */ - const float *k_norm; /* [head_dim] */ - const float *input_norm; /* [hidden] */ - const float *post_attn_norm;/* [hidden] */ - pplx_weight_ref_t gate_proj;/* [intermediate, hidden] */ - pplx_weight_ref_t up_proj; /* [intermediate, hidden] */ - pplx_weight_ref_t down_proj;/* [hidden, intermediate] */ -} pplx_layer_t; - -typedef struct { - pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ - pplx_layer_t *layers; /* [n_layers] heap-allocated */ - const float *norm; /* [hidden] */ -} pplx_weights_t; - -struct pplx_model { - pplx_config_t config; - pplx_weights_t weights; - void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ - int layer_start; /* loaded transformer range, inclusive */ - int layer_end; /* loaded transformer range, exclusive */ -}; - -struct pplx_workspace { - const pplx_model_t *model; /* immutable model this workspace belongs to */ - int buf_seq_cap; - float *x; /* [seq, hidden] */ - float *x_norm; /* [seq, hidden] */ - float *q; /* [seq, q_dim] */ - float *k; /* [seq, kv_dim] */ - float *v; /* [seq, kv_dim] */ - float *attn_out; /* [seq, q_dim] */ - float *proj_out; /* [seq, hidden] */ - float *ffn_gate; /* [seq, intermediate] */ - float *ffn_up; /* [seq, intermediate] */ - float *attn_scores; /* reusable BLAS attention score tiles */ - size_t attn_scores_bytes; - - int *offsets; /* [batch + 1] packed sequence offsets */ - int offsets_cap; - - /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */ - float *rope_cos; - float *rope_sin; - int rope_cache_cap; -}; - struct pplx_late_model { pplx_model_t *base; safetensors_file_t *dense_sf; diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu new file mode 100644 index 0000000..3e677a0 --- /dev/null +++ b/pplx_embed_cuda.cu @@ -0,0 +1,731 @@ +extern "C" { +#include "pplx_embed_cuda.h" +#include "pplx_embed_internal.h" +} + +#include <cuda_runtime.h> +#include <cublas_v2.h> + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +typedef struct { + float *d; + int rows; + int cols; +} cuda_matrix_t; + +typedef struct { + cuda_matrix_t wq, wk, wv, wo; + float *q_norm; + float *k_norm; + float *input_norm; + float *post_attn_norm; + cuda_matrix_t gate_proj, up_proj, down_proj; +} cuda_layer_t; + +struct pplx_cuda_ctx { + pplx_model_t *cpu; + pplx_config_t config; + cudaStream_t stream; + cublasHandle_t blas; + cuda_matrix_t embed_tokens; + cuda_layer_t *layers; + float *norm; + + int seq_cap; + int batch_cap; + int max_seq_cap; + float *x; + float *x_norm; + float *q; + float *k; + float *v; + float *attn_out; + float *proj_out; + float *ffn_gate; + float *ffn_up; + float *rope_cos; + float *rope_sin; + int *token_ids; + int *offsets; + int *positions; +}; + +#define CUDA_CHECK(expr) do { \ + cudaError_t _e = (expr); \ + if (_e != cudaSuccess) { \ + fprintf(stderr, "cuda: %s failed: %s\n", #expr, \ + cudaGetErrorString(_e)); \ + return -1; \ + } \ +} while (0) + +#define CUBLAS_CHECK(expr) do { \ + cublasStatus_t _s = (expr); \ + if (_s != CUBLAS_STATUS_SUCCESS) { \ + fprintf(stderr, "cuda: %s failed: cublas status %d\n", \ + #expr, (int)_s); \ + return -1; \ + } \ +} while (0) + +static void cuda_matrix_free(cuda_matrix_t *m) +{ + if (!m) return; + cudaFree(m->d); + memset(m, 0, sizeof(*m)); +} + +static float bf16_to_f32(uint16_t v) +{ + uint32_t u = ((uint32_t)v) << 16; + float f; + memcpy(&f, &u, sizeof(f)); + return f; +} + +static int copy_weight_host_f32(float *dst, const pplx_weight_ref_t *w, + size_t count) +{ + if (!dst || !w || !w->data) return -1; + if (w->dtype == DTYPE_F32) { + memcpy(dst, w->data, count * sizeof(float)); + return 0; + } + if (w->dtype == DTYPE_BF16) { + const uint16_t *src = (const uint16_t *)w->data; + for (size_t i = 0; i < count; i++) + dst[i] = bf16_to_f32(src[i]); + return 0; + } + return -1; +} + +static int load_matrix(cuda_matrix_t *m, const pplx_weight_ref_t *w, + int rows, int cols) +{ + if (!m || !w || rows <= 0 || cols <= 0) return -1; + size_t count = (size_t)rows * (size_t)cols; + float *tmp = (float *)malloc(count * sizeof(float)); + if (!tmp) return -1; + if (copy_weight_host_f32(tmp, w, count) != 0) { + free(tmp); + return -1; + } + cudaError_t e = cudaMalloc((void **)&m->d, count * sizeof(float)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e)); + free(tmp); + return -1; + } + e = cudaMemcpy(m->d, tmp, count * sizeof(float), cudaMemcpyHostToDevice); + free(tmp); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e)); + cuda_matrix_free(m); + return -1; + } + m->rows = rows; + m->cols = cols; + return 0; +} + +static int load_vector(float **out, const float *src, int n) +{ + if (!out || !src || n <= 0) return -1; + cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e)); + return -1; + } + e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e)); + cudaFree(*out); + *out = NULL; + return -1; + } + return 0; +} + +__global__ static void embed_lookup_kernel(float *x, const int *ids, + const float *emb, int total, + int hidden, int vocab) +{ + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int count = total * hidden; + if (idx >= count) return; + int tok = idx / hidden; + int d = idx - tok * hidden; + int id = ids[tok]; + x[idx] = (id >= 0 && id < vocab) ? emb[(size_t)id * hidden + d] : 0.0f; +} + +__global__ static void rms_norm_kernel(float *out, const float *x, + const float *weight, int rows, + int dim, float eps) +{ + extern __shared__ float sh[]; + int row = blockIdx.x; + int tid = threadIdx.x; + float sum = 0.0f; + for (int d = tid; d < dim; d += blockDim.x) { + float v = x[(size_t)row * dim + d]; + sum += v * v; + } + sh[tid] = sum; + __syncthreads(); + for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) { + if (tid < stride) sh[tid] += sh[tid + stride]; + __syncthreads(); + } + float inv = rsqrtf(sh[0] / (float)dim + eps); + for (int d = tid; d < dim; d += blockDim.x) + out[(size_t)row * dim + d] = x[(size_t)row * dim + d] * inv * weight[d]; +} + +__global__ static void rms_norm_head_kernel(float *x, const float *weight, + int rows, int heads, int head_dim, + float eps) +{ + __shared__ float sh[128]; + int row = blockIdx.x; + int head = blockIdx.y; + int tid = threadIdx.x; + size_t base = ((size_t)row * heads + head) * head_dim; + float v = tid < head_dim ? x[base + tid] : 0.0f; + sh[tid] = v * v; + __syncthreads(); + for (int stride = 64; stride > 0; stride >>= 1) { + if (tid < stride) sh[tid] += sh[tid + stride]; + __syncthreads(); + } + if (tid < head_dim) { + float inv = rsqrtf(sh[0] / (float)head_dim + eps); + x[base + tid] = v * inv * weight[tid]; + } +} + +__global__ static void rope_kernel(float *x, const int *positions, + const float *cosv, const float *sinv, + int total, int heads, int head_dim) +{ + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int count = total * heads * head_dim; + if (idx >= count) return; + int d = idx % head_dim; + int htmp = idx / head_dim; + int tok = htmp / heads; + int half = head_dim / 2; + int pair_d = d < half ? d + half : d - half; + float sign = d < half ? -1.0f : 1.0f; + int pos = positions[tok]; + float c = cosv[(size_t)pos * head_dim + d]; + float s = sinv[(size_t)pos * head_dim + d]; + float a = x[idx]; + float b = x[((size_t)tok * heads + (htmp % heads)) * head_dim + pair_d]; + x[idx] = a * c + sign * b * s; +} + +__global__ static void add_kernel(float *x, const float *y, int n) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) x[i] += y[i]; +} + +__global__ static void silu_mul_kernel(float *gate, const float *up, int n) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) { + float g = gate[i]; + gate[i] = (g / (1.0f + expf(-g))) * up[i]; + } +} + +__device__ static int find_seq_for_token(const int *offsets, int batch, int tok) +{ + int lo = 0, hi = batch - 1; + while (lo <= hi) { + int mid = (lo + hi) >> 1; + if (tok < offsets[mid]) hi = mid - 1; + else if (tok >= offsets[mid + 1]) lo = mid + 1; + else return mid; + } + return 0; +} + +__global__ static void gqa_attention_kernel(float *out, const float *q, + const float *k, const float *v, + const int *offsets, int batch, + int total, int n_heads, + int n_kv_heads, int head_dim, + float scale) +{ + __shared__ float red[128]; + __shared__ float score_s; + __shared__ float max_s; + __shared__ float sum_s; + + int row = blockIdx.x; + int head = blockIdx.y; + int tid = threadIdx.x; + int seq = find_seq_for_token(offsets, batch, row); + int start = offsets[seq]; + int end = offsets[seq + 1]; + int kv_head = head / (n_heads / n_kv_heads); + + const float *qv = q + ((size_t)row * n_heads + head) * head_dim; + float acc = 0.0f; + + if (tid == 0) max_s = -3.402823466e+38F; + __syncthreads(); + for (int j = start; j < end; j++) { + const float *kk = k + ((size_t)j * n_kv_heads + kv_head) * head_dim; + float part = tid < head_dim ? qv[tid] * kk[tid] : 0.0f; + red[tid] = part; + __syncthreads(); + for (int stride = 64; stride > 0; stride >>= 1) { + if (tid < stride) red[tid] += red[tid + stride]; + __syncthreads(); + } + if (tid == 0) { + score_s = red[0] * scale; + if (score_s > max_s) max_s = score_s; + } + __syncthreads(); + } + + if (tid == 0) sum_s = 0.0f; + __syncthreads(); + for (int j = start; j < end; j++) { + const float *kk = k + ((size_t)j * n_kv_heads + kv_head) * head_dim; + float part = tid < head_dim ? qv[tid] * kk[tid] : 0.0f; + red[tid] = part; + __syncthreads(); + for (int stride = 64; stride > 0; stride >>= 1) { + if (tid < stride) red[tid] += red[tid + stride]; + __syncthreads(); + } + if (tid == 0) { + score_s = expf(red[0] * scale - max_s); + sum_s += score_s; + } + __syncthreads(); + if (tid < head_dim) { + const float *vv = v + ((size_t)j * n_kv_heads + kv_head) * head_dim; + acc += score_s * vv[tid]; + } + __syncthreads(); + } + if (tid < head_dim) + out[((size_t)row * n_heads + head) * head_dim + tid] = acc / sum_s; +} + +__global__ static void mean_pool_kernel(float *out, const float *x, + const int *offsets, int batch, + int hidden) +{ + int b = blockIdx.x; + int tid = threadIdx.x; + int start = offsets[b]; + int end = offsets[b + 1]; + int len = end - start; + for (int d = tid; d < hidden; d += blockDim.x) { + float sum = 0.0f; + for (int t = start; t < end; t++) + sum += x[(size_t)t * hidden + d]; + out[(size_t)b * hidden + d] = sum / (float)len; + } +} + +static int launch_check(void) +{ + cudaError_t e = cudaGetLastError(); + if (e != cudaSuccess) { + fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e)); + return -1; + } + return 0; +} + +static int linear(cublasHandle_t blas, const cuda_matrix_t *w, + const float *x, float *y, int rows, int in_dim, int out_dim) +{ + const float alpha = 1.0f; + const float beta = 0.0f; + CUBLAS_CHECK(cublasSgemm(blas, CUBLAS_OP_T, CUBLAS_OP_N, + out_dim, rows, in_dim, + &alpha, w->d, in_dim, x, in_dim, + &beta, y, out_dim)); + return 0; +} + +static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_seq) +{ + const pplx_config_t *c = &ctx->config; + if (total > ctx->seq_cap) { + cudaFree(ctx->x); + cudaFree(ctx->x_norm); + cudaFree(ctx->q); + cudaFree(ctx->k); + cudaFree(ctx->v); + cudaFree(ctx->attn_out); + cudaFree(ctx->proj_out); + cudaFree(ctx->ffn_gate); + cudaFree(ctx->ffn_up); + cudaFree(ctx->token_ids); + cudaFree(ctx->positions); + ctx->x = ctx->x_norm = ctx->q = ctx->k = ctx->v = NULL; + ctx->attn_out = ctx->proj_out = ctx->ffn_gate = ctx->ffn_up = NULL; + ctx->token_ids = ctx->positions = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->q, (size_t)total * c->q_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->k, (size_t)total * c->kv_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->v, (size_t)total * c->kv_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->proj_out, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate, (size_t)total * c->intermediate_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_up, (size_t)total * c->intermediate_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int))); + ctx->seq_cap = total; + } + if (batch + 1 > ctx->batch_cap) { + cudaFree(ctx->offsets); + CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int))); + ctx->batch_cap = batch + 1; + } + if (max_seq > ctx->max_seq_cap) { + cudaFree(ctx->rope_cos); + cudaFree(ctx->rope_sin); + float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); + float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); + if (!hcos || !hsin) { + free(hcos); + free(hsin); + return -1; + } + int half = c->head_dim / 2; + for (int pos = 0; pos < max_seq; pos++) { + for (int d = 0; d < half; d++) { + float inv = 1.0f / powf(c->rope_theta, + (float)(2 * d) / (float)c->head_dim); + float angle = (float)pos * inv; + float cv = cosf(angle); + float sv = sinf(angle); + hcos[(size_t)pos * c->head_dim + d] = cv; + hcos[(size_t)pos * c->head_dim + half + d] = cv; + hsin[(size_t)pos * c->head_dim + d] = sv; + hsin[(size_t)pos * c->head_dim + half + d] = sv; + } + } + CUDA_CHECK(cudaMalloc((void **)&ctx->rope_cos, + (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->rope_sin, + (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, + (size_t)max_seq * c->head_dim * sizeof(float), + cudaMemcpyHostToDevice, ctx->stream)); + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin, + (size_t)max_seq * c->head_dim * sizeof(float), + cudaMemcpyHostToDevice, ctx->stream)); + free(hcos); + free(hsin); + ctx->max_seq_cap = max_seq; + } + return 0; +} + +static int load_layer(cuda_layer_t *dst, const pplx_layer_t *src, + const pplx_config_t *c) +{ + return + load_matrix(&dst->wq, &src->wq, c->q_dim, c->hidden_size) || + load_matrix(&dst->wk, &src->wk, c->kv_dim, c->hidden_size) || + load_matrix(&dst->wv, &src->wv, c->kv_dim, c->hidden_size) || + load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || + load_vector(&dst->q_norm, src->q_norm, c->head_dim) || + load_vector(&dst->k_norm, src->k_norm, c->head_dim) || + load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || + load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || + load_matrix(&dst->gate_proj, &src->gate_proj, + c->intermediate_size, c->hidden_size) || + load_matrix(&dst->up_proj, &src->up_proj, + c->intermediate_size, c->hidden_size) || + load_matrix(&dst->down_proj, &src->down_proj, + c->hidden_size, c->intermediate_size); +} + +static void free_layer(cuda_layer_t *l) +{ + cuda_matrix_free(&l->wq); + cuda_matrix_free(&l->wk); + cuda_matrix_free(&l->wv); + cuda_matrix_free(&l->wo); + cudaFree(l->q_norm); + cudaFree(l->k_norm); + cudaFree(l->input_norm); + cudaFree(l->post_attn_norm); + cuda_matrix_free(&l->gate_proj); + cuda_matrix_free(&l->up_proj); + cuda_matrix_free(&l->down_proj); +} + +pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir) +{ + pplx_model_t *cpu = pplx_model_load(model_dir); + if (!cpu) return NULL; + + pplx_cuda_ctx_t *ctx = (pplx_cuda_ctx_t *)calloc(1, sizeof(*ctx)); + if (!ctx) { + pplx_model_free(cpu); + return NULL; + } + ctx->cpu = cpu; + ctx->config = cpu->config; + + if (cudaStreamCreate(&ctx->stream) != cudaSuccess || + cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS || + cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) { + pplx_cuda_free(ctx); + return NULL; + } + + const pplx_config_t *c = &ctx->config; + ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers)); + if (!ctx->layers) { + pplx_cuda_free(ctx); + return NULL; + } + if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, + c->vocab_size, c->hidden_size) != 0 || + load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) { + pplx_cuda_free(ctx); + return NULL; + } + for (int i = 0; i < c->n_layers; i++) { + if (load_layer(&ctx->layers[i], &cpu->weights.layers[i], c) != 0) { + fprintf(stderr, "cuda: failed to load layer %d\n", i); + pplx_cuda_free(ctx); + return NULL; + } + } + cudaStreamSynchronize(ctx->stream); + return ctx; +} + +void pplx_cuda_free(pplx_cuda_ctx_t *ctx) +{ + if (!ctx) return; + cuda_matrix_free(&ctx->embed_tokens); + if (ctx->layers) { + for (int i = 0; i < ctx->config.n_layers; i++) + free_layer(&ctx->layers[i]); + free(ctx->layers); + } + cudaFree(ctx->norm); + cudaFree(ctx->x); + cudaFree(ctx->x_norm); + cudaFree(ctx->q); + cudaFree(ctx->k); + cudaFree(ctx->v); + cudaFree(ctx->attn_out); + cudaFree(ctx->proj_out); + cudaFree(ctx->ffn_gate); + cudaFree(ctx->ffn_up); + cudaFree(ctx->rope_cos); + cudaFree(ctx->rope_sin); + cudaFree(ctx->token_ids); + cudaFree(ctx->offsets); + cudaFree(ctx->positions); + if (ctx->blas) cublasDestroy(ctx->blas); + if (ctx->stream) cudaStreamDestroy(ctx->stream); + pplx_model_free(ctx->cpu); + free(ctx); +} + +const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx) +{ + return ctx ? &ctx->config : NULL; +} + +int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + const pplx_config_t *c = &ctx->config; + int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); + if (!h_offsets) return -1; + int total = 0, max_seq = 0; + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) { + free(h_offsets); + return -1; + } + h_offsets[b] = total; + total += inputs[b].n_tokens; + if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + } + h_offsets[batch] = total; + int *h_ids = (int *)malloc((size_t)total * sizeof(int)); + int *h_pos = (int *)malloc((size_t)total * sizeof(int)); + if (!h_offsets || !h_ids || !h_pos) { + free(h_offsets); free(h_ids); free(h_pos); + return -1; + } + for (int b = 0; b < batch; b++) { + int off = h_offsets[b]; + memcpy(h_ids + off, inputs[b].ids, + (size_t)inputs[b].n_tokens * sizeof(int)); + for (int i = 0; i < inputs[b].n_tokens; i++) + h_pos[off + i] = i; + } + if (ensure_buffers(ctx, total, batch, max_seq) != 0) { + free(h_offsets); free(h_ids); free(h_pos); + return -1; + } + cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, + (size_t)total * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->positions, h_pos, + (size_t)total * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->offsets, h_offsets, + (size_t)(batch + 1) * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + free(h_offsets); free(h_ids); free(h_pos); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce)); + return -1; + } + + int threads = 256; + int blocks_hidden = (total * c->hidden_size + threads - 1) / threads; + embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->token_ids, ctx->embed_tokens.d, total, + c->hidden_size, c->vocab_size); + if (launch_check() != 0) return -1; + + float scale = 1.0f / sqrtf((float)c->head_dim); + dim3 q_grid; + for (int layer = 0; layer < c->n_layers; layer++) { + cuda_layer_t *l = &ctx->layers[layer]; + rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>( + ctx->x_norm, ctx->x, l->input_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + + if (linear(ctx->blas, &l->wq, ctx->x_norm, ctx->q, + total, c->hidden_size, c->q_dim) != 0 || + linear(ctx->blas, &l->wk, ctx->x_norm, ctx->k, + total, c->hidden_size, c->kv_dim) != 0 || + linear(ctx->blas, &l->wv, ctx->x_norm, ctx->v, + total, c->hidden_size, c->kv_dim) != 0) + return -1; + + rms_norm_head_kernel<<<dim3(total, c->n_heads), 128, 0, ctx->stream>>>( + ctx->q, l->q_norm, total, c->n_heads, c->head_dim, c->rms_norm_eps); + rms_norm_head_kernel<<<dim3(total, c->n_kv_heads), 128, 0, ctx->stream>>>( + ctx->k, l->k_norm, total, c->n_kv_heads, c->head_dim, c->rms_norm_eps); + if (launch_check() != 0) return -1; + + q_grid = dim3((total * c->q_dim + threads - 1) / threads); + rope_kernel<<<q_grid, threads, 0, ctx->stream>>>( + ctx->q, ctx->positions, ctx->rope_cos, ctx->rope_sin, + total, c->n_heads, c->head_dim); + dim3 k_grid((total * c->kv_dim + threads - 1) / threads); + rope_kernel<<<k_grid, threads, 0, ctx->stream>>>( + ctx->k, ctx->positions, ctx->rope_cos, ctx->rope_sin, + total, c->n_kv_heads, c->head_dim); + if (launch_check() != 0) return -1; + + gqa_attention_kernel<<<dim3(total, c->n_heads), 128, 0, ctx->stream>>>( + ctx->attn_out, ctx->q, ctx->k, ctx->v, ctx->offsets, batch, + total, c->n_heads, c->n_kv_heads, c->head_dim, scale); + if (launch_check() != 0) return -1; + + if (linear(ctx->blas, &l->wo, ctx->attn_out, ctx->proj_out, + total, c->q_dim, c->hidden_size) != 0) + return -1; + add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->proj_out, total * c->hidden_size); + if (launch_check() != 0) return -1; + + rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>( + ctx->x_norm, ctx->x, l->post_attn_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + + if (linear(ctx->blas, &l->gate_proj, ctx->x_norm, ctx->ffn_gate, + total, c->hidden_size, c->intermediate_size) != 0 || + linear(ctx->blas, &l->up_proj, ctx->x_norm, ctx->ffn_up, + total, c->hidden_size, c->intermediate_size) != 0) + return -1; + int inter_count = total * c->intermediate_size; + silu_mul_kernel<<<(inter_count + threads - 1) / threads, threads, 0, + ctx->stream>>>(ctx->ffn_gate, ctx->ffn_up, inter_count); + if (launch_check() != 0) return -1; + if (linear(ctx->blas, &l->down_proj, ctx->ffn_gate, ctx->proj_out, + total, c->intermediate_size, c->hidden_size) != 0) + return -1; + add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->proj_out, total * c->hidden_size); + if (launch_check() != 0) return -1; + } + + rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>( + ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + + float *d_out = NULL; + ce = cudaMalloc((void **)&d_out, (size_t)batch * c->hidden_size * sizeof(float)); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda output allocation failed: %s\n", + cudaGetErrorString(ce)); + return -1; + } + mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>( + d_out, ctx->x, ctx->offsets, batch, c->hidden_size); + if (launch_check() != 0) { + cudaFree(d_out); + return -1; + } + ce = cudaMemcpyAsync(out_embeddings, d_out, + (size_t)batch * c->hidden_size * sizeof(float), + cudaMemcpyDeviceToHost, ctx->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(ctx->stream); + cudaFree(d_out); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce)); + return -1; + } + return 0; +} + +int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding) +{ + pplx_input_t input = { token_ids, n_tokens }; + return pplx_cuda_embed_batch(ctx, &input, 1, out_embedding); +} + +float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens) +{ + if (!ctx) return NULL; + int hidden = ctx->config.hidden_size; + float *out = (float *)malloc((size_t)hidden * sizeof(float)); + if (!out) return NULL; + if (pplx_cuda_embed_into(ctx, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; + } + return out; +} diff --git a/pplx_embed_cuda.h b/pplx_embed_cuda.h new file mode 100644 index 0000000..af90230 --- /dev/null +++ b/pplx_embed_cuda.h @@ -0,0 +1,24 @@ +/* + * pplx_embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference. + */ + +#ifndef PPLX_EMBED_CUDA_H +#define PPLX_EMBED_CUDA_H + +#include "pplx_embed.h" + +typedef struct pplx_cuda_ctx pplx_cuda_ctx_t; + +pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir); +void pplx_cuda_free(pplx_cuda_ctx_t *ctx); + +const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx); + +int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding); +float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens); +int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings); + +#endif /* PPLX_EMBED_CUDA_H */ diff --git a/pplx_embed_internal.h b/pplx_embed_internal.h new file mode 100644 index 0000000..fa028e9 --- /dev/null +++ b/pplx_embed_internal.h @@ -0,0 +1,64 @@ +#ifndef PPLX_EMBED_INTERNAL_H +#define PPLX_EMBED_INTERNAL_H + +#include "pplx_embed.h" +#include "qwen_asr_safetensors.h" + +typedef struct { + safetensor_dtype_t dtype; + const void *data; +} pplx_weight_ref_t; + +typedef struct { + pplx_weight_ref_t wq; /* [q_dim, hidden] */ + pplx_weight_ref_t wk; /* [kv_dim, hidden] */ + pplx_weight_ref_t wv; /* [kv_dim, hidden] */ + pplx_weight_ref_t wo; /* [hidden, q_dim] */ + const float *q_norm; /* [head_dim] */ + const float *k_norm; /* [head_dim] */ + const float *input_norm; /* [hidden] */ + const float *post_attn_norm; /* [hidden] */ + pplx_weight_ref_t gate_proj; /* [intermediate, hidden] */ + pplx_weight_ref_t up_proj; /* [intermediate, hidden] */ + pplx_weight_ref_t down_proj; /* [hidden, intermediate] */ +} pplx_layer_t; + +typedef struct { + pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ + pplx_layer_t *layers; /* [n_layers] heap-allocated */ + const float *norm; /* [hidden] */ +} pplx_weights_t; + +struct pplx_model { + pplx_config_t config; + pplx_weights_t weights; + void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ +}; + +struct pplx_workspace { + const pplx_model_t *model; /* immutable model this workspace belongs to */ + int buf_seq_cap; + float *x; /* [seq, hidden] */ + float *x_norm; /* [seq, hidden] */ + float *q; /* [seq, q_dim] */ + float *k; /* [seq, kv_dim] */ + float *v; /* [seq, kv_dim] */ + float *attn_out; /* [seq, q_dim] */ + float *proj_out; /* [seq, hidden] */ + float *ffn_gate; /* [seq, intermediate] */ + float *ffn_up; /* [seq, intermediate] */ + float *attn_scores; /* reusable BLAS attention score tiles */ + size_t attn_scores_bytes; + + int *offsets; /* [batch + 1] packed sequence offsets */ + int offsets_cap; + + /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */ + float *rope_cos; + float *rope_sin; + int rope_cache_cap; +}; + +#endif /* PPLX_EMBED_INTERNAL_H */ |
