aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-08 05:14:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit96e5dc95397cd9038212dc69cbe0384b24640dc3 (patch)
treee1eda6b5b3f6b4656a7a667edb98931ef797e163
parent6554530a57a6d73ff58cdf7e601ec3cab824a4c1 (diff)
downloadffwd-96e5dc95397cd9038212dc69cbe0384b24640dc3.tar.gz
ffwd-96e5dc95397cd9038212dc69cbe0384b24640dc3.zip
cuda: add initial CUDA embedding backend
-rw-r--r--Makefile22
-rw-r--r--main.c80
-rw-r--r--pplx_embed.c58
-rw-r--r--pplx_embed_cuda.cu731
-rw-r--r--pplx_embed_cuda.h24
-rw-r--r--pplx_embed_internal.h64
6 files changed, 911 insertions, 68 deletions
diff --git a/Makefile b/Makefile
index 97d138e..4db4490 100644
--- a/Makefile
+++ b/Makefile
@@ -26,12 +26,13 @@ SRCS = pplx_embed.c \
deps/ae/monotonic.c
MLX_SRCS = pplx_embed_mlx.c
+CUDA_SRCS = pplx_embed_cuda.cu
OBJS = $(SRCS:.c=.o)
MLX_OBJS = $(MLX_SRCS:.c=.o)
TARGET = pplx_embed
-.PHONY: all blas mlx debug clean help
+.PHONY: all blas mlx cuda debug clean help
all: help
@@ -41,6 +42,7 @@ help:
@echo "Build targets:"
@echo " make blas Build with BLAS acceleration (CPU)"
@echo " make mlx Build with Apple MLX GPU backend (recommended on Apple Silicon)"
+ @echo " make cuda Build with CUDA/cuBLAS backend (Linux/NVIDIA)"
@echo " make debug Debug build with AddressSanitizer"
@echo " make clean Remove build artifacts"
@echo ""
@@ -90,6 +92,22 @@ mlx:
$(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o"
# =============================================================================
+# CUDA build (Linux NVIDIA GPU - uses CUDA Runtime + cuBLAS)
+# =============================================================================
+CUDA_HOME ?= /usr/local/cuda
+NVCC ?= $(CUDA_HOME)/bin/nvcc
+
+cuda: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include
+cuda: LDFLAGS += -L$(CUDA_HOME)/lib64 -lopenblas -lcudart -lcublas
+cuda:
+ $(MAKE) clean
+ $(MAKE) pplx_embed_cuda.o
+ $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o"
+
+pplx_embed_cuda.o: pplx_embed_cuda.cu pplx_embed_cuda.h pplx_embed_internal.h pplx_embed.h
+ $(NVCC) -O3 -std=c++17 -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $<
+
+# =============================================================================
# Debug build
# =============================================================================
SANITIZE ?= address,undefined
@@ -147,4 +165,4 @@ deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h
# =============================================================================
clean:
- rm -f $(OBJS) pplx_embed_mlx.o main.o $(TARGET)
+ rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o $(TARGET)
diff --git a/main.c b/main.c
index 18afd00..7952538 100644
--- a/main.c
+++ b/main.c
@@ -9,6 +9,9 @@
#ifdef USE_MLX
#include "pplx_embed_mlx.h"
#endif
+#ifdef USE_CUDA
+#include "pplx_embed_cuda.h"
+#endif
#include <stdio.h>
#include <stdlib.h>
@@ -31,6 +34,7 @@ static void print_usage(const char *prog)
"Options:\n"
" -d <dir> Model directory (required)\n"
" --mlx Use Apple MLX GPU backend\n"
+ " --cuda Use CUDA/cuBLAS GPU backend\n"
" --stdin Read lines from stdin, write JSON embeddings to stdout\n"
" --serve Serve the Perplexity-compatible HTTP API\n"
" --dist-worker\n"
@@ -42,7 +46,7 @@ static void print_usage(const char *prog)
" Distributed activation transport width: 32 or 16 (default: 32)\n"
" --model ID=DIR\n"
" Load model ID from directory for --serve (repeatable)\n"
- " --backend cpu|mlx\n"
+ " --backend cpu|mlx|cuda\n"
" Backend for --serve (default: cpu)\n"
" --mlx-quantize-bits N\n"
" Quantize MLX linear weights to 8 bits at load time\n"
@@ -92,6 +96,9 @@ typedef struct {
#ifdef USE_MLX
pplx_mlx_ctx_t *mlx_ctx;
#endif
+#ifdef USE_CUDA
+ pplx_cuda_ctx_t *cuda_ctx;
+#endif
qwen_tokenizer_t *tok;
qwen_tokenizer_workspace_t *tok_ws;
int dim;
@@ -117,10 +124,13 @@ static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs,
#ifdef USE_MLX
if (e->mlx_ctx)
return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings);
- else
#endif
- return pplx_model_embed_batch(e->model, e->workspace,
- inputs, batch, out_embeddings);
+#ifdef USE_CUDA
+ if (e->cuda_ctx)
+ return pplx_cuda_embed_batch(e->cuda_ctx, inputs, batch, out_embeddings);
+#endif
+ return pplx_model_embed_batch(e->model, e->workspace,
+ inputs, batch, out_embeddings);
}
static int parse_layers(const char *arg, int *start, int *end)
@@ -514,6 +524,7 @@ int main(int argc, char *argv[])
int print_embs = 0;
int verbose = 0;
int use_mlx = 0;
+ int use_cuda = 0;
int stdin_mode = 0;
int serve_mode = 0;
int dist_worker_mode = 0;
@@ -568,13 +579,24 @@ int main(int argc, char *argv[])
if (!strcmp(backend, "mlx")) {
#ifdef USE_MLX
use_mlx = 1;
+ use_cuda = 0;
#else
fprintf(stderr, "mlx backend not available (build with: make mlx)\n");
free_model_specs(&model_specs);
return 1;
#endif
+ } else if (!strcmp(backend, "cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "cuda backend not available (build with: make cuda)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
} else if (!strcmp(backend, "cpu")) {
use_mlx = 0;
+ use_cuda = 0;
} else {
fprintf(stderr, "invalid --backend: %s\n", backend);
free_model_specs(&model_specs);
@@ -629,12 +651,23 @@ int main(int argc, char *argv[])
else if (!strcmp(f, "--mlx")) {
#ifdef USE_MLX
use_mlx = 1;
+ use_cuda = 0;
#else
fprintf(stderr, "--mlx not available (build with: make mlx)\n");
free_model_specs(&model_specs);
return 1;
#endif
}
+ else if (!strcmp(f, "--cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "--cuda not available (build with: make cuda)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
print_usage(argv[0]);
free_model_specs(&model_specs);
@@ -664,12 +697,17 @@ int main(int argc, char *argv[])
free_model_specs(&model_specs);
return 1;
}
+ if (use_cuda) {
+ fprintf(stderr, "--backend cuda is not available in --serve mode yet\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
if (mlx_quantize_bits && !use_mlx) {
fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n");
free_model_specs(&model_specs);
return 1;
}
- if (!use_mlx) {
+ if (!use_mlx && !use_cuda) {
if (n_threads <= 0) n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
@@ -726,14 +764,19 @@ int main(int argc, char *argv[])
fprintf(stderr, "--dist-remote coordinator layers must start at zero\n");
return 1;
}
+ if ((dist_worker_mode || dist_remote) && use_cuda) {
+ fprintf(stderr, "distributed mode is not available with --cuda yet\n");
+ return 1;
+ }
/* Threads (CPU backend) */
- if (!use_mlx) {
+ if (!use_mlx && !use_cuda) {
if (n_threads <= 0) n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
} else {
- if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ if (verbose >= 1)
+ fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
}
if (mlx_quantize_bits && !use_mlx) {
@@ -798,6 +841,17 @@ int main(int argc, char *argv[])
e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size;
} else
#endif
+#ifdef USE_CUDA
+ if (use_cuda) {
+ e.cuda_ctx = pplx_cuda_load(model_dir);
+ if (!e.cuda_ctx) {
+ fprintf(stderr, "failed to load CUDA model\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.dim = pplx_cuda_config(e.cuda_ctx)->hidden_size;
+ } else
+#endif
{
if (dist_remote) {
char remote_host[1024];
@@ -835,8 +889,10 @@ int main(int argc, char *argv[])
}
}
if (verbose >= 1)
- fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
- e.dim, now_ms() - t0, use_mlx ? " (MLX)" : "");
+ fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n",
+ e.dim, now_ms() - t0,
+ use_mlx ? " (MLX)" : "",
+ use_cuda ? " (CUDA)" : "");
e.tok_ws = qwen_tokenizer_workspace_new();
if (!e.tok_ws) {
@@ -847,6 +903,9 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
#endif
+#ifdef USE_CUDA
+ if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx);
+#endif
qwen_tokenizer_free(tok);
free_model_specs(&model_specs);
return 1;
@@ -866,6 +925,9 @@ int main(int argc, char *argv[])
#ifdef USE_MLX
if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
#endif
+#ifdef USE_CUDA
+ if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx);
+#endif
qwen_tokenizer_workspace_free(e.tok_ws);
qwen_tokenizer_free(tok);
free_model_specs(&model_specs);
diff --git a/pplx_embed.c b/pplx_embed.c
index 2c0ade6..4dc358e 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -1,4 +1,5 @@
#include "pplx_embed.h"
+#include "pplx_embed_internal.h"
#include "qwen_asr_kernels.h"
#include "qwen_asr_safetensors.h"
@@ -9,63 +10,6 @@
#include <limits.h>
#include <float.h>
-typedef struct {
- safetensor_dtype_t dtype;
- const void *data;
-} pplx_weight_ref_t;
-
-typedef struct {
- pplx_weight_ref_t wq; /* [q_dim, hidden] */
- pplx_weight_ref_t wk; /* [kv_dim, hidden] */
- pplx_weight_ref_t wv; /* [kv_dim, hidden] */
- pplx_weight_ref_t wo; /* [hidden, q_dim] */
- const float *q_norm; /* [head_dim] */
- const float *k_norm; /* [head_dim] */
- const float *input_norm; /* [hidden] */
- const float *post_attn_norm;/* [hidden] */
- pplx_weight_ref_t gate_proj;/* [intermediate, hidden] */
- pplx_weight_ref_t up_proj; /* [intermediate, hidden] */
- pplx_weight_ref_t down_proj;/* [hidden, intermediate] */
-} pplx_layer_t;
-
-typedef struct {
- pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
- pplx_layer_t *layers; /* [n_layers] heap-allocated */
- const float *norm; /* [hidden] */
-} pplx_weights_t;
-
-struct pplx_model {
- pplx_config_t config;
- pplx_weights_t weights;
- void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
- int layer_start; /* loaded transformer range, inclusive */
- int layer_end; /* loaded transformer range, exclusive */
-};
-
-struct pplx_workspace {
- const pplx_model_t *model; /* immutable model this workspace belongs to */
- int buf_seq_cap;
- float *x; /* [seq, hidden] */
- float *x_norm; /* [seq, hidden] */
- float *q; /* [seq, q_dim] */
- float *k; /* [seq, kv_dim] */
- float *v; /* [seq, kv_dim] */
- float *attn_out; /* [seq, q_dim] */
- float *proj_out; /* [seq, hidden] */
- float *ffn_gate; /* [seq, intermediate] */
- float *ffn_up; /* [seq, intermediate] */
- float *attn_scores; /* reusable BLAS attention score tiles */
- size_t attn_scores_bytes;
-
- int *offsets; /* [batch + 1] packed sequence offsets */
- int offsets_cap;
-
- /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */
- float *rope_cos;
- float *rope_sin;
- int rope_cache_cap;
-};
-
struct pplx_late_model {
pplx_model_t *base;
safetensors_file_t *dense_sf;
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu
new file mode 100644
index 0000000..3e677a0
--- /dev/null
+++ b/pplx_embed_cuda.cu
@@ -0,0 +1,731 @@
+extern "C" {
+#include "pplx_embed_cuda.h"
+#include "pplx_embed_internal.h"
+}
+
+#include <cuda_runtime.h>
+#include <cublas_v2.h>
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+ float *d;
+ int rows;
+ int cols;
+} cuda_matrix_t;
+
+typedef struct {
+ cuda_matrix_t wq, wk, wv, wo;
+ float *q_norm;
+ float *k_norm;
+ float *input_norm;
+ float *post_attn_norm;
+ cuda_matrix_t gate_proj, up_proj, down_proj;
+} cuda_layer_t;
+
+struct pplx_cuda_ctx {
+ pplx_model_t *cpu;
+ pplx_config_t config;
+ cudaStream_t stream;
+ cublasHandle_t blas;
+ cuda_matrix_t embed_tokens;
+ cuda_layer_t *layers;
+ float *norm;
+
+ int seq_cap;
+ int batch_cap;
+ int max_seq_cap;
+ float *x;
+ float *x_norm;
+ float *q;
+ float *k;
+ float *v;
+ float *attn_out;
+ float *proj_out;
+ float *ffn_gate;
+ float *ffn_up;
+ float *rope_cos;
+ float *rope_sin;
+ int *token_ids;
+ int *offsets;
+ int *positions;
+};
+
+#define CUDA_CHECK(expr) do { \
+ cudaError_t _e = (expr); \
+ if (_e != cudaSuccess) { \
+ fprintf(stderr, "cuda: %s failed: %s\n", #expr, \
+ cudaGetErrorString(_e)); \
+ return -1; \
+ } \
+} while (0)
+
+#define CUBLAS_CHECK(expr) do { \
+ cublasStatus_t _s = (expr); \
+ if (_s != CUBLAS_STATUS_SUCCESS) { \
+ fprintf(stderr, "cuda: %s failed: cublas status %d\n", \
+ #expr, (int)_s); \
+ return -1; \
+ } \
+} while (0)
+
+static void cuda_matrix_free(cuda_matrix_t *m)
+{
+ if (!m) return;
+ cudaFree(m->d);
+ memset(m, 0, sizeof(*m));
+}
+
+static float bf16_to_f32(uint16_t v)
+{
+ uint32_t u = ((uint32_t)v) << 16;
+ float f;
+ memcpy(&f, &u, sizeof(f));
+ return f;
+}
+
+static int copy_weight_host_f32(float *dst, const pplx_weight_ref_t *w,
+ size_t count)
+{
+ if (!dst || !w || !w->data) return -1;
+ if (w->dtype == DTYPE_F32) {
+ memcpy(dst, w->data, count * sizeof(float));
+ return 0;
+ }
+ if (w->dtype == DTYPE_BF16) {
+ const uint16_t *src = (const uint16_t *)w->data;
+ for (size_t i = 0; i < count; i++)
+ dst[i] = bf16_to_f32(src[i]);
+ return 0;
+ }
+ return -1;
+}
+
+static int load_matrix(cuda_matrix_t *m, const pplx_weight_ref_t *w,
+ int rows, int cols)
+{
+ if (!m || !w || rows <= 0 || cols <= 0) return -1;
+ size_t count = (size_t)rows * (size_t)cols;
+ float *tmp = (float *)malloc(count * sizeof(float));
+ if (!tmp) return -1;
+ if (copy_weight_host_f32(tmp, w, count) != 0) {
+ free(tmp);
+ return -1;
+ }
+ cudaError_t e = cudaMalloc((void **)&m->d, count * sizeof(float));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e));
+ free(tmp);
+ return -1;
+ }
+ e = cudaMemcpy(m->d, tmp, count * sizeof(float), cudaMemcpyHostToDevice);
+ free(tmp);
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e));
+ cuda_matrix_free(m);
+ return -1;
+ }
+ m->rows = rows;
+ m->cols = cols;
+ return 0;
+}
+
+static int load_vector(float **out, const float *src, int n)
+{
+ if (!out || !src || n <= 0) return -1;
+ cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice);
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e));
+ cudaFree(*out);
+ *out = NULL;
+ return -1;
+ }
+ return 0;
+}
+
+__global__ static void embed_lookup_kernel(float *x, const int *ids,
+ const float *emb, int total,
+ int hidden, int vocab)
+{
+ int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ int count = total * hidden;
+ if (idx >= count) return;
+ int tok = idx / hidden;
+ int d = idx - tok * hidden;
+ int id = ids[tok];
+ x[idx] = (id >= 0 && id < vocab) ? emb[(size_t)id * hidden + d] : 0.0f;
+}
+
+__global__ static void rms_norm_kernel(float *out, const float *x,
+ const float *weight, int rows,
+ int dim, float eps)
+{
+ extern __shared__ float sh[];
+ int row = blockIdx.x;
+ int tid = threadIdx.x;
+ float sum = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float v = x[(size_t)row * dim + d];
+ sum += v * v;
+ }
+ sh[tid] = sum;
+ __syncthreads();
+ for (int stride = blockDim.x / 2; stride > 0; stride >>= 1) {
+ if (tid < stride) sh[tid] += sh[tid + stride];
+ __syncthreads();
+ }
+ float inv = rsqrtf(sh[0] / (float)dim + eps);
+ for (int d = tid; d < dim; d += blockDim.x)
+ out[(size_t)row * dim + d] = x[(size_t)row * dim + d] * inv * weight[d];
+}
+
+__global__ static void rms_norm_head_kernel(float *x, const float *weight,
+ int rows, int heads, int head_dim,
+ float eps)
+{
+ __shared__ float sh[128];
+ int row = blockIdx.x;
+ int head = blockIdx.y;
+ int tid = threadIdx.x;
+ size_t base = ((size_t)row * heads + head) * head_dim;
+ float v = tid < head_dim ? x[base + tid] : 0.0f;
+ sh[tid] = v * v;
+ __syncthreads();
+ for (int stride = 64; stride > 0; stride >>= 1) {
+ if (tid < stride) sh[tid] += sh[tid + stride];
+ __syncthreads();
+ }
+ if (tid < head_dim) {
+ float inv = rsqrtf(sh[0] / (float)head_dim + eps);
+ x[base + tid] = v * inv * weight[tid];
+ }
+}
+
+__global__ static void rope_kernel(float *x, const int *positions,
+ const float *cosv, const float *sinv,
+ int total, int heads, int head_dim)
+{
+ int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ int count = total * heads * head_dim;
+ if (idx >= count) return;
+ int d = idx % head_dim;
+ int htmp = idx / head_dim;
+ int tok = htmp / heads;
+ int half = head_dim / 2;
+ int pair_d = d < half ? d + half : d - half;
+ float sign = d < half ? -1.0f : 1.0f;
+ int pos = positions[tok];
+ float c = cosv[(size_t)pos * head_dim + d];
+ float s = sinv[(size_t)pos * head_dim + d];
+ float a = x[idx];
+ float b = x[((size_t)tok * heads + (htmp % heads)) * head_dim + pair_d];
+ x[idx] = a * c + sign * b * s;
+}
+
+__global__ static void add_kernel(float *x, const float *y, int n)
+{
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) x[i] += y[i];
+}
+
+__global__ static void silu_mul_kernel(float *gate, const float *up, int n)
+{
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) {
+ float g = gate[i];
+ gate[i] = (g / (1.0f + expf(-g))) * up[i];
+ }
+}
+
+__device__ static int find_seq_for_token(const int *offsets, int batch, int tok)
+{
+ int lo = 0, hi = batch - 1;
+ while (lo <= hi) {
+ int mid = (lo + hi) >> 1;
+ if (tok < offsets[mid]) hi = mid - 1;
+ else if (tok >= offsets[mid + 1]) lo = mid + 1;
+ else return mid;
+ }
+ return 0;
+}
+
+__global__ static void gqa_attention_kernel(float *out, const float *q,
+ const float *k, const float *v,
+ const int *offsets, int batch,
+ int total, int n_heads,
+ int n_kv_heads, int head_dim,
+ float scale)
+{
+ __shared__ float red[128];
+ __shared__ float score_s;
+ __shared__ float max_s;
+ __shared__ float sum_s;
+
+ int row = blockIdx.x;
+ int head = blockIdx.y;
+ int tid = threadIdx.x;
+ int seq = find_seq_for_token(offsets, batch, row);
+ int start = offsets[seq];
+ int end = offsets[seq + 1];
+ int kv_head = head / (n_heads / n_kv_heads);
+
+ const float *qv = q + ((size_t)row * n_heads + head) * head_dim;
+ float acc = 0.0f;
+
+ if (tid == 0) max_s = -3.402823466e+38F;
+ __syncthreads();
+ for (int j = start; j < end; j++) {
+ const float *kk = k + ((size_t)j * n_kv_heads + kv_head) * head_dim;
+ float part = tid < head_dim ? qv[tid] * kk[tid] : 0.0f;
+ red[tid] = part;
+ __syncthreads();
+ for (int stride = 64; stride > 0; stride >>= 1) {
+ if (tid < stride) red[tid] += red[tid + stride];
+ __syncthreads();
+ }
+ if (tid == 0) {
+ score_s = red[0] * scale;
+ if (score_s > max_s) max_s = score_s;
+ }
+ __syncthreads();
+ }
+
+ if (tid == 0) sum_s = 0.0f;
+ __syncthreads();
+ for (int j = start; j < end; j++) {
+ const float *kk = k + ((size_t)j * n_kv_heads + kv_head) * head_dim;
+ float part = tid < head_dim ? qv[tid] * kk[tid] : 0.0f;
+ red[tid] = part;
+ __syncthreads();
+ for (int stride = 64; stride > 0; stride >>= 1) {
+ if (tid < stride) red[tid] += red[tid + stride];
+ __syncthreads();
+ }
+ if (tid == 0) {
+ score_s = expf(red[0] * scale - max_s);
+ sum_s += score_s;
+ }
+ __syncthreads();
+ if (tid < head_dim) {
+ const float *vv = v + ((size_t)j * n_kv_heads + kv_head) * head_dim;
+ acc += score_s * vv[tid];
+ }
+ __syncthreads();
+ }
+ if (tid < head_dim)
+ out[((size_t)row * n_heads + head) * head_dim + tid] = acc / sum_s;
+}
+
+__global__ static void mean_pool_kernel(float *out, const float *x,
+ const int *offsets, int batch,
+ int hidden)
+{
+ int b = blockIdx.x;
+ int tid = threadIdx.x;
+ int start = offsets[b];
+ int end = offsets[b + 1];
+ int len = end - start;
+ for (int d = tid; d < hidden; d += blockDim.x) {
+ float sum = 0.0f;
+ for (int t = start; t < end; t++)
+ sum += x[(size_t)t * hidden + d];
+ out[(size_t)b * hidden + d] = sum / (float)len;
+ }
+}
+
+static int launch_check(void)
+{
+ cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ return 0;
+}
+
+static int linear(cublasHandle_t blas, const cuda_matrix_t *w,
+ const float *x, float *y, int rows, int in_dim, int out_dim)
+{
+ const float alpha = 1.0f;
+ const float beta = 0.0f;
+ CUBLAS_CHECK(cublasSgemm(blas, CUBLAS_OP_T, CUBLAS_OP_N,
+ out_dim, rows, in_dim,
+ &alpha, w->d, in_dim, x, in_dim,
+ &beta, y, out_dim));
+ return 0;
+}
+
+static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_seq)
+{
+ const pplx_config_t *c = &ctx->config;
+ if (total > ctx->seq_cap) {
+ cudaFree(ctx->x);
+ cudaFree(ctx->x_norm);
+ cudaFree(ctx->q);
+ cudaFree(ctx->k);
+ cudaFree(ctx->v);
+ cudaFree(ctx->attn_out);
+ cudaFree(ctx->proj_out);
+ cudaFree(ctx->ffn_gate);
+ cudaFree(ctx->ffn_up);
+ cudaFree(ctx->token_ids);
+ cudaFree(ctx->positions);
+ ctx->x = ctx->x_norm = ctx->q = ctx->k = ctx->v = NULL;
+ ctx->attn_out = ctx->proj_out = ctx->ffn_gate = ctx->ffn_up = NULL;
+ ctx->token_ids = ctx->positions = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->q, (size_t)total * c->q_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->k, (size_t)total * c->kv_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->v, (size_t)total * c->kv_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->proj_out, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate, (size_t)total * c->intermediate_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_up, (size_t)total * c->intermediate_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int)));
+ ctx->seq_cap = total;
+ }
+ if (batch + 1 > ctx->batch_cap) {
+ cudaFree(ctx->offsets);
+ CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int)));
+ ctx->batch_cap = batch + 1;
+ }
+ if (max_seq > ctx->max_seq_cap) {
+ cudaFree(ctx->rope_cos);
+ cudaFree(ctx->rope_sin);
+ float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
+ float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
+ if (!hcos || !hsin) {
+ free(hcos);
+ free(hsin);
+ return -1;
+ }
+ int half = c->head_dim / 2;
+ for (int pos = 0; pos < max_seq; pos++) {
+ for (int d = 0; d < half; d++) {
+ float inv = 1.0f / powf(c->rope_theta,
+ (float)(2 * d) / (float)c->head_dim);
+ float angle = (float)pos * inv;
+ float cv = cosf(angle);
+ float sv = sinf(angle);
+ hcos[(size_t)pos * c->head_dim + d] = cv;
+ hcos[(size_t)pos * c->head_dim + half + d] = cv;
+ hsin[(size_t)pos * c->head_dim + d] = sv;
+ hsin[(size_t)pos * c->head_dim + half + d] = sv;
+ }
+ }
+ CUDA_CHECK(cudaMalloc((void **)&ctx->rope_cos,
+ (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->rope_sin,
+ (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos,
+ (size_t)max_seq * c->head_dim * sizeof(float),
+ cudaMemcpyHostToDevice, ctx->stream));
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin,
+ (size_t)max_seq * c->head_dim * sizeof(float),
+ cudaMemcpyHostToDevice, ctx->stream));
+ free(hcos);
+ free(hsin);
+ ctx->max_seq_cap = max_seq;
+ }
+ return 0;
+}
+
+static int load_layer(cuda_layer_t *dst, const pplx_layer_t *src,
+ const pplx_config_t *c)
+{
+ return
+ load_matrix(&dst->wq, &src->wq, c->q_dim, c->hidden_size) ||
+ load_matrix(&dst->wk, &src->wk, c->kv_dim, c->hidden_size) ||
+ load_matrix(&dst->wv, &src->wv, c->kv_dim, c->hidden_size) ||
+ load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
+ load_vector(&dst->q_norm, src->q_norm, c->head_dim) ||
+ load_vector(&dst->k_norm, src->k_norm, c->head_dim) ||
+ load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
+ load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
+ load_matrix(&dst->gate_proj, &src->gate_proj,
+ c->intermediate_size, c->hidden_size) ||
+ load_matrix(&dst->up_proj, &src->up_proj,
+ c->intermediate_size, c->hidden_size) ||
+ load_matrix(&dst->down_proj, &src->down_proj,
+ c->hidden_size, c->intermediate_size);
+}
+
+static void free_layer(cuda_layer_t *l)
+{
+ cuda_matrix_free(&l->wq);
+ cuda_matrix_free(&l->wk);
+ cuda_matrix_free(&l->wv);
+ cuda_matrix_free(&l->wo);
+ cudaFree(l->q_norm);
+ cudaFree(l->k_norm);
+ cudaFree(l->input_norm);
+ cudaFree(l->post_attn_norm);
+ cuda_matrix_free(&l->gate_proj);
+ cuda_matrix_free(&l->up_proj);
+ cuda_matrix_free(&l->down_proj);
+}
+
+pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir)
+{
+ pplx_model_t *cpu = pplx_model_load(model_dir);
+ if (!cpu) return NULL;
+
+ pplx_cuda_ctx_t *ctx = (pplx_cuda_ctx_t *)calloc(1, sizeof(*ctx));
+ if (!ctx) {
+ pplx_model_free(cpu);
+ return NULL;
+ }
+ ctx->cpu = cpu;
+ ctx->config = cpu->config;
+
+ if (cudaStreamCreate(&ctx->stream) != cudaSuccess ||
+ cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS ||
+ cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) {
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+
+ const pplx_config_t *c = &ctx->config;
+ ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers));
+ if (!ctx->layers) {
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+ if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens,
+ c->vocab_size, c->hidden_size) != 0 ||
+ load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) {
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+ for (int i = 0; i < c->n_layers; i++) {
+ if (load_layer(&ctx->layers[i], &cpu->weights.layers[i], c) != 0) {
+ fprintf(stderr, "cuda: failed to load layer %d\n", i);
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+ }
+ cudaStreamSynchronize(ctx->stream);
+ return ctx;
+}
+
+void pplx_cuda_free(pplx_cuda_ctx_t *ctx)
+{
+ if (!ctx) return;
+ cuda_matrix_free(&ctx->embed_tokens);
+ if (ctx->layers) {
+ for (int i = 0; i < ctx->config.n_layers; i++)
+ free_layer(&ctx->layers[i]);
+ free(ctx->layers);
+ }
+ cudaFree(ctx->norm);
+ cudaFree(ctx->x);
+ cudaFree(ctx->x_norm);
+ cudaFree(ctx->q);
+ cudaFree(ctx->k);
+ cudaFree(ctx->v);
+ cudaFree(ctx->attn_out);
+ cudaFree(ctx->proj_out);
+ cudaFree(ctx->ffn_gate);
+ cudaFree(ctx->ffn_up);
+ cudaFree(ctx->rope_cos);
+ cudaFree(ctx->rope_sin);
+ cudaFree(ctx->token_ids);
+ cudaFree(ctx->offsets);
+ cudaFree(ctx->positions);
+ if (ctx->blas) cublasDestroy(ctx->blas);
+ if (ctx->stream) cudaStreamDestroy(ctx->stream);
+ pplx_model_free(ctx->cpu);
+ free(ctx);
+}
+
+const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx)
+{
+ return ctx ? &ctx->config : NULL;
+}
+
+int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ const pplx_config_t *c = &ctx->config;
+ int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
+ if (!h_offsets) return -1;
+ int total = 0, max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) {
+ free(h_offsets);
+ return -1;
+ }
+ h_offsets[b] = total;
+ total += inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ }
+ h_offsets[batch] = total;
+ int *h_ids = (int *)malloc((size_t)total * sizeof(int));
+ int *h_pos = (int *)malloc((size_t)total * sizeof(int));
+ if (!h_offsets || !h_ids || !h_pos) {
+ free(h_offsets); free(h_ids); free(h_pos);
+ return -1;
+ }
+ for (int b = 0; b < batch; b++) {
+ int off = h_offsets[b];
+ memcpy(h_ids + off, inputs[b].ids,
+ (size_t)inputs[b].n_tokens * sizeof(int));
+ for (int i = 0; i < inputs[b].n_tokens; i++)
+ h_pos[off + i] = i;
+ }
+ if (ensure_buffers(ctx, total, batch, max_seq) != 0) {
+ free(h_offsets); free(h_ids); free(h_pos);
+ return -1;
+ }
+ cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids,
+ (size_t)total * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->positions, h_pos,
+ (size_t)total * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->offsets, h_offsets,
+ (size_t)(batch + 1) * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ free(h_offsets); free(h_ids); free(h_pos);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce));
+ return -1;
+ }
+
+ int threads = 256;
+ int blocks_hidden = (total * c->hidden_size + threads - 1) / threads;
+ embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->token_ids, ctx->embed_tokens.d, total,
+ c->hidden_size, c->vocab_size);
+ if (launch_check() != 0) return -1;
+
+ float scale = 1.0f / sqrtf((float)c->head_dim);
+ dim3 q_grid;
+ for (int layer = 0; layer < c->n_layers; layer++) {
+ cuda_layer_t *l = &ctx->layers[layer];
+ rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>(
+ ctx->x_norm, ctx->x, l->input_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+
+ if (linear(ctx->blas, &l->wq, ctx->x_norm, ctx->q,
+ total, c->hidden_size, c->q_dim) != 0 ||
+ linear(ctx->blas, &l->wk, ctx->x_norm, ctx->k,
+ total, c->hidden_size, c->kv_dim) != 0 ||
+ linear(ctx->blas, &l->wv, ctx->x_norm, ctx->v,
+ total, c->hidden_size, c->kv_dim) != 0)
+ return -1;
+
+ rms_norm_head_kernel<<<dim3(total, c->n_heads), 128, 0, ctx->stream>>>(
+ ctx->q, l->q_norm, total, c->n_heads, c->head_dim, c->rms_norm_eps);
+ rms_norm_head_kernel<<<dim3(total, c->n_kv_heads), 128, 0, ctx->stream>>>(
+ ctx->k, l->k_norm, total, c->n_kv_heads, c->head_dim, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+
+ q_grid = dim3((total * c->q_dim + threads - 1) / threads);
+ rope_kernel<<<q_grid, threads, 0, ctx->stream>>>(
+ ctx->q, ctx->positions, ctx->rope_cos, ctx->rope_sin,
+ total, c->n_heads, c->head_dim);
+ dim3 k_grid((total * c->kv_dim + threads - 1) / threads);
+ rope_kernel<<<k_grid, threads, 0, ctx->stream>>>(
+ ctx->k, ctx->positions, ctx->rope_cos, ctx->rope_sin,
+ total, c->n_kv_heads, c->head_dim);
+ if (launch_check() != 0) return -1;
+
+ gqa_attention_kernel<<<dim3(total, c->n_heads), 128, 0, ctx->stream>>>(
+ ctx->attn_out, ctx->q, ctx->k, ctx->v, ctx->offsets, batch,
+ total, c->n_heads, c->n_kv_heads, c->head_dim, scale);
+ if (launch_check() != 0) return -1;
+
+ if (linear(ctx->blas, &l->wo, ctx->attn_out, ctx->proj_out,
+ total, c->q_dim, c->hidden_size) != 0)
+ return -1;
+ add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->proj_out, total * c->hidden_size);
+ if (launch_check() != 0) return -1;
+
+ rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>(
+ ctx->x_norm, ctx->x, l->post_attn_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+
+ if (linear(ctx->blas, &l->gate_proj, ctx->x_norm, ctx->ffn_gate,
+ total, c->hidden_size, c->intermediate_size) != 0 ||
+ linear(ctx->blas, &l->up_proj, ctx->x_norm, ctx->ffn_up,
+ total, c->hidden_size, c->intermediate_size) != 0)
+ return -1;
+ int inter_count = total * c->intermediate_size;
+ silu_mul_kernel<<<(inter_count + threads - 1) / threads, threads, 0,
+ ctx->stream>>>(ctx->ffn_gate, ctx->ffn_up, inter_count);
+ if (launch_check() != 0) return -1;
+ if (linear(ctx->blas, &l->down_proj, ctx->ffn_gate, ctx->proj_out,
+ total, c->intermediate_size, c->hidden_size) != 0)
+ return -1;
+ add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->proj_out, total * c->hidden_size);
+ if (launch_check() != 0) return -1;
+ }
+
+ rms_norm_kernel<<<total, 256, 256 * sizeof(float), ctx->stream>>>(
+ ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+
+ float *d_out = NULL;
+ ce = cudaMalloc((void **)&d_out, (size_t)batch * c->hidden_size * sizeof(float));
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda output allocation failed: %s\n",
+ cudaGetErrorString(ce));
+ return -1;
+ }
+ mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
+ d_out, ctx->x, ctx->offsets, batch, c->hidden_size);
+ if (launch_check() != 0) {
+ cudaFree(d_out);
+ return -1;
+ }
+ ce = cudaMemcpyAsync(out_embeddings, d_out,
+ (size_t)batch * c->hidden_size * sizeof(float),
+ cudaMemcpyDeviceToHost, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(ctx->stream);
+ cudaFree(d_out);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce));
+ return -1;
+ }
+ return 0;
+}
+
+int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids,
+ int n_tokens, float *out_embedding)
+{
+ pplx_input_t input = { token_ids, n_tokens };
+ return pplx_cuda_embed_batch(ctx, &input, 1, out_embedding);
+}
+
+float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids,
+ int n_tokens)
+{
+ if (!ctx) return NULL;
+ int hidden = ctx->config.hidden_size;
+ float *out = (float *)malloc((size_t)hidden * sizeof(float));
+ if (!out) return NULL;
+ if (pplx_cuda_embed_into(ctx, token_ids, n_tokens, out) != 0) {
+ free(out);
+ return NULL;
+ }
+ return out;
+}
diff --git a/pplx_embed_cuda.h b/pplx_embed_cuda.h
new file mode 100644
index 0000000..af90230
--- /dev/null
+++ b/pplx_embed_cuda.h
@@ -0,0 +1,24 @@
+/*
+ * pplx_embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference.
+ */
+
+#ifndef PPLX_EMBED_CUDA_H
+#define PPLX_EMBED_CUDA_H
+
+#include "pplx_embed.h"
+
+typedef struct pplx_cuda_ctx pplx_cuda_ctx_t;
+
+pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir);
+void pplx_cuda_free(pplx_cuda_ctx_t *ctx);
+
+const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx);
+
+int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids,
+ int n_tokens, float *out_embedding);
+float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids,
+ int n_tokens);
+int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings);
+
+#endif /* PPLX_EMBED_CUDA_H */
diff --git a/pplx_embed_internal.h b/pplx_embed_internal.h
new file mode 100644
index 0000000..fa028e9
--- /dev/null
+++ b/pplx_embed_internal.h
@@ -0,0 +1,64 @@
+#ifndef PPLX_EMBED_INTERNAL_H
+#define PPLX_EMBED_INTERNAL_H
+
+#include "pplx_embed.h"
+#include "qwen_asr_safetensors.h"
+
+typedef struct {
+ safetensor_dtype_t dtype;
+ const void *data;
+} pplx_weight_ref_t;
+
+typedef struct {
+ pplx_weight_ref_t wq; /* [q_dim, hidden] */
+ pplx_weight_ref_t wk; /* [kv_dim, hidden] */
+ pplx_weight_ref_t wv; /* [kv_dim, hidden] */
+ pplx_weight_ref_t wo; /* [hidden, q_dim] */
+ const float *q_norm; /* [head_dim] */
+ const float *k_norm; /* [head_dim] */
+ const float *input_norm; /* [hidden] */
+ const float *post_attn_norm; /* [hidden] */
+ pplx_weight_ref_t gate_proj; /* [intermediate, hidden] */
+ pplx_weight_ref_t up_proj; /* [intermediate, hidden] */
+ pplx_weight_ref_t down_proj; /* [hidden, intermediate] */
+} pplx_layer_t;
+
+typedef struct {
+ pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
+ pplx_layer_t *layers; /* [n_layers] heap-allocated */
+ const float *norm; /* [hidden] */
+} pplx_weights_t;
+
+struct pplx_model {
+ pplx_config_t config;
+ pplx_weights_t weights;
+ void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
+ int layer_start; /* loaded transformer range, inclusive */
+ int layer_end; /* loaded transformer range, exclusive */
+};
+
+struct pplx_workspace {
+ const pplx_model_t *model; /* immutable model this workspace belongs to */
+ int buf_seq_cap;
+ float *x; /* [seq, hidden] */
+ float *x_norm; /* [seq, hidden] */
+ float *q; /* [seq, q_dim] */
+ float *k; /* [seq, kv_dim] */
+ float *v; /* [seq, kv_dim] */
+ float *attn_out; /* [seq, q_dim] */
+ float *proj_out; /* [seq, hidden] */
+ float *ffn_gate; /* [seq, intermediate] */
+ float *ffn_up; /* [seq, intermediate] */
+ float *attn_scores; /* reusable BLAS attention score tiles */
+ size_t attn_scores_bytes;
+
+ int *offsets; /* [batch + 1] packed sequence offsets */
+ int offsets_cap;
+
+ /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */
+ float *rope_cos;
+ float *rope_sin;
+ int rope_cache_cap;
+};
+
+#endif /* PPLX_EMBED_INTERNAL_H */