aboutsummaryrefslogtreecommitdiff
path: root/embed_cuda.cu
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 08:24:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit2c37727e653d2340d8977dd5db4addc767dd81ea (patch)
tree578ad6affac511f847a346a0efc6d40ebf6a279f /embed_cuda.cu
parentec417bcb52f2aa8e24aeec312e8ddbf3a0798056 (diff)
downloadffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.tar.gz
ffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.zip
gitignore: track new binary and library names
Diffstat (limited to 'embed_cuda.cu')
-rw-r--r--embed_cuda.cu1204
1 files changed, 1204 insertions, 0 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu
new file mode 100644
index 0000000..a00fc84
--- /dev/null
+++ b/embed_cuda.cu
@@ -0,0 +1,1204 @@
+extern "C" {
+#include "pplx_embed_cuda.h"
+#include "pplx_embed_internal.h"
+}
+
+#include <cuda_runtime.h>
+#include <cuda_bf16.h>
+#include <cublas_v2.h>
+
+#include <limits.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+ float *d; // device buffer; holds __nv_bfloat16 elements when bf16
+ int rows;
+ int cols;
+ int bf16; // 1 if d holds __nv_bfloat16, 0 if F32
+} cuda_matrix_t;
+
+typedef struct {
+ cuda_matrix_t qkv, wo;
+ float *q_norm;
+ float *k_norm;
+ float *input_norm;
+ float *post_attn_norm;
+ cuda_matrix_t gate_up_proj, down_proj;
+} cuda_layer_t;
+
+struct pplx_cuda_ctx {
+ pplx_model_t *cpu;
+ pplx_config_t config;
+ cudaStream_t stream;
+ cublasHandle_t blas;
+ cuda_matrix_t embed_tokens;
+ cuda_layer_t *layers;
+ float *norm;
+
+ int seq_cap;
+ int batch_cap;
+ int max_seq_cap;
+ float *x;
+ float *x_norm;
+ float *qkv;
+ float *attn_out;
+ float *ffn_gate_up;
+ void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights)
+ float *pooled_out;
+ float *rope_cos;
+ float *rope_sin;
+ int *token_ids;
+ int *offsets;
+ int *offsets_host;
+ int *positions;
+ float *kexp;
+ float *vexp;
+ float *attn_scores;
+ int *span_starts;
+ int *span_lens;
+ int pooled_rows_cap;
+ int span_cap;
+};
+
+#define CUDA_CHECK(expr) do { \
+ cudaError_t _e = (expr); \
+ if (_e != cudaSuccess) { \
+ fprintf(stderr, "cuda: %s failed: %s\n", #expr, \
+ cudaGetErrorString(_e)); \
+ return -1; \
+ } \
+} while (0)
+
+#define CUBLAS_CHECK(expr) do { \
+ cublasStatus_t _s = (expr); \
+ if (_s != CUBLAS_STATUS_SUCCESS) { \
+ fprintf(stderr, "cuda: %s failed: cublas status %d\n", \
+ #expr, (int)_s); \
+ return -1; \
+ } \
+} while (0)
+
+static int parse_gemm_compute(const char *mode, cublasComputeType_t *out)
+{
+ if (!mode || !mode[0] || !strcmp(mode, "f32")) {
+ *out = CUBLAS_COMPUTE_32F;
+ return 0;
+ }
+ if (!strcmp(mode, "tf32")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_TF32;
+ return 0;
+ }
+ if (!strcmp(mode, "bf16")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_16BF;
+ return 0;
+ }
+ if (!strcmp(mode, "16f")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_16F;
+ return 0;
+ }
+ return -1;
+}
+
+static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F;
+static int g_gemm_compute_set = 0;
+
+int pplx_cuda_set_fast_gemm(const char *mode)
+{
+ cublasComputeType_t ct;
+ if (parse_gemm_compute(mode, &ct) != 0)
+ return -1;
+ g_gemm_compute = ct;
+ g_gemm_compute_set = 1;
+ return 0;
+}
+
+static void cuda_matrix_free(cuda_matrix_t *m)
+{
+ if (!m) return;
+ cudaFree(m->d);
+ memset(m, 0, sizeof(*m));
+}
+
+static float bf16_to_f32(uint16_t v)
+{
+ uint32_t u = ((uint32_t)v) << 16;
+ float f;
+ memcpy(&f, &u, sizeof(f));
+ return f;
+}
+
+// Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are
+// preserved. BF16 shares F32's 8-bit exponent, so this never overflows.
+static uint16_t f32_to_bf16(float f)
+{
+ uint32_t x;
+ memcpy(&x, &f, sizeof(x));
+ if ((x & 0x7fffffffu) > 0x7f800000u) // NaN
+ return (uint16_t)((x >> 16) | 0x0040u);
+ x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even
+ return (uint16_t)(x >> 16);
+}
+
+// Store weights as native BF16 on the device when set (halves weight memory and
+// feeds BF16 tensor cores); default keeps exact F32. Read at load time.
+static int g_weights_bf16 = 0;
+
+int pplx_cuda_set_weights_bf16(int on)
+{
+ g_weights_bf16 = on ? 1 : 0;
+ return 0;
+}
+
+static int copy_weight_host_f32(float *dst, const pplx_weight_ref_t *w,
+ size_t count)
+{
+ if (!dst || !w || !w->data) return -1;
+ if (w->dtype == DTYPE_F32) {
+ memcpy(dst, w->data, count * sizeof(float));
+ return 0;
+ }
+ if (w->dtype == DTYPE_BF16) {
+ const uint16_t *src = (const uint16_t *)w->data;
+ for (size_t i = 0; i < count; i++)
+ dst[i] = bf16_to_f32(src[i]);
+ return 0;
+ }
+ return -1;
+}
+
+// Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is
+// set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16.
+static int upload_weight(cuda_matrix_t *m, const float *src, size_t count,
+ int rows, int cols)
+{
+ cudaError_t e;
+ if (g_weights_bf16) {
+ uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t));
+ if (!tmp16) return -1;
+ for (size_t i = 0; i < count; i++)
+ tmp16[i] = f32_to_bf16(src[i]);
+ e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e));
+ free(tmp16);
+ return -1;
+ }
+ e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16),
+ cudaMemcpyHostToDevice);
+ free(tmp16);
+ m->bf16 = 1;
+ } else {
+ e = cudaMalloc((void **)&m->d, count * sizeof(float));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ e = cudaMemcpy(m->d, src, count * sizeof(float), cudaMemcpyHostToDevice);
+ m->bf16 = 0;
+ }
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e));
+ cuda_matrix_free(m);
+ return -1;
+ }
+ m->rows = rows;
+ m->cols = cols;
+ return 0;
+}
+
+static int load_matrix(cuda_matrix_t *m, const pplx_weight_ref_t *w,
+ int rows, int cols)
+{
+ if (!m || !w || rows <= 0 || cols <= 0) return -1;
+ size_t count = (size_t)rows * (size_t)cols;
+ float *tmp = (float *)malloc(count * sizeof(float));
+ if (!tmp) return -1;
+ if (copy_weight_host_f32(tmp, w, count) != 0) {
+ free(tmp);
+ return -1;
+ }
+ int r = upload_weight(m, tmp, count, rows, cols);
+ free(tmp);
+ return r;
+}
+
+static int load_qkv_matrix(cuda_matrix_t *m, const pplx_layer_t *src,
+ const pplx_config_t *c)
+{
+ if (!m || !src || !c) return -1;
+ int rows = c->q_dim + 2 * c->kv_dim;
+ int cols = c->hidden_size;
+ size_t q_count = (size_t)c->q_dim * cols;
+ size_t kv_count = (size_t)c->kv_dim * cols;
+ size_t total = q_count + 2 * kv_count;
+ float *tmp = (float *)malloc(total * sizeof(float));
+ if (!tmp) return -1;
+ if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 ||
+ copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 ||
+ copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) {
+ free(tmp);
+ return -1;
+ }
+ int r = upload_weight(m, tmp, total, rows, cols);
+ free(tmp);
+ return r;
+}
+
+static int load_gate_up_matrix(cuda_matrix_t *m, const pplx_layer_t *src,
+ const pplx_config_t *c)
+{
+ if (!m || !src || !c) return -1;
+ int rows = 2 * c->intermediate_size;
+ int cols = c->hidden_size;
+ size_t proj_count = (size_t)c->intermediate_size * cols;
+ size_t total = 2 * proj_count;
+ float *tmp = (float *)malloc(total * sizeof(float));
+ if (!tmp) return -1;
+ if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 ||
+ copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) {
+ free(tmp);
+ return -1;
+ }
+ int r = upload_weight(m, tmp, total, rows, cols);
+ free(tmp);
+ return r;
+}
+
+static int load_vector(float **out, const float *src, int n)
+{
+ if (!out || !src || n <= 0) return -1;
+ cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice);
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e));
+ cudaFree(*out);
+ *out = NULL;
+ return -1;
+ }
+ return 0;
+}
+
+__global__ static void embed_lookup_kernel(float *x, const int *ids,
+ const void *emb, int emb_bf16,
+ int total, int hidden, int vocab)
+{
+ int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ int count = total * hidden;
+ if (idx >= count) return;
+ int tok = idx / hidden;
+ int d = idx - tok * hidden;
+ int id = ids[tok];
+ if (id < 0 || id >= vocab) {
+ x[idx] = 0.0f;
+ return;
+ }
+ size_t off = (size_t)id * hidden + d;
+ x[idx] = emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off])
+ : ((const float *)emb)[off];
+}
+
+// Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs).
+__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out,
+ const float *in, size_t n)
+{
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) out[i] = __float2bfloat16(in[i]);
+}
+
+__device__ static float block_sum(float v)
+{
+ __shared__ float partial[8];
+ __shared__ float total;
+ int lane = threadIdx.x & 31;
+ int warp = threadIdx.x >> 5;
+ int warps = blockDim.x >> 5;
+
+ for (int offset = 16; offset > 0; offset >>= 1)
+ v += __shfl_down_sync(0xffffffff, v, offset);
+ if (lane == 0) partial[warp] = v;
+ __syncthreads();
+
+ if (warp == 0) {
+ v = lane < warps ? partial[lane] : 0.0f;
+ for (int offset = 16; offset > 0; offset >>= 1)
+ v += __shfl_down_sync(0xffffffff, v, offset);
+ if (lane == 0) total = v;
+ }
+ __syncthreads();
+ return total;
+}
+
+__device__ static inline void store_act(float *out, size_t i, float v)
+{
+ out[i] = v;
+}
+
+__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v)
+{
+ out[i] = __float2bfloat16(v);
+}
+
+// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight
+// projection GEMM directly (same rounding as a separate cast kernel).
+template <typename OUT_T>
+__global__ static void rms_norm_kernel(OUT_T *out, const float *x,
+ const float *weight, int rows,
+ int dim, float eps)
+{
+ int row = blockIdx.x;
+ int tid = threadIdx.x;
+ float sum = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float v = x[(size_t)row * dim + d];
+ sum += v * v;
+ }
+ float total = block_sum(sum);
+ float inv = rsqrtf(total / (float)dim + eps);
+ for (int d = tid; d < dim; d += blockDim.x)
+ store_act(out, (size_t)row * dim + d,
+ x[(size_t)row * dim + d] * inv * weight[d]);
+}
+
+__global__ static void rms_norm_rope_head_kernel(float *x, const float *weight,
+ const int *positions,
+ const float *cosv,
+ const float *sinv,
+ int rows, int heads,
+ int head_dim,
+ int row_stride,
+ int base_offset,
+ float eps)
+{
+ __shared__ float partial[4];
+ __shared__ float total;
+ int row = blockIdx.x;
+ int head = blockIdx.y;
+ int tid = threadIdx.x;
+ int lane = tid & 31;
+ int warp = tid >> 5;
+ int half = head_dim / 2;
+ int pair_d = tid < half ? tid + half : tid - half;
+ float sign = tid < half ? -1.0f : 1.0f;
+ size_t base = (size_t)row * row_stride + base_offset + head * head_dim;
+ float v = tid < head_dim ? x[base + tid] : 0.0f;
+ float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f;
+ float sum = v * v;
+
+ for (int offset = 16; offset > 0; offset >>= 1)
+ sum += __shfl_down_sync(0xffffffff, sum, offset);
+ if (lane == 0) partial[warp] = sum;
+ __syncthreads();
+ if (warp == 0) {
+ sum = lane < 4 ? partial[lane] : 0.0f;
+ for (int offset = 16; offset > 0; offset >>= 1)
+ sum += __shfl_down_sync(0xffffffff, sum, offset);
+ if (lane == 0) total = sum;
+ }
+ __syncthreads();
+
+ if (tid < head_dim) {
+ float inv = rsqrtf(total / (float)head_dim + eps);
+ int pos = positions[row];
+ float c = cosv[(size_t)pos * head_dim + tid];
+ float s = sinv[(size_t)pos * head_dim + tid];
+ float a = v * inv * weight[tid];
+ float b = pair_v * inv * weight[pair_d];
+ x[base + tid] = a * c + sign * b * s;
+ }
+}
+
+// Writes SiLU(gate)*up to out with out_stride. The F32 path writes in place
+// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16
+// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM.
+template <typename OUT_T>
+__global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride,
+ const float *gate_up, int rows,
+ int intermediate)
+{
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
+ int n = rows * intermediate;
+ if (i < n) {
+ int row = i / intermediate;
+ int d = i - row * intermediate;
+ size_t base = (size_t)row * (2 * intermediate);
+ float g = gate_up[base + d];
+ float u = gate_up[base + intermediate + d];
+ store_act(out, (size_t)row * out_stride + d,
+ (g / (1.0f + expf(-g))) * u);
+ }
+}
+
+// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim]
+// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times)
+// so the attention GEMMs can stride uniformly over query heads.
+__global__ static void attn_expand_kv_kernel(
+ float *kexp, float *vexp, const float *qkv, int total, int n_heads,
+ int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset)
+{
+ int q_dim = n_heads * head_dim;
+ size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ size_t count = (size_t)total * q_dim;
+ if (idx >= count) return;
+ int t = (int)(idx / q_dim);
+ int rem = (int)(idx - (size_t)t * q_dim);
+ int h = rem / head_dim;
+ int e = rem - h * head_dim;
+ int kv = h / (n_heads / n_kv_heads);
+ const float *base = qkv + (size_t)t * qkv_dim;
+ kexp[idx] = base[k_offset + kv * head_dim + e];
+ vexp[idx] = base[v_offset + kv * head_dim + e];
+}
+
+// Row softmax over the key dimension for one sequence's score tensor laid out
+// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i).
+// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM.
+__global__ static void attn_softmax_kernel(float *scores, int L)
+{
+ float *s = scores + (size_t)blockIdx.x * L;
+ int tid = threadIdx.x;
+ __shared__ float red[256];
+
+ float m = -3.402823466e+38F;
+ for (int j = tid; j < L; j += blockDim.x) m = fmaxf(m, s[j]);
+ red[tid] = m;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]);
+ __syncthreads();
+ }
+ m = red[0];
+ __syncthreads();
+
+ float sum = 0.0f;
+ for (int j = tid; j < L; j += blockDim.x) {
+ float e = __expf(s[j] - m);
+ s[j] = e;
+ sum += e;
+ }
+ red[tid] = sum;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o) red[tid] += red[tid + o];
+ __syncthreads();
+ }
+ float inv = 1.0f / red[0];
+ for (int j = tid; j < L; j += blockDim.x) s[j] *= inv;
+}
+
+__global__ static void mean_pool_kernel(float *out, const float *x,
+ const int *offsets, int batch,
+ int hidden)
+{
+ int b = blockIdx.x;
+ int tid = threadIdx.x;
+ int start = offsets[b];
+ int end = offsets[b + 1];
+ int len = end - start;
+ for (int d = tid; d < hidden; d += blockDim.x) {
+ float sum = 0.0f;
+ for (int t = start; t < end; t++)
+ sum += x[(size_t)t * hidden + d];
+ out[(size_t)b * hidden + d] = sum / (float)len;
+ }
+}
+
+__global__ static void span_pool_kernel(float *out, const float *x,
+ const int *starts, const int *lens,
+ int n_spans, int hidden)
+{
+ int s = blockIdx.x;
+ int tid = threadIdx.x;
+ int start = starts[s];
+ int len = lens[s];
+ for (int d = tid; d < hidden; d += blockDim.x) {
+ float sum = 0.0f;
+ for (int t = 0; t < len; t++)
+ sum += x[(size_t)(start + t) * hidden + d];
+ out[(size_t)s * hidden + d] = sum / (float)len;
+ }
+}
+
+static int launch_check(void)
+{
+ cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ return 0;
+}
+
+// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm).
+// PPLX_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core
+// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands
+// internally.
+static cublasComputeType_t gemm_compute(void)
+{
+ static int env_init = 0;
+ if (!g_gemm_compute_set && !env_init) {
+ const char *e = getenv("PPLX_CUDA_GEMM_MODE");
+ if (e && parse_gemm_compute(e, &g_gemm_compute) == 0)
+ g_gemm_compute_set = 1;
+ env_init = 1;
+ }
+ return g_gemm_compute;
+}
+
+// y = w @ x (+ beta*y). With BF16 weights both GEMM inputs are BF16 and
+// accumulation/output stay F32: when x_is_bf16 the activation operand was
+// already produced in BF16 by the upstream kernel, otherwise the F32 operand
+// is cast into ctx->act_bf16 first. With F32 weights this is the exact path,
+// honoring gemm_compute().
+static int linear_ex(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w,
+ const void *x, int x_is_bf16, float *y, int rows,
+ int in_dim, int out_dim, int x_stride, float beta)
+{
+ const float alpha = 1.0f;
+ if (w->bf16) {
+ const void *xb = x;
+ if (!x_is_bf16) {
+ size_t n = (size_t)rows * (size_t)x_stride;
+ int threads = 256;
+ cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0,
+ ctx->stream>>>(
+ (__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n);
+ if (launch_check() != 0) return -1;
+ xb = ctx->act_bf16;
+ }
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N,
+ out_dim, rows, in_dim,
+ &alpha, w->d, CUDA_R_16BF, in_dim,
+ xb, CUDA_R_16BF, x_stride,
+ &beta, y, CUDA_R_32F, out_dim,
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ return 0;
+ }
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N,
+ out_dim, rows, in_dim,
+ &alpha, w->d, CUDA_R_32F, in_dim,
+ x, CUDA_R_32F, x_stride,
+ &beta, y, CUDA_R_32F, out_dim,
+ gemm_compute(), CUBLAS_GEMM_DEFAULT));
+ return 0;
+}
+
+static int linear(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w,
+ const float *x, float *y, int rows, int in_dim, int out_dim)
+{
+ return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f);
+}
+
+// x points at activations already stored as BF16 (e.g. ctx->act_bf16).
+static int linear_bf16x(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w,
+ const void *x, float *y, int rows, int in_dim,
+ int out_dim, int x_stride, float beta)
+{
+ return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta);
+}
+
+static int linear_accum(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w,
+ const float *x, float *y,
+ int rows, int in_dim, int out_dim)
+{
+ return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f);
+}
+
+static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa,
+ cublasOperation_t transb, int m, int n, int k,
+ const float *alpha, const float *A, int lda,
+ long long strideA, const float *B, int ldb,
+ long long strideB, const float *beta,
+ float *C, int ldc, long long strideC,
+ int batch_count)
+{
+ cublasComputeType_t ct = gemm_compute();
+ if (ct == CUBLAS_COMPUTE_32F) {
+ CUBLAS_CHECK(cublasSgemmStridedBatched(
+ blas, transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb,
+ strideB, beta, C, ldc, strideC, batch_count));
+ } else {
+ CUBLAS_CHECK(cublasGemmStridedBatchedEx(
+ blas, transa, transb, m, n, k,
+ alpha, A, CUDA_R_32F, lda, strideA,
+ B, CUDA_R_32F, ldb, strideB,
+ beta, C, CUDA_R_32F, ldc, strideC,
+ batch_count, ct, CUBLAS_GEMM_DEFAULT));
+ }
+ return 0;
+}
+
+// Attention for one micro-batch. Expands K/V to the per-query-head layout,
+// then for each sequence runs batched-over-heads Q@K^T (scaled) -> row softmax
+// -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and
+// ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets
+// is the host copy of the packed sequence boundaries.
+static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets,
+ int batch, int q_offset, int k_offset,
+ int v_offset, int qkv_dim, float scale)
+{
+ const pplx_config_t *c = &ctx->config;
+ int q_dim = c->q_dim;
+ int hd = c->head_dim;
+ int H = c->n_heads;
+ int total = h_offsets[batch];
+ const float alpha = scale, beta = 0.0f, one = 1.0f;
+
+ int threads = 256;
+ size_t exp_count = (size_t)total * q_dim;
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads,
+ 0, ctx->stream>>>(
+ ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim,
+ k_offset, v_offset);
+ if (launch_check() != 0) return -1;
+
+ for (int b = 0; b < batch; b++) {
+ int start = h_offsets[b];
+ int L = h_offsets[b + 1] - start;
+ const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset;
+ const float *K = ctx->kexp + (size_t)start * q_dim;
+ const float *V = ctx->vexp + (size_t)start * q_dim;
+ float *O = ctx->attn_out + (size_t)start * q_dim;
+ /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */
+ if (gemm_strided_batched(
+ ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd,
+ &alpha, K, q_dim, hd, Q, qkv_dim, hd,
+ &beta, ctx->attn_scores, L, (long long)L * L, H) != 0)
+ return -1;
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, L);
+ if (launch_check() != 0) return -1;
+ /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
+ if (gemm_strided_batched(
+ ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
+ &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L,
+ &beta, O, q_dim, hd, H) != 0)
+ return -1;
+ }
+ return 0;
+}
+
+static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_seq)
+{
+ const pplx_config_t *c = &ctx->config;
+ if (total > ctx->seq_cap) {
+ cudaFree(ctx->x);
+ cudaFree(ctx->x_norm);
+ cudaFree(ctx->qkv);
+ cudaFree(ctx->attn_out);
+ cudaFree(ctx->ffn_gate_up);
+ cudaFree(ctx->token_ids);
+ cudaFree(ctx->positions);
+ cudaFree(ctx->kexp);
+ cudaFree(ctx->vexp);
+ cudaFree(ctx->act_bf16);
+ ctx->x = ctx->x_norm = ctx->qkv = NULL;
+ ctx->attn_out = ctx->ffn_gate_up = NULL;
+ ctx->token_ids = ctx->positions = NULL;
+ ctx->kexp = ctx->vexp = NULL;
+ ctx->act_bf16 = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up,
+ (size_t)total * 2 * c->intermediate_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float)));
+ if (g_weights_bf16)
+ CUDA_CHECK(cudaMalloc(&ctx->act_bf16,
+ (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16)));
+ ctx->seq_cap = total;
+ }
+ if (batch + 1 > ctx->batch_cap) {
+ cudaFree(ctx->offsets);
+ free(ctx->offsets_host);
+ CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int)));
+ ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int));
+ if (!ctx->offsets_host) return -1;
+ ctx->batch_cap = batch + 1;
+ }
+ if (max_seq > ctx->max_seq_cap) {
+ cudaFree(ctx->rope_cos);
+ cudaFree(ctx->rope_sin);
+ cudaFree(ctx->attn_scores);
+ ctx->attn_scores = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores,
+ (size_t)c->n_heads * max_seq * max_seq * sizeof(float)));
+ float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
+ float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
+ if (!hcos || !hsin) {
+ free(hcos);
+ free(hsin);
+ return -1;
+ }
+ int half = c->head_dim / 2;
+ for (int pos = 0; pos < max_seq; pos++) {
+ for (int d = 0; d < half; d++) {
+ float inv = 1.0f / powf(c->rope_theta,
+ (float)(2 * d) / (float)c->head_dim);
+ float angle = (float)pos * inv;
+ float cv = cosf(angle);
+ float sv = sinf(angle);
+ hcos[(size_t)pos * c->head_dim + d] = cv;
+ hcos[(size_t)pos * c->head_dim + half + d] = cv;
+ hsin[(size_t)pos * c->head_dim + d] = sv;
+ hsin[(size_t)pos * c->head_dim + half + d] = sv;
+ }
+ }
+ CUDA_CHECK(cudaMalloc((void **)&ctx->rope_cos,
+ (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->rope_sin,
+ (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos,
+ (size_t)max_seq * c->head_dim * sizeof(float),
+ cudaMemcpyHostToDevice, ctx->stream));
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin,
+ (size_t)max_seq * c->head_dim * sizeof(float),
+ cudaMemcpyHostToDevice, ctx->stream));
+ free(hcos);
+ free(hsin);
+ ctx->max_seq_cap = max_seq;
+ }
+ return 0;
+}
+
+static int ensure_pooled_rows(pplx_cuda_ctx_t *ctx, int rows)
+{
+ if (rows <= ctx->pooled_rows_cap) return 0;
+ const pplx_config_t *c = &ctx->config;
+ cudaFree(ctx->pooled_out);
+ ctx->pooled_out = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out,
+ (size_t)rows * c->hidden_size * sizeof(float)));
+ ctx->pooled_rows_cap = rows;
+ return 0;
+}
+
+static int ensure_span_buffers(pplx_cuda_ctx_t *ctx, int n_spans)
+{
+ if (n_spans <= ctx->span_cap) return 0;
+ cudaFree(ctx->span_starts);
+ cudaFree(ctx->span_lens);
+ ctx->span_starts = NULL;
+ ctx->span_lens = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts,
+ (size_t)n_spans * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens,
+ (size_t)n_spans * sizeof(int)));
+ ctx->span_cap = n_spans;
+ return 0;
+}
+
+static int load_layer(cuda_layer_t *dst, const pplx_layer_t *src,
+ const pplx_config_t *c)
+{
+ return
+ load_qkv_matrix(&dst->qkv, src, c) ||
+ load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
+ load_vector(&dst->q_norm, src->q_norm, c->head_dim) ||
+ load_vector(&dst->k_norm, src->k_norm, c->head_dim) ||
+ load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
+ load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
+ load_gate_up_matrix(&dst->gate_up_proj, src, c) ||
+ load_matrix(&dst->down_proj, &src->down_proj,
+ c->hidden_size, c->intermediate_size);
+}
+
+static void free_layer(cuda_layer_t *l)
+{
+ cuda_matrix_free(&l->qkv);
+ cuda_matrix_free(&l->wo);
+ cudaFree(l->q_norm);
+ cudaFree(l->k_norm);
+ cudaFree(l->input_norm);
+ cudaFree(l->post_attn_norm);
+ cuda_matrix_free(&l->gate_up_proj);
+ cuda_matrix_free(&l->down_proj);
+}
+
+pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir)
+{
+ pplx_model_t *cpu = pplx_model_load(model_dir);
+ if (!cpu) return NULL;
+
+ pplx_cuda_ctx_t *ctx = (pplx_cuda_ctx_t *)calloc(1, sizeof(*ctx));
+ if (!ctx) {
+ pplx_model_free(cpu);
+ return NULL;
+ }
+ ctx->cpu = cpu;
+ ctx->config = cpu->config;
+
+ if (cudaStreamCreate(&ctx->stream) != cudaSuccess ||
+ cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS ||
+ cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) {
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+
+ const pplx_config_t *c = &ctx->config;
+ ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers));
+ if (!ctx->layers) {
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+ if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens,
+ c->vocab_size, c->hidden_size) != 0 ||
+ load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) {
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+ for (int i = 0; i < c->n_layers; i++) {
+ if (load_layer(&ctx->layers[i], &cpu->weights.layers[i], c) != 0) {
+ fprintf(stderr, "cuda: failed to load layer %d\n", i);
+ pplx_cuda_free(ctx);
+ return NULL;
+ }
+ }
+ cudaStreamSynchronize(ctx->stream);
+ return ctx;
+}
+
+void pplx_cuda_free(pplx_cuda_ctx_t *ctx)
+{
+ if (!ctx) return;
+ cuda_matrix_free(&ctx->embed_tokens);
+ if (ctx->layers) {
+ for (int i = 0; i < ctx->config.n_layers; i++)
+ free_layer(&ctx->layers[i]);
+ free(ctx->layers);
+ }
+ cudaFree(ctx->norm);
+ cudaFree(ctx->x);
+ cudaFree(ctx->x_norm);
+ cudaFree(ctx->qkv);
+ cudaFree(ctx->attn_out);
+ cudaFree(ctx->ffn_gate_up);
+ cudaFree(ctx->pooled_out);
+ cudaFree(ctx->rope_cos);
+ cudaFree(ctx->rope_sin);
+ cudaFree(ctx->token_ids);
+ cudaFree(ctx->offsets);
+ free(ctx->offsets_host);
+ cudaFree(ctx->positions);
+ cudaFree(ctx->kexp);
+ cudaFree(ctx->vexp);
+ cudaFree(ctx->act_bf16);
+ cudaFree(ctx->attn_scores);
+ cudaFree(ctx->span_starts);
+ cudaFree(ctx->span_lens);
+ if (ctx->blas) cublasDestroy(ctx->blas);
+ if (ctx->stream) cudaStreamDestroy(ctx->stream);
+ pplx_model_free(ctx->cpu);
+ free(ctx);
+}
+
+const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx)
+{
+ return ctx ? &ctx->config : NULL;
+}
+
+static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch)
+{
+ if (!ctx || !inputs || batch <= 0) return -1;
+ const pplx_config_t *c = &ctx->config;
+ int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
+ if (!h_offsets) return -1;
+ int total = 0, max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) {
+ free(h_offsets);
+ return -1;
+ }
+ h_offsets[b] = total;
+ total += inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ }
+ h_offsets[batch] = total;
+ int *h_ids = (int *)malloc((size_t)total * sizeof(int));
+ int *h_pos = (int *)malloc((size_t)total * sizeof(int));
+ if (!h_offsets || !h_ids || !h_pos) {
+ free(h_offsets); free(h_ids); free(h_pos);
+ return -1;
+ }
+
+ for (int b = 0; b < batch; b++) {
+ int off = h_offsets[b];
+ memcpy(h_ids + off, inputs[b].ids,
+ (size_t)inputs[b].n_tokens * sizeof(int));
+ for (int i = 0; i < inputs[b].n_tokens; i++)
+ h_pos[off + i] = i;
+ }
+ if (ensure_buffers(ctx, total, batch, max_seq) != 0) {
+ free(h_offsets); free(h_ids); free(h_pos);
+ return -1;
+ }
+ memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int));
+ cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids,
+ (size_t)total * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->positions, h_pos,
+ (size_t)total * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->offsets, h_offsets,
+ (size_t)(batch + 1) * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ free(h_offsets); free(h_ids); free(h_pos);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce));
+ return -1;
+ }
+
+ int threads = 256;
+ int blocks_hidden = (total * c->hidden_size + threads - 1) / threads;
+ embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16,
+ total, c->hidden_size, c->vocab_size);
+ if (launch_check() != 0) return -1;
+
+ float scale = 1.0f / sqrtf((float)c->head_dim);
+ int q_offset = 0;
+ int k_offset = c->q_dim;
+ int v_offset = c->q_dim + c->kv_dim;
+ int qkv_dim = c->q_dim + 2 * c->kv_dim;
+ // With BF16 weights the norm and SiLU kernels store their results as BF16
+ // directly into ctx->act_bf16, which the next projection GEMM consumes
+ // before any later producer reuses the buffer (single stream). This
+ // removes three cast launches per layer; only the attention output still
+ // needs a cast because cuBLAS cannot emit a BF16 C from F32 A/B operands.
+ __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16;
+ for (int layer = 0; layer < c->n_layers; layer++) {
+ cuda_layer_t *l = &ctx->layers[layer];
+ int wbf16 = l->qkv.bf16;
+ if (wbf16) {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
+ act, ctx->x, l->input_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+ if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv,
+ total, c->hidden_size, qkv_dim,
+ c->hidden_size, 0.0f) != 0)
+ return -1;
+ } else {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
+ ctx->x_norm, ctx->x, l->input_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+ if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv,
+ total, c->hidden_size, qkv_dim) != 0)
+ return -1;
+ }
+
+ rms_norm_rope_head_kernel<<<dim3(total, c->n_heads), 128, 0, ctx->stream>>>(
+ ctx->qkv, l->q_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin,
+ total, c->n_heads, c->head_dim, qkv_dim, q_offset,
+ c->rms_norm_eps);
+ rms_norm_rope_head_kernel<<<dim3(total, c->n_kv_heads), 128, 0, ctx->stream>>>(
+ ctx->qkv, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin,
+ total, c->n_kv_heads, c->head_dim, qkv_dim, k_offset,
+ c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+
+ if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset,
+ k_offset, v_offset, qkv_dim, scale) != 0)
+ return -1;
+
+ if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x,
+ total, c->q_dim, c->hidden_size) != 0)
+ return -1;
+
+ int inter_count = total * c->intermediate_size;
+ if (wbf16) {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
+ act, ctx->x, l->post_attn_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+ if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up,
+ total, c->hidden_size, 2 * c->intermediate_size,
+ c->hidden_size, 0.0f) != 0)
+ return -1;
+ silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads,
+ threads, 0, ctx->stream>>>(
+ act, c->intermediate_size, ctx->ffn_gate_up, total,
+ c->intermediate_size);
+ if (launch_check() != 0) return -1;
+ if (linear_bf16x(ctx, &l->down_proj, act, ctx->x,
+ total, c->intermediate_size, c->hidden_size,
+ c->intermediate_size, 1.0f) != 0)
+ return -1;
+ } else {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
+ ctx->x_norm, ctx->x, l->post_attn_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+ if (linear(ctx, &l->gate_up_proj, ctx->x_norm,
+ ctx->ffn_gate_up, total, c->hidden_size,
+ 2 * c->intermediate_size) != 0)
+ return -1;
+ silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads,
+ threads, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, 2 * c->intermediate_size, ctx->ffn_gate_up,
+ total, c->intermediate_size);
+ if (launch_check() != 0) return -1;
+ if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x,
+ total, c->intermediate_size, c->hidden_size,
+ 2 * c->intermediate_size, 1.0f) != 0)
+ return -1;
+ }
+ }
+
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(
+ ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0) return -1;
+
+ return 0;
+}
+
+int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ const pplx_config_t *c = &ctx->config;
+ if (cuda_forward_batch(ctx, inputs, batch) != 0)
+ return -1;
+ if (ensure_pooled_rows(ctx, batch) != 0)
+ return -1;
+ mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(
+ ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size);
+ if (launch_check() != 0) return -1;
+ cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
+ (size_t)batch * c->hidden_size * sizeof(float),
+ cudaMemcpyDeviceToHost, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce));
+ return -1;
+ }
+ return 0;
+}
+
+int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx,
+ const pplx_context_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ const pplx_config_t *c = &ctx->config;
+ int total_spans = 0;
+ int total_tokens = 0;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ int n_tokens = input->input.n_tokens;
+ if (!input->input.ids || n_tokens <= 0 || !input->spans ||
+ input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans ||
+ total_tokens > INT_MAX - n_tokens)
+ return -1;
+ for (int s = 0; s < input->n_spans; s++) {
+ int start = input->spans[s].start;
+ int len = input->spans[s].n_tokens;
+ if (start < 0 || len <= 0 || start > n_tokens ||
+ len > n_tokens - start)
+ return -1;
+ }
+ total_spans += input->n_spans;
+ total_tokens += n_tokens;
+ }
+
+ pplx_input_t *packed = (pplx_input_t *)malloc((size_t)batch * sizeof(*packed));
+ int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int));
+ int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int));
+ if (!packed || !h_starts || !h_lens) {
+ free(packed);
+ free(h_starts);
+ free(h_lens);
+ return -1;
+ }
+
+ int token_offset = 0;
+ int span_offset = 0;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ packed[b] = input->input;
+ for (int s = 0; s < input->n_spans; s++) {
+ h_starts[span_offset] = token_offset + input->spans[s].start;
+ h_lens[span_offset] = input->spans[s].n_tokens;
+ span_offset++;
+ }
+ token_offset += input->input.n_tokens;
+ }
+
+ int rc = -1;
+ cudaError_t ce;
+ if (cuda_forward_batch(ctx, packed, batch) != 0)
+ goto cleanup;
+ if (ensure_pooled_rows(ctx, total_spans) != 0 ||
+ ensure_span_buffers(ctx, total_spans) != 0)
+ goto cleanup;
+ ce = cudaMemcpyAsync(ctx->span_starts, h_starts,
+ (size_t)total_spans * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->span_lens, h_lens,
+ (size_t)total_spans * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce));
+ goto cleanup;
+ }
+ span_pool_kernel<<<total_spans, 256, 0, ctx->stream>>>(
+ ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens,
+ total_spans, c->hidden_size);
+ if (launch_check() != 0)
+ goto cleanup;
+ ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
+ (size_t)total_spans * c->hidden_size * sizeof(float),
+ cudaMemcpyDeviceToHost, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda span output copy failed: %s\n",
+ cudaGetErrorString(ce));
+ goto cleanup;
+ }
+ rc = 0;
+
+cleanup:
+ free(h_lens);
+ free(h_starts);
+ free(packed);
+ return rc;
+}
+
+int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids,
+ int n_tokens, float *out_embedding)
+{
+ pplx_input_t input = { token_ids, n_tokens };
+ return pplx_cuda_embed_batch(ctx, &input, 1, out_embedding);
+}
+
+float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids,
+ int n_tokens)
+{
+ if (!ctx) return NULL;
+ int hidden = ctx->config.hidden_size;
+ float *out = (float *)malloc((size_t)hidden * sizeof(float));
+ if (!out) return NULL;
+ if (pplx_cuda_embed_into(ctx, token_ids, n_tokens, out) != 0) {
+ free(out);
+ return NULL;
+ }
+ return out;
+}