aboutsummaryrefslogtreecommitdiff
path: root/src/cuda.cu
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
commitd1ba1ebc06ef0d62ca932388fae83e6992594719 (patch)
treef962207b779016593258f990348216999a58766a /src/cuda.cu
parentbe20dfda0f836808dccfc14db244592e41ba28bc (diff)
downloadffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz
ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'src/cuda.cu')
-rw-r--r--src/cuda.cu2052
1 files changed, 2052 insertions, 0 deletions
diff --git a/src/cuda.cu b/src/cuda.cu
new file mode 100644
index 0000000..6ce7dfb
--- /dev/null
+++ b/src/cuda.cu
@@ -0,0 +1,2052 @@
+extern "C" {
+#include "cuda.h"
+#include "internal.h"
+}
+
+#include <cuda_runtime.h>
+#include <cuda_bf16.h>
+#include <cublas_v2.h>
+
+#include <limits.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+ float *d; // device buffer; holds __nv_bfloat16 elements when bf16
+ int rows;
+ int cols;
+ int bf16; // 1 if d holds __nv_bfloat16, 0 if F32
+} cuda_matrix_t;
+
+typedef struct {
+ cuda_matrix_t qkv, wo;
+ float *qkv_bias;
+ float *q_norm;
+ float *k_norm;
+ float *input_norm;
+ float *post_attn_norm;
+ cuda_matrix_t gate_up_proj, down_proj;
+ /* BERT family (NULL for Qwen3). The two block LayerNorm weights reuse
+ * input_norm (post-attention) and post_attn_norm (post-FFN); gate_up_proj
+ * holds the single BERT up_proj. These add the biases those slots lack. */
+ float *o_bias;
+ float *attn_ln_bias;
+ float *ffn_inter_bias;
+ float *ffn_out_bias;
+ float *ffn_ln_bias;
+} cuda_layer_t;
+
+struct embed_cuda_ctx {
+ embed_model_t *cpu;
+ int own_cpu; // free ctx->cpu in embed_cuda_free only when this context owns it
+ embed_config_t config;
+ cudaStream_t stream;
+ cublasHandle_t blas;
+ cuda_matrix_t embed_tokens;
+ cuda_layer_t *layers;
+ float *norm;
+ /* BERT family (NULL for Qwen3): learned absolute position embeddings, the
+ * token-type[0] embedding (row 0 only, [hidden]), and the embedding
+ * LayerNorm weight/bias. */
+ float *position_embeddings;
+ float *token_type_embedding;
+ float *embed_ln_w;
+ float *embed_ln_b;
+
+ int seq_cap;
+ int batch_cap;
+ int max_seq_cap;
+ float *x;
+ float *x_norm;
+ float *qkv;
+ float *attn_out;
+ float *ffn_gate_up;
+ void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights)
+ float *weight_f32; // F32 widen scratch for one BF16 weight (memory-only bf16)
+ size_t weight_f32_elems;
+ float *pooled_out;
+ float *rope_cos;
+ float *rope_sin;
+ int *token_ids;
+ int *offsets;
+ int *offsets_host;
+ int *positions;
+ float *kexp;
+ float *vexp;
+ float *attn_scores;
+ void *attn_probs; // BF16 softmax output (bf16 weights)
+ long long attn_scores_elems;
+ long long attn_probs_elems;
+ // Equal-length micro-batch attention: pointer arrays for batched GEMMs,
+ // built once per forward pass (layer-invariant buffer addresses).
+ // attn_G is the number of sequences whose score tensors fit the scratch
+ // budget at once; 0 disables the batched path for this forward.
+ const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads
+ const void **attn_ptrs_host;
+ int attn_ptrs_cap; // capacity in pointer entries
+ int attn_G;
+ int attn_L;
+ int *span_starts;
+ int *span_lens;
+ int pooled_rows_cap;
+ int span_cap;
+};
+
+// Scratch budget for the batched-attention score tensors (F32 elements).
+enum { CUDA_ATTN_SCORES_BUDGET = 64 * 1024 * 1024 };
+// Above this sequence length the per-sequence strided-batched loop beats the
+// pointer-array batched GEMMs (L4: batched wins <=80tok, ties ~140, loses
+// >=256; both paths are identical precision).
+enum { CUDA_ATTN_BATCHED_MAX_LEN = 192 };
+
+#define CUDA_CHECK(expr) \
+ do { \
+ cudaError_t _e = (expr); \
+ if (_e != cudaSuccess) { \
+ fprintf(stderr, "cuda: %s failed: %s\n", #expr, cudaGetErrorString(_e)); \
+ return -1; \
+ } \
+ } while (0)
+
+#define CUBLAS_CHECK(expr) \
+ do { \
+ cublasStatus_t _s = (expr); \
+ if (_s != CUBLAS_STATUS_SUCCESS) { \
+ fprintf(stderr, "cuda: %s failed: cublas status %d\n", #expr, (int)_s); \
+ return -1; \
+ } \
+ } while (0)
+
+static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) {
+ if (!mode || !mode[0] || !strcmp(mode, "f32")) {
+ *out = CUBLAS_COMPUTE_32F;
+ return 0;
+ }
+ if (!strcmp(mode, "tf32")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_TF32;
+ return 0;
+ }
+ if (!strcmp(mode, "bf16")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_16BF;
+ return 0;
+ }
+ if (!strcmp(mode, "16f")) {
+ *out = CUBLAS_COMPUTE_32F_FAST_16F;
+ return 0;
+ }
+ return -1;
+}
+
+static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F;
+static int g_gemm_compute_set = 0;
+
+int embed_cuda_set_fast_gemm(const char *mode) {
+ cublasComputeType_t ct;
+ if (parse_gemm_compute(mode, &ct) != 0)
+ return -1;
+ g_gemm_compute = ct;
+ g_gemm_compute_set = 1;
+ return 0;
+}
+
+static void cuda_matrix_free(cuda_matrix_t *m) {
+ if (!m)
+ return;
+ cudaFree(m->d);
+ memset(m, 0, sizeof(*m));
+}
+
+static float bf16_to_f32(uint16_t v) {
+ uint32_t u = ((uint32_t)v) << 16;
+ float f;
+ memcpy(&f, &u, sizeof(f));
+ return f;
+}
+
+// Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are
+// preserved. BF16 shares F32's 8-bit exponent, so this never overflows.
+static uint16_t f32_to_bf16(float f) {
+ uint32_t x;
+ memcpy(&x, &f, sizeof(x));
+ if ((x & 0x7fffffffu) > 0x7f800000u) // NaN
+ return (uint16_t)((x >> 16) | 0x0040u);
+ x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even
+ return (uint16_t)(x >> 16);
+}
+
+// Store weights as native BF16 on the device when set (halves weight memory and
+// feeds BF16 tensor cores); default keeps exact F32 for F32 model files. When
+// the flag was never set explicitly, the loader follows the model file dtype,
+// matching the CPU and MLX backends.
+static int g_weights_bf16 = 0;
+static int g_weights_bf16_set = 0;
+
+int embed_cuda_set_weights_bf16(int on) {
+ g_weights_bf16 = on ? 1 : 0;
+ g_weights_bf16_set = 1;
+ return 0;
+}
+
+static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, size_t count) {
+ if (!dst || !w || !w->data)
+ return -1;
+ if (w->dtype == DTYPE_F32) {
+ memcpy(dst, w->data, count * sizeof(float));
+ return 0;
+ }
+ if (w->dtype == DTYPE_BF16) {
+ const uint16_t *src = (const uint16_t *)w->data;
+ for (size_t i = 0; i < count; i++)
+ dst[i] = bf16_to_f32(src[i]);
+ return 0;
+ }
+ return -1;
+}
+
+// Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is
+// set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16.
+static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, int rows, int cols) {
+ cudaError_t e;
+ if (g_weights_bf16) {
+ uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t));
+ if (!tmp16)
+ return -1;
+ for (size_t i = 0; i < count; i++)
+ tmp16[i] = f32_to_bf16(src[i]);
+ e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e));
+ free(tmp16);
+ return -1;
+ }
+ e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), cudaMemcpyHostToDevice);
+ free(tmp16);
+ m->bf16 = 1;
+ } else {
+ e = cudaMalloc((void **)&m->d, count * sizeof(float));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ e = cudaMemcpy(m->d, src, count * sizeof(float), cudaMemcpyHostToDevice);
+ m->bf16 = 0;
+ }
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e));
+ cuda_matrix_free(m);
+ return -1;
+ }
+ m->rows = rows;
+ m->cols = cols;
+ return 0;
+}
+
+static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, int cols) {
+ if (!m || !w || rows <= 0 || cols <= 0)
+ return -1;
+ size_t count = (size_t)rows * (size_t)cols;
+ float *tmp = (float *)malloc(count * sizeof(float));
+ if (!tmp)
+ return -1;
+ if (copy_weight_host_f32(tmp, w, count) != 0) {
+ free(tmp);
+ return -1;
+ }
+ int r = upload_weight(m, tmp, count, rows, cols);
+ free(tmp);
+ return r;
+}
+
+static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
+ if (!m || !src || !c)
+ return -1;
+ int rows = c->q_dim + 2 * c->kv_dim;
+ int cols = c->hidden_size;
+ size_t q_count = (size_t)c->q_dim * cols;
+ size_t kv_count = (size_t)c->kv_dim * cols;
+ size_t total = q_count + 2 * kv_count;
+ float *tmp = (float *)malloc(total * sizeof(float));
+ if (!tmp)
+ return -1;
+ if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 ||
+ copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 ||
+ copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) {
+ free(tmp);
+ return -1;
+ }
+ int r = upload_weight(m, tmp, total, rows, cols);
+ free(tmp);
+ return r;
+}
+
+static int
+load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
+ if (!m || !src || !c)
+ return -1;
+ int rows = 2 * c->intermediate_size;
+ int cols = c->hidden_size;
+ size_t proj_count = (size_t)c->intermediate_size * cols;
+ size_t total = 2 * proj_count;
+ float *tmp = (float *)malloc(total * sizeof(float));
+ if (!tmp)
+ return -1;
+ if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 ||
+ copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) {
+ free(tmp);
+ return -1;
+ }
+ int r = upload_weight(m, tmp, total, rows, cols);
+ free(tmp);
+ return r;
+}
+
+static int load_vector(float **out, const float *src, int n) {
+ if (!out || !src || n <= 0)
+ return -1;
+ cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float));
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice);
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e));
+ cudaFree(*out);
+ *out = NULL;
+ return -1;
+ }
+ return 0;
+}
+
+static int load_qkv_bias(float **out, const embed_layer_t *src, const embed_config_t *c) {
+ int n = c->q_dim + 2 * c->kv_dim;
+ float *host = (float *)malloc((size_t)n * sizeof(float));
+ if (!host)
+ return -1;
+ memcpy(host, src->q_bias, (size_t)c->q_dim * sizeof(float));
+ memcpy(host + c->q_dim, src->k_bias, (size_t)c->kv_dim * sizeof(float));
+ memcpy(host + c->q_dim + c->kv_dim, src->v_bias, (size_t)c->kv_dim * sizeof(float));
+ int rc = load_vector(out, host, n);
+ free(host);
+ return rc;
+}
+
+__global__ static void add_row_bias_kernel(float *x, const float *bias, int rows, int dim) {
+ int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ int count = rows * dim;
+ if (idx < count)
+ x[idx] += bias[idx % dim];
+}
+
+__global__ static void embed_lookup_kernel(
+ float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) {
+ int idx = blockIdx.x * blockDim.x + threadIdx.x;
+ int count = total * hidden;
+ if (idx >= count)
+ return;
+ int tok = idx / hidden;
+ int d = idx - tok * hidden;
+ int id = ids[tok];
+ if (id < 0 || id >= vocab) {
+ x[idx] = 0.0f;
+ return;
+ }
+ size_t off = (size_t)id * hidden + d;
+ x[idx] =
+ emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) : ((const float *)emb)[off];
+}
+
+// Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs).
+__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, const float *in, size_t n) {
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n)
+ out[i] = __float2bfloat16(in[i]);
+}
+
+// Widen n BF16 values to F32 (used to run an exact F32 GEMM on BF16-stored
+// weights - memory-only BF16, no precision loss from the operands).
+__global__ static void cast_bf16_to_f32_kernel(float *out, const __nv_bfloat16 *in, size_t n) {
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n)
+ out[i] = __bfloat162float(in[i]);
+}
+
+__device__ static float block_sum(float v) {
+ __shared__ float partial[8];
+ __shared__ float total;
+ int lane = threadIdx.x & 31;
+ int warp = threadIdx.x >> 5;
+ int warps = blockDim.x >> 5;
+
+ for (int offset = 16; offset > 0; offset >>= 1)
+ v += __shfl_down_sync(0xffffffff, v, offset);
+ if (lane == 0)
+ partial[warp] = v;
+ __syncthreads();
+
+ if (warp == 0) {
+ v = lane < warps ? partial[lane] : 0.0f;
+ for (int offset = 16; offset > 0; offset >>= 1)
+ v += __shfl_down_sync(0xffffffff, v, offset);
+ if (lane == 0)
+ total = v;
+ }
+ __syncthreads();
+ return total;
+}
+
+__device__ static inline void store_act(float *out, size_t i, float v) { out[i] = v; }
+
+__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) {
+ out[i] = __float2bfloat16(v);
+}
+
+// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight
+// projection GEMM directly (same rounding as a separate cast kernel).
+template <typename OUT_T>
+__global__ static void
+rms_norm_kernel(OUT_T *out, const float *x, const float *weight, int rows, int dim, float eps) {
+ int row = blockIdx.x;
+ int tid = threadIdx.x;
+ float sum = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float v = x[(size_t)row * dim + d];
+ sum += v * v;
+ }
+ float total = block_sum(sum);
+ float inv = rsqrtf(total / (float)dim + eps);
+ for (int d = tid; d < dim; d += blockDim.x)
+ store_act(out, (size_t)row * dim + d, x[(size_t)row * dim + d] * inv * weight[d]);
+}
+
+// One grid covers Q and K: blockIdx.y < n_heads selects the Q head with
+// q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset.
+__global__ static void rms_norm_rope_qk_kernel(float *x,
+ const float *q_norm,
+ const float *k_norm,
+ const int *positions,
+ const float *cosv,
+ const float *sinv,
+ int rows,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ int row_stride,
+ int q_offset,
+ int k_offset,
+ int qk_norm,
+ float eps) {
+ __shared__ float partial[4];
+ __shared__ float total;
+ int row = blockIdx.x;
+ int head = blockIdx.y;
+ int tid = threadIdx.x;
+ int lane = tid & 31;
+ int warp = tid >> 5;
+ int half = head_dim / 2;
+ int pair_d = tid < half ? tid + half : tid - half;
+ float sign = tid < half ? -1.0f : 1.0f;
+ const float *weight = head < n_heads ? q_norm : k_norm;
+ int base_offset = head < n_heads ? q_offset : k_offset;
+ if (head >= n_heads)
+ head -= n_heads;
+ size_t base = (size_t)row * row_stride + base_offset + head * head_dim;
+ float v = tid < head_dim ? x[base + tid] : 0.0f;
+ float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f;
+ if (qk_norm) {
+ float sum = v * v;
+ for (int offset = 16; offset > 0; offset >>= 1)
+ sum += __shfl_down_sync(0xffffffff, sum, offset);
+ if (lane == 0)
+ partial[warp] = sum;
+ __syncthreads();
+ if (warp == 0) {
+ sum = lane < 4 ? partial[lane] : 0.0f;
+ for (int offset = 16; offset > 0; offset >>= 1)
+ sum += __shfl_down_sync(0xffffffff, sum, offset);
+ if (lane == 0)
+ total = sum;
+ }
+ __syncthreads();
+ }
+
+ if (tid < head_dim) {
+ int pos = positions[row];
+ float c = cosv[(size_t)pos * head_dim + tid];
+ float s = sinv[(size_t)pos * head_dim + tid];
+ float a = v;
+ float b = pair_v;
+ if (qk_norm) {
+ float inv = rsqrtf(total / (float)head_dim + eps);
+ a *= inv * weight[tid];
+ b *= inv * weight[pair_d];
+ }
+ x[base + tid] = a * c + sign * b * s;
+ }
+}
+
+// Writes SiLU(gate)*up to out with out_stride. The F32 path writes in place
+// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16
+// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM.
+template <typename OUT_T>
+__global__ static void silu_mul_packed_kernel(
+ OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) {
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
+ int n = rows * intermediate;
+ if (i < n) {
+ int row = i / intermediate;
+ int d = i - row * intermediate;
+ size_t base = (size_t)row * (2 * intermediate);
+ float g = gate_up[base + d];
+ float u = gate_up[base + intermediate + d];
+ store_act(out, (size_t)row * out_stride + d, (g / (1.0f + expf(-g))) * u);
+ }
+}
+
+// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim]
+// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times)
+// so the attention GEMMs can stride uniformly over query heads.
+// V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly.
+template <typename V_T>
+__global__ static void attn_expand_kv_kernel(float *kexp,
+ V_T *vexp,
+ const float *qkv,
+ int total,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ int qkv_dim,
+ int k_offset,
+ int v_offset) {
+ int q_dim = n_heads * head_dim;
+ size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ size_t count = (size_t)total * q_dim;
+ if (idx >= count)
+ return;
+ int t = (int)(idx / q_dim);
+ int rem = (int)(idx - (size_t)t * q_dim);
+ int h = rem / head_dim;
+ int e = rem - h * head_dim;
+ int kv = h / (n_heads / n_kv_heads);
+ const float *base = qkv + (size_t)t * qkv_dim;
+ kexp[idx] = base[k_offset + kv * head_dim + e];
+ store_act(vexp, idx, base[v_offset + kv * head_dim + e]);
+}
+
+// Row softmax over the key dimension for one sequence's score tensor laid out
+// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i).
+// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM.
+// P_T is float with probs == scores (in place), or __nv_bfloat16 when the
+// probabilities feed a BF16 @V GEMM.
+template <typename P_T>
+__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) {
+ float *s = scores + (size_t)blockIdx.x * L;
+ P_T *p = probs + (size_t)blockIdx.x * L;
+ int tid = threadIdx.x;
+ __shared__ float red[256];
+
+ float m = -3.402823466e+38F;
+ for (int j = tid; j < L; j += blockDim.x)
+ m = fmaxf(m, s[j]);
+ red[tid] = m;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o)
+ red[tid] = fmaxf(red[tid], red[tid + o]);
+ __syncthreads();
+ }
+ m = red[0];
+ __syncthreads();
+
+ float sum = 0.0f;
+ for (int j = tid; j < L; j += blockDim.x) {
+ float e = __expf(s[j] - m);
+ s[j] = e;
+ sum += e;
+ }
+ red[tid] = sum;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o)
+ red[tid] += red[tid + o];
+ __syncthreads();
+ }
+ float inv = 1.0f / red[0];
+ for (int j = tid; j < L; j += blockDim.x)
+ store_act(p, (size_t)j, s[j] * inv);
+}
+
+template <typename P_T>
+__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, int L) {
+ float *s = scores + (size_t)blockIdx.x * L;
+ P_T *p = probs + (size_t)blockIdx.x * L;
+ int query = blockIdx.x % L;
+ int keys = query + 1;
+ int tid = threadIdx.x;
+ __shared__ float red[256];
+
+ float m = -3.402823466e+38F;
+ for (int j = tid; j < keys; j += blockDim.x)
+ m = fmaxf(m, s[j]);
+ red[tid] = m;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o)
+ red[tid] = fmaxf(red[tid], red[tid + o]);
+ __syncthreads();
+ }
+ m = red[0];
+ __syncthreads();
+
+ float sum = 0.0f;
+ for (int j = tid; j < keys; j += blockDim.x) {
+ float e = __expf(s[j] - m);
+ s[j] = e;
+ sum += e;
+ }
+ red[tid] = sum;
+ __syncthreads();
+ for (int o = blockDim.x / 2; o > 0; o >>= 1) {
+ if (tid < o)
+ red[tid] += red[tid + o];
+ __syncthreads();
+ }
+ float inv = 1.0f / red[0];
+ for (int j = tid; j < L; j += blockDim.x)
+ store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f);
+}
+
+__global__ static void
+mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
+ int b = blockIdx.x;
+ int tid = threadIdx.x;
+ int start = offsets[b];
+ int end = offsets[b + 1];
+ int len = end - start;
+ for (int d = tid; d < hidden; d += blockDim.x) {
+ float sum = 0.0f;
+ for (int t = start; t < end; t++)
+ sum += x[(size_t)t * hidden + d];
+ out[(size_t)b * hidden + d] = sum / (float)len;
+ }
+}
+
+__global__ static void
+last_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
+ int b = blockIdx.x;
+ int tid = threadIdx.x;
+ int last = offsets[b + 1] - 1;
+ for (int d = tid; d < hidden; d += blockDim.x)
+ out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d];
+}
+
+/* CLS pooling: gather the first token of each packed sequence. */
+__global__ static void
+first_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
+ int b = blockIdx.x;
+ int tid = threadIdx.x;
+ int first = offsets[b];
+ for (int d = tid; d < hidden; d += blockDim.x)
+ out[(size_t)b * hidden + d] = x[(size_t)first * hidden + d];
+}
+
+__global__ static void span_pool_kernel(
+ float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) {
+ int s = blockIdx.x;
+ int tid = threadIdx.x;
+ int start = starts[s];
+ int len = lens[s];
+ for (int d = tid; d < hidden; d += blockDim.x) {
+ float sum = 0.0f;
+ for (int t = 0; t < len; t++)
+ sum += x[(size_t)(start + t) * hidden + d];
+ out[(size_t)s * hidden + d] = sum / (float)len;
+ }
+}
+
+// BERT LayerNorm: mean-subtract, biased variance (/dim), scale by gamma, shift
+// by beta. One block per row, two block reductions before any write, so it is
+// safe in place (out == x). Matches the CPU embed_layer_norm.
+__global__ static void layer_norm_kernel(float *out,
+ const float *x,
+ const float *gamma,
+ const float *beta,
+ int rows,
+ int dim,
+ float eps) {
+ int row = blockIdx.x;
+ int tid = threadIdx.x;
+ const float *xr = x + (size_t)row * dim;
+ float s = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x)
+ s += xr[d];
+ float mean = block_sum(s) / (float)dim;
+ float vs = 0.0f;
+ for (int d = tid; d < dim; d += blockDim.x) {
+ float c = xr[d] - mean;
+ vs += c * c;
+ }
+ float inv = rsqrtf(block_sum(vs) / (float)dim + eps);
+ float *o = out + (size_t)row * dim;
+ for (int d = tid; d < dim; d += blockDim.x)
+ o[d] = (xr[d] - mean) * inv * gamma[d] + beta[d];
+}
+
+// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). The released encoders
+// use this; the tanh approximation below is for gelu_new / gelu_pytorch_tanh.
+__global__ static void gelu_kernel(float *x, size_t n) {
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) {
+ float v = x[i];
+ x[i] = 0.5f * v * (1.0f + erff(v * 0.70710678118654752f));
+ }
+}
+
+// Tanh-approximation GeLU in place: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))).
+__global__ static void gelu_tanh_kernel(float *x, size_t n) {
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) {
+ float v = x[i];
+ float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
+ x[i] = 0.5f * v * (1.0f + tanhf(inner));
+ }
+}
+
+// Add the learned absolute position embedding (indexed by the within-sequence
+// position) and the token-type[0] embedding to the token embeddings already in
+// x. token_type holds row 0 only; every input token has segment id 0.
+__global__ static void bert_pos_type_add_kernel(float *x,
+ const int *positions,
+ const float *pos_emb,
+ const float *token_type,
+ int total,
+ int hidden) {
+ size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ size_t count = (size_t)total * hidden;
+ if (idx >= count)
+ return;
+ int tok = (int)(idx / hidden);
+ int d = (int)(idx - (size_t)tok * hidden);
+ int pos = positions[tok];
+ x[idx] += pos_emb[(size_t)pos * hidden + d] + token_type[d];
+}
+
+static int launch_check(void) {
+ cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess) {
+ fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e));
+ return -1;
+ }
+ return 0;
+}
+
+// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm).
+// EMBED_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core
+// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands
+// internally.
+static cublasComputeType_t gemm_compute(void) {
+ static int env_init = 0;
+ if (!g_gemm_compute_set && !env_init) {
+ const char *e = getenv("EMBED_CUDA_GEMM_MODE");
+ if (e && parse_gemm_compute(e, &g_gemm_compute) == 0)
+ g_gemm_compute_set = 1;
+ env_init = 1;
+ }
+ return g_gemm_compute;
+}
+
+// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only
+// BF16 path materializes one F32 weight at a time into this buffer.
+static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) {
+ if (ctx->weight_f32_elems >= elems)
+ return 0;
+ cudaFree(ctx->weight_f32);
+ ctx->weight_f32 = NULL;
+ ctx->weight_f32_elems = 0;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->weight_f32, elems * sizeof(float)));
+ ctx->weight_f32_elems = elems;
+ return 0;
+}
+
+// y = w @ x (+ beta*y). With BF16 weights and an already-BF16 activation
+// (x_is_bf16 - the Qwen fused path) both inputs are BF16 with F32 accumulation.
+// For the BERT family the activation stays F32 and the BF16 weight is widened
+// back to F32 here, so the GEMM is exact (honoring gemm_compute): BF16 is then a
+// storage choice that costs no quality, which the post-norm LayerNorm needs.
+// With F32 weights this is the exact path.
+static int linear_ex(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const void *x,
+ int x_is_bf16,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim,
+ int x_stride,
+ float beta) {
+ const float alpha = 1.0f;
+ if (w->bf16) {
+ if (!x_is_bf16 && ctx->config.family == EMBED_FAMILY_BERT) {
+ size_t welems = (size_t)out_dim * (size_t)in_dim;
+ if (ensure_weight_f32(ctx, welems) != 0)
+ return -1;
+ int wt = 256;
+ cast_bf16_to_f32_kernel<<<(welems + wt - 1) / wt, wt, 0, ctx->stream>>>(
+ ctx->weight_f32, (const __nv_bfloat16 *)w->d, welems);
+ if (launch_check() != 0)
+ return -1;
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim,
+ &alpha, ctx->weight_f32, CUDA_R_32F, in_dim, x, CUDA_R_32F,
+ x_stride, &beta, y, CUDA_R_32F, out_dim, gemm_compute(),
+ CUBLAS_GEMM_DEFAULT));
+ return 0;
+ }
+ const void *xb = x;
+ if (!x_is_bf16) {
+ size_t n = (size_t)rows * (size_t)x_stride;
+ int threads = 256;
+ cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ (__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n);
+ if (launch_check() != 0)
+ return -1;
+ xb = ctx->act_bf16;
+ }
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim,
+ &alpha, w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride,
+ &beta, y, CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F,
+ CUBLAS_GEMM_DEFAULT));
+ return 0;
+ }
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha,
+ w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y,
+ CUDA_R_32F, out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT));
+ return 0;
+}
+
+static int linear(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const float *x,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim) {
+ return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f);
+}
+
+// x points at activations already stored as BF16 (e.g. ctx->act_bf16).
+static int linear_bf16x(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const void *x,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim,
+ int x_stride,
+ float beta) {
+ return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta);
+}
+
+static int linear_accum(embed_cuda_ctx_t *ctx,
+ const cuda_matrix_t *w,
+ const float *x,
+ float *y,
+ int rows,
+ int in_dim,
+ int out_dim) {
+ return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f);
+}
+
+static int gemm_strided_batched(cublasHandle_t blas,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m,
+ int n,
+ int k,
+ const float *alpha,
+ const float *A,
+ int lda,
+ long long strideA,
+ const float *B,
+ int ldb,
+ long long strideB,
+ const float *beta,
+ float *C,
+ int ldc,
+ long long strideC,
+ int batch_count) {
+ cublasComputeType_t ct = gemm_compute();
+ if (ct == CUBLAS_COMPUTE_32F) {
+ CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda,
+ strideA, B, ldb, strideB, beta, C, ldc, strideC,
+ batch_count));
+ } else {
+ CUBLAS_CHECK(cublasGemmStridedBatchedEx(
+ blas, transa, transb, m, n, k, alpha, A, CUDA_R_32F, lda, strideA, B, CUDA_R_32F, ldb,
+ strideB, beta, C, CUDA_R_32F, ldc, strideC, batch_count, ct, CUBLAS_GEMM_DEFAULT));
+ }
+ return 0;
+}
+
+// Attention for one micro-batch. Expands K/V to the per-query-head layout,
+// then for each sequence runs batched-over-heads Q@K^T (scaled) -> row softmax
+// -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and
+// ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets
+// is the host copy of the packed sequence boundaries.
+// Prepare the equal-length batched-attention path for one forward pass:
+// grow the score/probability scratch to hold attn_G sequences at once and
+// upload the pointer arrays for the two batched GEMMs (buffer addresses are
+// layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged
+// batches, single sequences, or when one sequence's scores exceed the
+// budget.
+static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) {
+ const embed_config_t *c = &ctx->config;
+ ctx->attn_G = 0;
+ if (batch < 2)
+ return 0;
+ int L = ctx->offsets_host[1] - ctx->offsets_host[0];
+ if (L > CUDA_ATTN_BATCHED_MAX_LEN)
+ return 0;
+ for (int b = 1; b < batch; b++)
+ if (ctx->offsets_host[b + 1] - ctx->offsets_host[b] != L)
+ return 0;
+ int H = c->n_heads, hd = c->head_dim, q_dim = c->q_dim;
+ long long per = (long long)H * L * L;
+ if (per <= 0 || per > CUDA_ATTN_SCORES_BUDGET / 2)
+ return 0;
+ int G = (int)(CUDA_ATTN_SCORES_BUDGET / per);
+ if (G > batch)
+ G = batch;
+ if (G < 2)
+ return 0;
+
+ int bf16 = g_weights_bf16;
+ long long want = (long long)G * per;
+ if (want > ctx->attn_scores_elems) {
+ cudaFree(ctx->attn_scores);
+ ctx->attn_scores = NULL;
+ ctx->attn_scores_elems = 0;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)want * sizeof(float)));
+ ctx->attn_scores_elems = want;
+ }
+ if (bf16 && want > ctx->attn_probs_elems) {
+ cudaFree(ctx->attn_probs);
+ ctx->attn_probs = NULL;
+ ctx->attn_probs_elems = 0;
+ CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)want * sizeof(__nv_bfloat16)));
+ ctx->attn_probs_elems = want;
+ }
+
+ int entries = 6 * batch * H;
+ if (entries > ctx->attn_ptrs_cap) {
+ cudaFree((void *)ctx->attn_ptrs);
+ free((void *)ctx->attn_ptrs_host);
+ ctx->attn_ptrs = NULL;
+ ctx->attn_ptrs_cap = 0;
+ ctx->attn_ptrs_host = (const void **)malloc((size_t)entries * sizeof(void *));
+ if (!ctx->attn_ptrs_host)
+ return -1;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_ptrs, (size_t)entries * sizeof(void *)));
+ ctx->attn_ptrs_cap = entries;
+ }
+
+ const void **K = ctx->attn_ptrs_host;
+ const void **Q = K + (size_t)batch * H;
+ const void **S = Q + (size_t)batch * H;
+ const void **V = S + (size_t)batch * H;
+ const void **P = V + (size_t)batch * H;
+ const void **O = P + (size_t)batch * H;
+ const __nv_bfloat16 *vb = (const __nv_bfloat16 *)ctx->vexp;
+ const __nv_bfloat16 *pb = (const __nv_bfloat16 *)ctx->attn_probs;
+ const __nv_bfloat16 *ab = (const __nv_bfloat16 *)ctx->act_bf16;
+ for (int b = 0; b < batch; b++) {
+ // Score/probability rows cycle every G sequences (chunked reuse).
+ long long srow = (long long)(b % G) * per;
+ size_t tok = (size_t)b * L;
+ for (int h = 0; h < H; h++) {
+ size_t i = (size_t)b * H + h;
+ K[i] = ctx->kexp + tok * q_dim + (size_t)h * hd;
+ Q[i] = ctx->qkv + tok * qkv_dim + q_offset + (size_t)h * hd;
+ S[i] = ctx->attn_scores + srow + (long long)h * L * L;
+ if (bf16) {
+ V[i] = vb + tok * q_dim + (size_t)h * hd;
+ P[i] = pb + srow + (long long)h * L * L;
+ O[i] = ab + tok * q_dim + (size_t)h * hd;
+ } else {
+ V[i] = ctx->vexp + tok * q_dim + (size_t)h * hd;
+ P[i] = S[i];
+ O[i] = ctx->attn_out + tok * q_dim + (size_t)h * hd;
+ }
+ }
+ }
+ CUDA_CHECK(cudaMemcpy((void *)ctx->attn_ptrs, ctx->attn_ptrs_host,
+ (size_t)entries * sizeof(void *), cudaMemcpyHostToDevice));
+ ctx->attn_G = G;
+ ctx->attn_L = L;
+ return 0;
+}
+
+// out_bf16 != NULL selects the BF16-weight layout: V and the softmax
+// probabilities are stored BF16 and the @V GEMM emits BF16 attention output
+// at out_bf16 (stride q_dim), ready for the wo projection without a cast.
+static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
+ const int *h_offsets,
+ int batch,
+ int q_offset,
+ int k_offset,
+ int v_offset,
+ int qkv_dim,
+ float scale,
+ __nv_bfloat16 *out_bf16) {
+ const embed_config_t *c = &ctx->config;
+ int q_dim = c->q_dim;
+ int hd = c->head_dim;
+ int H = c->n_heads;
+ int total = h_offsets[batch];
+ int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
+ const float alpha = scale, beta = 0.0f, one = 1.0f;
+
+ int threads = 256;
+ size_t exp_count = (size_t)total * q_dim;
+ if (out_bf16) {
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim,
+ k_offset, v_offset);
+ } else {
+ attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset,
+ v_offset);
+ }
+ if (launch_check() != 0)
+ return -1;
+
+ if (ctx->attn_G >= 2) {
+ int G = ctx->attn_G, L = ctx->attn_L;
+ const void *const *K = (const void *const *)ctx->attn_ptrs;
+ const void *const *Q = K + (size_t)batch * H;
+ void *const *S = (void *const *)(Q + (size_t)batch * H);
+ const void *const *V = (const void *const *)S + (size_t)batch * H;
+ const void *const *P = V + (size_t)batch * H;
+ void *const *O = (void *const *)(P + (size_t)batch * H);
+ cudaDataType pv = out_bf16 ? CUDA_R_16BF : CUDA_R_32F;
+ cublasComputeType_t av_ct = out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute();
+ for (int s = 0; s < batch; s += G) {
+ int g = batch - s < G ? batch - s : G;
+ int n = g * H;
+ size_t off = (size_t)s * H;
+ /* scores = scale * Q @ K^T, batched over (seq, head) */
+ CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha,
+ K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F,
+ qkv_dim, &beta, S + off, CUDA_R_32F, L, n,
+ gemm_compute(), CUBLAS_GEMM_DEFAULT));
+ if (out_bf16) {
+ if (causal)
+ attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
+ else
+ attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
+ ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
+ } else {
+ if (causal)
+ attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
+ else
+ attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
+ }
+ if (launch_check() != 0)
+ return -1;
+ /* O = P @ V, batched over (seq, head) */
+ CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one,
+ V + off, pv, q_dim, P + off, pv, L, &beta, O + off, pv,
+ q_dim, n, av_ct, CUBLAS_GEMM_DEFAULT));
+ }
+ return 0;
+ }
+
+ for (int b = 0; b < batch; b++) {
+ int start = h_offsets[b];
+ int L = h_offsets[b + 1] - start;
+ const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset;
+ const float *K = ctx->kexp + (size_t)start * q_dim;
+ /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */
+ if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim,
+ hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L,
+ H) != 0)
+ return -1;
+ if (out_bf16) {
+ __nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs;
+ const __nv_bfloat16 *V = (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim;
+ __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim;
+ if (causal)
+ attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L);
+ else
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L);
+ if (launch_check() != 0)
+ return -1;
+ /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
+ CUBLAS_CHECK(cublasGemmStridedBatchedEx(
+ ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, CUDA_R_16BF, q_dim, hd, P,
+ CUDA_R_16BF, L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, hd, H,
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ } else {
+ const float *V = ctx->vexp + (size_t)start * q_dim;
+ float *O = ctx->attn_out + (size_t)start * q_dim;
+ if (causal)
+ attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
+ else
+ attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
+ ctx->attn_scores, L);
+ if (launch_check() != 0)
+ return -1;
+ /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
+ if (gemm_strided_batched(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, q_dim,
+ hd, ctx->attn_scores, L, (long long)L * L, &beta, O, q_dim, hd,
+ H) != 0)
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) {
+ const embed_config_t *c = &ctx->config;
+ if (total > ctx->seq_cap) {
+ cudaFree(ctx->x);
+ cudaFree(ctx->x_norm);
+ cudaFree(ctx->qkv);
+ cudaFree(ctx->attn_out);
+ cudaFree(ctx->ffn_gate_up);
+ cudaFree(ctx->token_ids);
+ cudaFree(ctx->positions);
+ cudaFree(ctx->kexp);
+ cudaFree(ctx->vexp);
+ cudaFree(ctx->act_bf16);
+ ctx->x = ctx->x_norm = ctx->qkv = NULL;
+ ctx->attn_out = ctx->ffn_gate_up = NULL;
+ ctx->token_ids = ctx->positions = NULL;
+ ctx->kexp = ctx->vexp = NULL;
+ ctx->act_bf16 = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->qkv,
+ (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up,
+ (size_t)total * 2 * c->intermediate_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float)));
+ if (g_weights_bf16)
+ CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * 2 * c->intermediate_size *
+ sizeof(__nv_bfloat16)));
+ ctx->seq_cap = total;
+ }
+ if (batch + 1 > ctx->batch_cap) {
+ cudaFree(ctx->offsets);
+ free(ctx->offsets_host);
+ CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int)));
+ ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int));
+ if (!ctx->offsets_host)
+ return -1;
+ ctx->batch_cap = batch + 1;
+ }
+ if (max_seq > ctx->max_seq_cap) {
+ cudaFree(ctx->rope_cos);
+ cudaFree(ctx->rope_sin);
+ long long score_elems = (long long)c->n_heads * max_seq * max_seq;
+ if (score_elems > ctx->attn_scores_elems) {
+ cudaFree(ctx->attn_scores);
+ ctx->attn_scores = NULL;
+ ctx->attn_scores_elems = 0;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)score_elems * sizeof(float)));
+ ctx->attn_scores_elems = score_elems;
+ }
+ if (g_weights_bf16 && score_elems > ctx->attn_probs_elems) {
+ cudaFree(ctx->attn_probs);
+ ctx->attn_probs = NULL;
+ ctx->attn_probs_elems = 0;
+ CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)score_elems * sizeof(__nv_bfloat16)));
+ ctx->attn_probs_elems = score_elems;
+ }
+ float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
+ float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
+ if (!hcos || !hsin) {
+ free(hcos);
+ free(hsin);
+ return -1;
+ }
+ int half = c->head_dim / 2;
+ for (int pos = 0; pos < max_seq; pos++) {
+ for (int d = 0; d < half; d++) {
+ float inv = 1.0f / powf(c->rope_theta, (float)(2 * d) / (float)c->head_dim);
+ float angle = (float)pos * inv;
+ float cv = cosf(angle);
+ float sv = sinf(angle);
+ hcos[(size_t)pos * c->head_dim + d] = cv;
+ hcos[(size_t)pos * c->head_dim + half + d] = cv;
+ hsin[(size_t)pos * c->head_dim + d] = sv;
+ hsin[(size_t)pos * c->head_dim + half + d] = sv;
+ }
+ }
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->rope_cos, (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->rope_sin, (size_t)max_seq * c->head_dim * sizeof(float)));
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos,
+ (size_t)max_seq * c->head_dim * sizeof(float),
+ cudaMemcpyHostToDevice, ctx->stream));
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin,
+ (size_t)max_seq * c->head_dim * sizeof(float),
+ cudaMemcpyHostToDevice, ctx->stream));
+ free(hcos);
+ free(hsin);
+ ctx->max_seq_cap = max_seq;
+ }
+ return 0;
+}
+
+static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) {
+ if (rows <= ctx->pooled_rows_cap)
+ return 0;
+ const embed_config_t *c = &ctx->config;
+ cudaFree(ctx->pooled_out);
+ ctx->pooled_out = NULL;
+ CUDA_CHECK(
+ cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float)));
+ ctx->pooled_rows_cap = rows;
+ return 0;
+}
+
+static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) {
+ if (n_spans <= ctx->span_cap)
+ return 0;
+ cudaFree(ctx->span_starts);
+ cudaFree(ctx->span_lens);
+ ctx->span_starts = NULL;
+ ctx->span_lens = NULL;
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, (size_t)n_spans * sizeof(int)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, (size_t)n_spans * sizeof(int)));
+ ctx->span_cap = n_spans;
+ return 0;
+}
+
+static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
+ return load_qkv_matrix(&dst->qkv, src, c) ||
+ load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
+ (c->qkv_bias && load_qkv_bias(&dst->qkv_bias, src, c)) ||
+ (c->qk_norm && load_vector(&dst->q_norm, src->q_norm, c->head_dim)) ||
+ (c->qk_norm && load_vector(&dst->k_norm, src->k_norm, c->head_dim)) ||
+ load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
+ load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
+ load_gate_up_matrix(&dst->gate_up_proj, src, c) ||
+ load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size);
+}
+
+/* BERT layer: attention reuses the fused qkv matrix and qkv_bias (q_dim ==
+ * kv_dim == hidden); the two block LayerNorm weights load into input_norm
+ * (post-attention) and post_attn_norm (post-FFN); gate_up_proj holds the single
+ * up_proj. The added fields carry the biases the Qwen slots lack. */
+static int load_bert_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
+ return load_qkv_matrix(&dst->qkv, src, c) ||
+ load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
+ load_qkv_bias(&dst->qkv_bias, src, c) ||
+ load_vector(&dst->o_bias, src->o_bias, c->hidden_size) ||
+ load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
+ load_vector(&dst->attn_ln_bias, src->attn_ln_bias, c->hidden_size) ||
+ load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
+ load_vector(&dst->ffn_ln_bias, src->ffn_ln_bias, c->hidden_size) ||
+ load_matrix(&dst->gate_up_proj, &src->up_proj, c->intermediate_size, c->hidden_size) ||
+ load_vector(&dst->ffn_inter_bias, src->ffn_inter_bias, c->intermediate_size) ||
+ load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size) ||
+ load_vector(&dst->ffn_out_bias, src->ffn_out_bias, c->hidden_size);
+}
+
+static void free_layer(cuda_layer_t *l) {
+ cuda_matrix_free(&l->qkv);
+ cuda_matrix_free(&l->wo);
+ cudaFree(l->qkv_bias);
+ cudaFree(l->q_norm);
+ cudaFree(l->k_norm);
+ cudaFree(l->input_norm);
+ cudaFree(l->post_attn_norm);
+ cuda_matrix_free(&l->gate_up_proj);
+ cuda_matrix_free(&l->down_proj);
+ cudaFree(l->o_bias);
+ cudaFree(l->attn_ln_bias);
+ cudaFree(l->ffn_inter_bias);
+ cudaFree(l->ffn_out_bias);
+ cudaFree(l->ffn_ln_bias);
+}
+
+/* Build a device context from an already-loaded CPU model. own_cpu records
+ * whether embed_cuda_free should release that CPU model: embed_cuda_load owns
+ * the model it loaded; the late path passes a model owned by its CPU late
+ * model and sets own_cpu = 0. */
+static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) {
+ if (!cpu)
+ return NULL;
+
+ embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx));
+ if (!ctx) {
+ if (own_cpu)
+ embed_model_free(cpu);
+ return NULL;
+ }
+ ctx->cpu = cpu;
+ ctx->own_cpu = own_cpu;
+ ctx->config = cpu->config;
+
+ /* Without an explicit --cuda-weight-dtype, store weights in the model
+ * file's dtype: BF16 snapshots load as BF16 (bit-exact pass-through),
+ * F32 snapshots keep the exact F32 default. */
+ if (!g_weights_bf16_set) {
+ g_weights_bf16 = cpu->weights.embed_tokens.dtype == DTYPE_BF16;
+ if (g_weights_bf16)
+ fprintf(stderr, "cuda: BF16 model file; storing weights as BF16 "
+ "(use --cuda-weight-dtype f32 to override)\n");
+ }
+
+ if (cudaStreamCreate(&ctx->stream) != cudaSuccess ||
+ cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS ||
+ cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) {
+ embed_cuda_free(ctx);
+ return NULL;
+ }
+
+ const embed_config_t *c = &ctx->config;
+ ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers));
+ if (!ctx->layers) {
+ embed_cuda_free(ctx);
+ return NULL;
+ }
+ if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, c->vocab_size,
+ c->hidden_size) != 0) {
+ embed_cuda_free(ctx);
+ return NULL;
+ }
+ if (c->family == EMBED_FAMILY_BERT) {
+ /* BERT has no final norm; it adds learned position + token-type[0]
+ * embeddings and an embedding LayerNorm (only row 0 of token-type is
+ * ever used, so load just that row). */
+ const embed_weights_t *w = &cpu->weights;
+ if (load_vector(&ctx->position_embeddings, w->position_embeddings,
+ c->max_position_embeddings * c->hidden_size) != 0 ||
+ load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) !=
+ 0 ||
+ load_vector(&ctx->embed_ln_w, w->embed_ln_w, c->hidden_size) != 0 ||
+ load_vector(&ctx->embed_ln_b, w->embed_ln_b, c->hidden_size) != 0) {
+ embed_cuda_free(ctx);
+ return NULL;
+ }
+ } else if (load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) {
+ embed_cuda_free(ctx);
+ return NULL;
+ }
+ for (int i = 0; i < c->n_layers; i++) {
+ int rc = c->family == EMBED_FAMILY_BERT
+ ? load_bert_layer(&ctx->layers[i], &cpu->weights.layers[i], c)
+ : load_layer(&ctx->layers[i], &cpu->weights.layers[i], c);
+ if (rc != 0) {
+ fprintf(stderr, "cuda: failed to load layer %d\n", i);
+ embed_cuda_free(ctx);
+ return NULL;
+ }
+ }
+ cudaStreamSynchronize(ctx->stream);
+ return ctx;
+}
+
+embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) {
+ return cuda_ctx_from_model(embed_model_load(model_dir), 1);
+}
+
+void embed_cuda_free(embed_cuda_ctx_t *ctx) {
+ if (!ctx)
+ return;
+ cuda_matrix_free(&ctx->embed_tokens);
+ if (ctx->layers) {
+ for (int i = 0; i < ctx->config.n_layers; i++)
+ free_layer(&ctx->layers[i]);
+ free(ctx->layers);
+ }
+ cudaFree(ctx->norm);
+ cudaFree(ctx->position_embeddings);
+ cudaFree(ctx->token_type_embedding);
+ cudaFree(ctx->embed_ln_w);
+ cudaFree(ctx->embed_ln_b);
+ cudaFree(ctx->x);
+ cudaFree(ctx->x_norm);
+ cudaFree(ctx->qkv);
+ cudaFree(ctx->attn_out);
+ cudaFree(ctx->ffn_gate_up);
+ cudaFree(ctx->pooled_out);
+ cudaFree(ctx->rope_cos);
+ cudaFree(ctx->rope_sin);
+ cudaFree(ctx->token_ids);
+ cudaFree(ctx->offsets);
+ free(ctx->offsets_host);
+ cudaFree(ctx->positions);
+ cudaFree(ctx->kexp);
+ cudaFree(ctx->vexp);
+ cudaFree(ctx->act_bf16);
+ cudaFree(ctx->weight_f32);
+ cudaFree(ctx->attn_scores);
+ cudaFree(ctx->attn_probs);
+ cudaFree((void *)ctx->attn_ptrs);
+ free((void *)ctx->attn_ptrs_host);
+ cudaFree(ctx->span_starts);
+ cudaFree(ctx->span_lens);
+ if (ctx->blas)
+ cublasDestroy(ctx->blas);
+ if (ctx->stream)
+ cudaStreamDestroy(ctx->stream);
+ if (ctx->own_cpu)
+ embed_model_free(ctx->cpu);
+ free(ctx);
+}
+
+const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) {
+ return ctx ? &ctx->config : NULL;
+}
+
+// Pack inputs into the device scratch every forward shares: token ids, the
+// per-sequence positions 0..L-1, and the batch offsets. Grows scratch via
+// ensure_buffers. Returns the packed token count in *out_total, or -1 on error.
+static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int batch,
+ int *out_total) {
+ int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
+ if (!h_offsets)
+ return -1;
+ int total = 0, max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) {
+ free(h_offsets);
+ return -1;
+ }
+ h_offsets[b] = total;
+ total += inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
+ }
+ h_offsets[batch] = total;
+ int *h_ids = (int *)malloc((size_t)total * sizeof(int));
+ int *h_pos = (int *)malloc((size_t)total * sizeof(int));
+ if (!h_ids || !h_pos) {
+ free(h_offsets);
+ free(h_ids);
+ free(h_pos);
+ return -1;
+ }
+ for (int b = 0; b < batch; b++) {
+ int off = h_offsets[b];
+ memcpy(h_ids + off, inputs[b].ids, (size_t)inputs[b].n_tokens * sizeof(int));
+ for (int i = 0; i < inputs[b].n_tokens; i++)
+ h_pos[off + i] = i;
+ }
+ if (ensure_buffers(ctx, total, batch, max_seq) != 0) {
+ free(h_offsets);
+ free(h_ids);
+ free(h_pos);
+ return -1;
+ }
+ memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int));
+ cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, (size_t)total * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->positions, h_pos, (size_t)total * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ free(h_offsets);
+ free(h_ids);
+ free(h_pos);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce));
+ return -1;
+ }
+ *out_total = total;
+ return 0;
+}
+
+// BERT-family forward (post-norm encoder), mirroring the CPU forward_packed_bert.
+// Embeddings: token + learned position + token-type[0], then the embedding
+// LayerNorm. Each layer: QKV(+bias) from x directly -> bidirectional attention
+// -> output dense(+bias) -> residual -> LayerNorm -> dense(+bias) -> GeLU ->
+// dense(+bias) -> residual -> LayerNorm. No RoPE, no qk-norm, no final norm.
+// F32 activations: BERT checkpoints are F32, so the linear/linear_accum F32
+// entry points are used throughout (they still handle bf16 weights via an
+// internal cast, which keeps a future bf16 BERT correct if not yet optimized).
+static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) {
+ const embed_config_t *c = &ctx->config;
+ int total = 0;
+ if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0)
+ return -1;
+
+ int hidden = c->hidden_size;
+ int inter = c->intermediate_size;
+ int qkv_dim = c->q_dim + 2 * c->kv_dim;
+ int q_offset = 0, k_offset = c->q_dim, v_offset = c->q_dim + c->kv_dim;
+ float scale = 1.0f / sqrtf((float)c->head_dim);
+ float eps = c->layer_norm_eps;
+ int threads = 256;
+ int blocks_hidden = (total * hidden + threads - 1) / threads;
+
+ embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, hidden,
+ c->vocab_size);
+ if (launch_check() != 0)
+ return -1;
+ bert_pos_type_add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->positions, ctx->position_embeddings, ctx->token_type_embedding, total, hidden);
+ if (launch_check() != 0)
+ return -1;
+ layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->embed_ln_w,
+ ctx->embed_ln_b, total, hidden, eps);
+ if (launch_check() != 0)
+ return -1;
+
+ if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
+ return -1;
+
+ for (int layer = 0; layer < c->n_layers; layer++) {
+ cuda_layer_t *l = &ctx->layers[layer];
+
+ // Self-attention (post-norm): QKV read x directly, then +bias.
+ if (linear(ctx, &l->qkv, ctx->x, ctx->qkv, total, hidden, qkv_dim) != 0)
+ return -1;
+ int qkv_count = total * qkv_dim;
+ add_row_bias_kernel<<<(qkv_count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias,
+ total, qkv_dim);
+ if (launch_check() != 0)
+ return -1;
+ if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset,
+ qkv_dim, scale, NULL) != 0)
+ return -1;
+
+ // Output dense(+bias), residual into x, then post-attention LayerNorm.
+ if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, hidden) != 0)
+ return -1;
+ add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>(
+ ctx->x, l->o_bias, total, hidden);
+ if (launch_check() != 0)
+ return -1;
+ layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->input_norm,
+ l->attn_ln_bias, total, hidden, eps);
+ if (launch_check() != 0)
+ return -1;
+
+ // Feed-forward (post-norm): dense(+bias) -> GeLU -> dense(+bias).
+ if (linear(ctx, &l->gate_up_proj, ctx->x, ctx->ffn_gate_up, total, hidden, inter) != 0)
+ return -1;
+ int inter_count = total * inter;
+ add_row_bias_kernel<<<(inter_count + 255) / 256, 256, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, l->ffn_inter_bias, total, inter);
+ if (launch_check() != 0)
+ return -1;
+ if (c->ffn_act == EMBED_ACT_GELU_TANH)
+ gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, (size_t)inter_count);
+ else
+ gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
+ ctx->ffn_gate_up, (size_t)inter_count);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_accum(ctx, &l->down_proj, ctx->ffn_gate_up, ctx->x, total, inter, hidden) != 0)
+ return -1;
+ add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>(
+ ctx->x, l->ffn_out_bias, total, hidden);
+ if (launch_check() != 0)
+ return -1;
+ layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->post_attn_norm,
+ l->ffn_ln_bias, total, hidden, eps);
+ if (launch_check() != 0)
+ return -1;
+ }
+ // No final norm; embed_cuda_encode_batch pools ctx->x directly.
+ return 0;
+}
+
+static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) {
+ if (!ctx || !inputs || batch <= 0)
+ return -1;
+ const embed_config_t *c = &ctx->config;
+ if (c->family == EMBED_FAMILY_BERT)
+ return cuda_forward_batch_bert(ctx, inputs, batch);
+ int total = 0;
+ if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0)
+ return -1;
+
+ int threads = 256;
+ int blocks_hidden = (total * c->hidden_size + threads - 1) / threads;
+ embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, c->hidden_size,
+ c->vocab_size);
+ if (launch_check() != 0)
+ return -1;
+
+ float scale = 1.0f / sqrtf((float)c->head_dim);
+ int q_offset = 0;
+ int k_offset = c->q_dim;
+ int v_offset = c->q_dim + c->kv_dim;
+ int qkv_dim = c->q_dim + 2 * c->kv_dim;
+ // With BF16 weights every producer (norm, SiLU, attention @V GEMM)
+ // stores its result as BF16 directly into ctx->act_bf16, which the next
+ // projection GEMM consumes before any later producer reuses the buffer
+ // (single stream). No cast launches remain in the layer loop.
+ __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16;
+ if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
+ return -1;
+ for (int layer = 0; layer < c->n_layers; layer++) {
+ cuda_layer_t *l = &ctx->layers[layer];
+ int wbf16 = l->qkv.bf16;
+ if (wbf16) {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->input_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, total, c->hidden_size, qkv_dim,
+ c->hidden_size, 0.0f) != 0)
+ return -1;
+ } else {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm,
+ total, c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, total, c->hidden_size, qkv_dim) != 0)
+ return -1;
+ }
+
+ if (c->qkv_bias) {
+ int count = total * qkv_dim;
+ add_row_bias_kernel<<<(count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias,
+ total, qkv_dim);
+ if (launch_check() != 0)
+ return -1;
+ }
+
+ rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>(
+ ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total,
+ c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->qk_norm,
+ c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+
+ if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset,
+ qkv_dim, scale, wbf16 ? act : NULL) != 0)
+ return -1;
+
+ if (wbf16) {
+ if (linear_bf16x(ctx, &l->wo, act, ctx->x, total, c->q_dim, c->hidden_size, c->q_dim,
+ 1.0f) != 0)
+ return -1;
+ } else {
+ if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, c->hidden_size) !=
+ 0)
+ return -1;
+ }
+
+ int inter_count = total * c->intermediate_size;
+ if (wbf16) {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->post_attn_norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, total, c->hidden_size,
+ 2 * c->intermediate_size, c->hidden_size, 0.0f) != 0)
+ return -1;
+ silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0,
+ ctx->stream>>>(act, c->intermediate_size, ctx->ffn_gate_up,
+ total, c->intermediate_size);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, total, c->intermediate_size,
+ c->hidden_size, c->intermediate_size, 1.0f) != 0)
+ return -1;
+ } else {
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->post_attn_norm,
+ total, c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+ if (linear(ctx, &l->gate_up_proj, ctx->x_norm, ctx->ffn_gate_up, total, c->hidden_size,
+ 2 * c->intermediate_size) != 0)
+ return -1;
+ silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0,
+ ctx->stream>>>(ctx->ffn_gate_up, 2 * c->intermediate_size,
+ ctx->ffn_gate_up, total, c->intermediate_size);
+ if (launch_check() != 0)
+ return -1;
+ if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, total,
+ c->intermediate_size, c->hidden_size, 2 * c->intermediate_size,
+ 1.0f) != 0)
+ return -1;
+ }
+ }
+
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total,
+ c->hidden_size, c->rms_norm_eps);
+ if (launch_check() != 0)
+ return -1;
+
+ return 0;
+}
+
+int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
+ const embed_config_t *c = &ctx->config;
+ if (cuda_forward_batch(ctx, inputs, batch) != 0)
+ return -1;
+ if (ensure_pooled_rows(ctx, batch) != 0)
+ return -1;
+ if (c->pooling_mode == EMBED_POOL_LAST_TOKEN)
+ last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
+ batch, c->hidden_size);
+ else if (c->pooling_mode == EMBED_POOL_CLS)
+ first_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
+ batch, c->hidden_size);
+ else
+ mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
+ batch, c->hidden_size);
+ if (launch_check() != 0)
+ return -1;
+ cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
+ (size_t)batch * c->hidden_size * sizeof(float),
+ cudaMemcpyDeviceToHost, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce));
+ return -1;
+ }
+ if (c->normalize_embeddings) {
+ for (int b = 0; b < batch; b++) {
+ if (embed_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) !=
+ 0)
+ return -1;
+ }
+ }
+ return 0;
+}
+
+int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
+ const embed_config_t *c = &ctx->config;
+ int total_spans = 0;
+ int total_tokens = 0;
+ for (int b = 0; b < batch; b++) {
+ const embed_context_input_t *input = &inputs[b];
+ int n_tokens = input->input.n_tokens;
+ if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
+ total_spans > INT_MAX - input->n_spans || total_tokens > INT_MAX - n_tokens)
+ return -1;
+ for (int s = 0; s < input->n_spans; s++) {
+ int start = input->spans[s].start;
+ int len = input->spans[s].n_tokens;
+ if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start)
+ return -1;
+ }
+ total_spans += input->n_spans;
+ total_tokens += n_tokens;
+ }
+
+ embed_input_t *packed = (embed_input_t *)malloc((size_t)batch * sizeof(*packed));
+ int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int));
+ int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int));
+ if (!packed || !h_starts || !h_lens) {
+ free(packed);
+ free(h_starts);
+ free(h_lens);
+ return -1;
+ }
+
+ int token_offset = 0;
+ int span_offset = 0;
+ for (int b = 0; b < batch; b++) {
+ const embed_context_input_t *input = &inputs[b];
+ packed[b] = input->input;
+ for (int s = 0; s < input->n_spans; s++) {
+ h_starts[span_offset] = token_offset + input->spans[s].start;
+ h_lens[span_offset] = input->spans[s].n_tokens;
+ span_offset++;
+ }
+ token_offset += input->input.n_tokens;
+ }
+
+ int rc = -1;
+ cudaError_t ce;
+ if (cuda_forward_batch(ctx, packed, batch) != 0)
+ goto cleanup;
+ if (ensure_pooled_rows(ctx, total_spans) != 0 || ensure_span_buffers(ctx, total_spans) != 0)
+ goto cleanup;
+ ce = cudaMemcpyAsync(ctx->span_starts, h_starts, (size_t)total_spans * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaMemcpyAsync(ctx->span_lens, h_lens, (size_t)total_spans * sizeof(int),
+ cudaMemcpyHostToDevice, ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce));
+ goto cleanup;
+ }
+ span_pool_kernel<<<total_spans, 256, 0, ctx->stream>>>(
+ ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, total_spans, c->hidden_size);
+ if (launch_check() != 0)
+ goto cleanup;
+ ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
+ (size_t)total_spans * c->hidden_size * sizeof(float),
+ cudaMemcpyDeviceToHost, ctx->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(ctx->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda span output copy failed: %s\n", cudaGetErrorString(ce));
+ goto cleanup;
+ }
+ rc = 0;
+
+cleanup:
+ free(h_lens);
+ free(h_starts);
+ free(packed);
+ return rc;
+}
+
+int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ embed_input_t input = {token_ids, n_tokens};
+ return embed_cuda_encode_batch(ctx, &input, 1, out_embedding);
+}
+
+float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) {
+ if (!ctx)
+ return NULL;
+ int hidden = ctx->config.hidden_size;
+ float *out = (float *)malloc((size_t)hidden * sizeof(float));
+ if (!out)
+ return NULL;
+ if (embed_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) {
+ free(out);
+ return NULL;
+ }
+ return out;
+}
+
+/* ========================================================================
+ * Late-interaction CUDA path
+ *
+ * The transformer forward and the 1_Dense projection run on the GPU; MaxSim
+ * scoring stays on the host, where the grouped-GEMM scorer is far faster than a
+ * device graph for this small op. So the encoders copy the projected token
+ * vectors back to host memory and the caller runs embed_late_maxsim_batch there.
+ * ======================================================================== */
+
+struct embed_cuda_late_ctx {
+ embed_cuda_ctx_t *base; // device base model; does not own its CPU model
+ embed_late_model_t *cpu_late; // owns the CPU base model + projection host data
+ cuda_matrix_t projection; // device [token_dim, hidden]
+ float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand
+ size_t proj_cap; // capacity of proj_dev in floats
+ int token_dim;
+};
+
+embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir) {
+ embed_late_model_t *cpu_late = embed_late_model_load(model_dir);
+ if (!cpu_late)
+ return NULL;
+
+ embed_cuda_late_ctx_t *ctx = (embed_cuda_late_ctx_t *)calloc(1, sizeof(*ctx));
+ if (!ctx) {
+ embed_late_model_free(cpu_late);
+ return NULL;
+ }
+ ctx->cpu_late = cpu_late;
+ ctx->token_dim = embed_late_model_token_dim(cpu_late);
+
+ // Upload the base transformer weights; the CPU late model keeps ownership.
+ ctx->base = cuda_ctx_from_model(embed_late_model_base(cpu_late), 0);
+ if (!ctx->base) {
+ embed_cuda_late_free(ctx);
+ return NULL;
+ }
+
+ // Upload the 1_Dense projection [token_dim, hidden].
+ const embed_weight_ref_t *proj = embed_late_model_projection(cpu_late);
+ if (!proj || ctx->token_dim <= 0 ||
+ load_matrix(&ctx->projection, proj, ctx->token_dim, ctx->base->config.hidden_size) != 0) {
+ embed_cuda_late_free(ctx);
+ return NULL;
+ }
+ return ctx;
+}
+
+void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx) {
+ if (!ctx)
+ return;
+ cuda_matrix_free(&ctx->projection);
+ cudaFree(ctx->proj_dev);
+ embed_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model
+ embed_late_model_free(ctx->cpu_late); // frees the CPU base model + projection
+ free(ctx);
+}
+
+const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx) {
+ return ctx ? &ctx->base->config : NULL;
+}
+
+int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
+
+static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) {
+ size_t need = (size_t)total * (size_t)ctx->token_dim;
+ if (need <= ctx->proj_cap)
+ return 0;
+ cudaFree(ctx->proj_dev);
+ ctx->proj_dev = NULL;
+ ctx->proj_cap = 0;
+ if (cudaMalloc((void **)&ctx->proj_dev, need * sizeof(float)) != cudaSuccess)
+ return -1;
+ ctx->proj_cap = need;
+ return 0;
+}
+
+/* Forward all inputs, project every token to token_dim, and copy the packed
+ * [total, token_dim] F32 result to a freshly malloc'd host buffer (caller frees).
+ * On success *proj_host and *total_out are set; base->offsets_host holds the
+ * packed per-document token offsets. */
+static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int n_docs,
+ float **proj_host,
+ int *total_out) {
+ embed_cuda_ctx_t *base = ctx->base;
+ const embed_config_t *c = &base->config;
+ if (cuda_forward_batch(base, inputs, n_docs) != 0)
+ return -1;
+ int total = base->offsets_host[n_docs];
+ if (total <= 0 || ensure_late_proj(ctx, total) != 0)
+ return -1;
+ if (linear(base, &ctx->projection, base->x, ctx->proj_dev, total, c->hidden_size,
+ ctx->token_dim) != 0)
+ return -1;
+ float *host = (float *)malloc((size_t)total * (size_t)ctx->token_dim * sizeof(float));
+ if (!host)
+ return -1;
+ cudaError_t ce =
+ cudaMemcpyAsync(host, ctx->proj_dev, (size_t)total * (size_t)ctx->token_dim * sizeof(float),
+ cudaMemcpyDeviceToHost, base->stream);
+ if (ce == cudaSuccess)
+ ce = cudaStreamSynchronize(base->stream);
+ if (ce != cudaSuccess) {
+ fprintf(stderr, "cuda late output copy failed: %s\n", cudaGetErrorString(ce));
+ free(host);
+ return -1;
+ }
+ *proj_host = host;
+ *total_out = total;
+ return 0;
+}
+
+int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
+ if (!ctx || !token_ids || n_tokens <= 0 || !out_vectors || ctx->token_dim <= 0)
+ return -1;
+ embed_input_t input = {token_ids, n_tokens};
+ float *proj_host = NULL;
+ int total = 0;
+ if (cuda_late_forward_project(ctx, &input, 1, &proj_host, &total) != 0)
+ return -1;
+ memcpy(out_vectors, proj_host, (size_t)total * (size_t)ctx->token_dim * sizeof(float));
+ free(proj_host);
+ if (normalize) {
+ for (int i = 0; i < total; i++)
+ if (embed_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0)
+ return -1;
+ }
+ return 0;
+}
+
+int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets) {
+ if (!ctx || !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors ||
+ !out_offsets || ctx->token_dim <= 0)
+ return -1;
+ int dim = ctx->token_dim;
+
+ embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
+ if (!inputs)
+ return -1;
+
+ int total_keep = 0;
+ out_offsets[0] = 0;
+ for (int d = 0; d < n_docs; d++) {
+ if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
+ n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
+ free(inputs);
+ return -1;
+ }
+ for (int k = 0; k < n_keep[d]; k++) {
+ if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
+ free(inputs);
+ return -1;
+ }
+ }
+ inputs[d].ids = doc_ids[d];
+ inputs[d].n_tokens = n_tokens[d];
+ total_keep += n_keep[d];
+ out_offsets[d + 1] = total_keep;
+ }
+
+ float *proj_host = NULL;
+ int total = 0;
+ if (cuda_late_forward_project(ctx, inputs, n_docs, &proj_host, &total) != 0) {
+ free(inputs);
+ return -1;
+ }
+
+ /* Gather each document's kept token rows (in packed forward order) into the
+ * output, the layout MaxSim consumes - matching the CPU/MLX paths. */
+ const int *off = ctx->base->offsets_host;
+ int pos = 0;
+ for (int d = 0; d < n_docs; d++) {
+ int start = off[d];
+ for (int k = 0; k < n_keep[d]; k++) {
+ memcpy(out_vectors + (size_t)pos * dim, proj_host + (size_t)(start + keep[d][k]) * dim,
+ (size_t)dim * sizeof(float));
+ pos++;
+ }
+ }
+ free(proj_host);
+ free(inputs);
+
+ if (normalize) {
+ for (int i = 0; i < total_keep; i++)
+ if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ return -1;
+ }
+ return 0;
+}