diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /src/cuda.cu | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'src/cuda.cu')
| -rw-r--r-- | src/cuda.cu | 2052 |
1 files changed, 2052 insertions, 0 deletions
diff --git a/src/cuda.cu b/src/cuda.cu new file mode 100644 index 0000000..6ce7dfb --- /dev/null +++ b/src/cuda.cu @@ -0,0 +1,2052 @@ +extern "C" { +#include "cuda.h" +#include "internal.h" +} + +#include <cuda_runtime.h> +#include <cuda_bf16.h> +#include <cublas_v2.h> + +#include <limits.h> +#include <math.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +typedef struct { + float *d; // device buffer; holds __nv_bfloat16 elements when bf16 + int rows; + int cols; + int bf16; // 1 if d holds __nv_bfloat16, 0 if F32 +} cuda_matrix_t; + +typedef struct { + cuda_matrix_t qkv, wo; + float *qkv_bias; + float *q_norm; + float *k_norm; + float *input_norm; + float *post_attn_norm; + cuda_matrix_t gate_up_proj, down_proj; + /* BERT family (NULL for Qwen3). The two block LayerNorm weights reuse + * input_norm (post-attention) and post_attn_norm (post-FFN); gate_up_proj + * holds the single BERT up_proj. These add the biases those slots lack. */ + float *o_bias; + float *attn_ln_bias; + float *ffn_inter_bias; + float *ffn_out_bias; + float *ffn_ln_bias; +} cuda_layer_t; + +struct embed_cuda_ctx { + embed_model_t *cpu; + int own_cpu; // free ctx->cpu in embed_cuda_free only when this context owns it + embed_config_t config; + cudaStream_t stream; + cublasHandle_t blas; + cuda_matrix_t embed_tokens; + cuda_layer_t *layers; + float *norm; + /* BERT family (NULL for Qwen3): learned absolute position embeddings, the + * token-type[0] embedding (row 0 only, [hidden]), and the embedding + * LayerNorm weight/bias. */ + float *position_embeddings; + float *token_type_embedding; + float *embed_ln_w; + float *embed_ln_b; + + int seq_cap; + int batch_cap; + int max_seq_cap; + float *x; + float *x_norm; + float *qkv; + float *attn_out; + float *ffn_gate_up; + void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights) + float *weight_f32; // F32 widen scratch for one BF16 weight (memory-only bf16) + size_t weight_f32_elems; + float *pooled_out; + float *rope_cos; + float *rope_sin; + int *token_ids; + int *offsets; + int *offsets_host; + int *positions; + float *kexp; + float *vexp; + float *attn_scores; + void *attn_probs; // BF16 softmax output (bf16 weights) + long long attn_scores_elems; + long long attn_probs_elems; + // Equal-length micro-batch attention: pointer arrays for batched GEMMs, + // built once per forward pass (layer-invariant buffer addresses). + // attn_G is the number of sequences whose score tensors fit the scratch + // budget at once; 0 disables the batched path for this forward. + const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads + const void **attn_ptrs_host; + int attn_ptrs_cap; // capacity in pointer entries + int attn_G; + int attn_L; + int *span_starts; + int *span_lens; + int pooled_rows_cap; + int span_cap; +}; + +// Scratch budget for the batched-attention score tensors (F32 elements). +enum { CUDA_ATTN_SCORES_BUDGET = 64 * 1024 * 1024 }; +// Above this sequence length the per-sequence strided-batched loop beats the +// pointer-array batched GEMMs (L4: batched wins <=80tok, ties ~140, loses +// >=256; both paths are identical precision). +enum { CUDA_ATTN_BATCHED_MAX_LEN = 192 }; + +#define CUDA_CHECK(expr) \ + do { \ + cudaError_t _e = (expr); \ + if (_e != cudaSuccess) { \ + fprintf(stderr, "cuda: %s failed: %s\n", #expr, cudaGetErrorString(_e)); \ + return -1; \ + } \ + } while (0) + +#define CUBLAS_CHECK(expr) \ + do { \ + cublasStatus_t _s = (expr); \ + if (_s != CUBLAS_STATUS_SUCCESS) { \ + fprintf(stderr, "cuda: %s failed: cublas status %d\n", #expr, (int)_s); \ + return -1; \ + } \ + } while (0) + +static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) { + if (!mode || !mode[0] || !strcmp(mode, "f32")) { + *out = CUBLAS_COMPUTE_32F; + return 0; + } + if (!strcmp(mode, "tf32")) { + *out = CUBLAS_COMPUTE_32F_FAST_TF32; + return 0; + } + if (!strcmp(mode, "bf16")) { + *out = CUBLAS_COMPUTE_32F_FAST_16BF; + return 0; + } + if (!strcmp(mode, "16f")) { + *out = CUBLAS_COMPUTE_32F_FAST_16F; + return 0; + } + return -1; +} + +static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F; +static int g_gemm_compute_set = 0; + +int embed_cuda_set_fast_gemm(const char *mode) { + cublasComputeType_t ct; + if (parse_gemm_compute(mode, &ct) != 0) + return -1; + g_gemm_compute = ct; + g_gemm_compute_set = 1; + return 0; +} + +static void cuda_matrix_free(cuda_matrix_t *m) { + if (!m) + return; + cudaFree(m->d); + memset(m, 0, sizeof(*m)); +} + +static float bf16_to_f32(uint16_t v) { + uint32_t u = ((uint32_t)v) << 16; + float f; + memcpy(&f, &u, sizeof(f)); + return f; +} + +// Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are +// preserved. BF16 shares F32's 8-bit exponent, so this never overflows. +static uint16_t f32_to_bf16(float f) { + uint32_t x; + memcpy(&x, &f, sizeof(x)); + if ((x & 0x7fffffffu) > 0x7f800000u) // NaN + return (uint16_t)((x >> 16) | 0x0040u); + x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even + return (uint16_t)(x >> 16); +} + +// Store weights as native BF16 on the device when set (halves weight memory and +// feeds BF16 tensor cores); default keeps exact F32 for F32 model files. When +// the flag was never set explicitly, the loader follows the model file dtype, +// matching the CPU and MLX backends. +static int g_weights_bf16 = 0; +static int g_weights_bf16_set = 0; + +int embed_cuda_set_weights_bf16(int on) { + g_weights_bf16 = on ? 1 : 0; + g_weights_bf16_set = 1; + return 0; +} + +static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, size_t count) { + if (!dst || !w || !w->data) + return -1; + if (w->dtype == DTYPE_F32) { + memcpy(dst, w->data, count * sizeof(float)); + return 0; + } + if (w->dtype == DTYPE_BF16) { + const uint16_t *src = (const uint16_t *)w->data; + for (size_t i = 0; i < count; i++) + dst[i] = bf16_to_f32(src[i]); + return 0; + } + return -1; +} + +// Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is +// set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16. +static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, int rows, int cols) { + cudaError_t e; + if (g_weights_bf16) { + uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t)); + if (!tmp16) + return -1; + for (size_t i = 0; i < count; i++) + tmp16[i] = f32_to_bf16(src[i]); + e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e)); + free(tmp16); + return -1; + } + e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), cudaMemcpyHostToDevice); + free(tmp16); + m->bf16 = 1; + } else { + e = cudaMalloc((void **)&m->d, count * sizeof(float)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e)); + return -1; + } + e = cudaMemcpy(m->d, src, count * sizeof(float), cudaMemcpyHostToDevice); + m->bf16 = 0; + } + if (e != cudaSuccess) { + fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e)); + cuda_matrix_free(m); + return -1; + } + m->rows = rows; + m->cols = cols; + return 0; +} + +static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, int cols) { + if (!m || !w || rows <= 0 || cols <= 0) + return -1; + size_t count = (size_t)rows * (size_t)cols; + float *tmp = (float *)malloc(count * sizeof(float)); + if (!tmp) + return -1; + if (copy_weight_host_f32(tmp, w, count) != 0) { + free(tmp); + return -1; + } + int r = upload_weight(m, tmp, count, rows, cols); + free(tmp); + return r; +} + +static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) { + if (!m || !src || !c) + return -1; + int rows = c->q_dim + 2 * c->kv_dim; + int cols = c->hidden_size; + size_t q_count = (size_t)c->q_dim * cols; + size_t kv_count = (size_t)c->kv_dim * cols; + size_t total = q_count + 2 * kv_count; + float *tmp = (float *)malloc(total * sizeof(float)); + if (!tmp) + return -1; + if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 || + copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 || + copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) { + free(tmp); + return -1; + } + int r = upload_weight(m, tmp, total, rows, cols); + free(tmp); + return r; +} + +static int +load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) { + if (!m || !src || !c) + return -1; + int rows = 2 * c->intermediate_size; + int cols = c->hidden_size; + size_t proj_count = (size_t)c->intermediate_size * cols; + size_t total = 2 * proj_count; + float *tmp = (float *)malloc(total * sizeof(float)); + if (!tmp) + return -1; + if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 || + copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) { + free(tmp); + return -1; + } + int r = upload_weight(m, tmp, total, rows, cols); + free(tmp); + return r; +} + +static int load_vector(float **out, const float *src, int n) { + if (!out || !src || n <= 0) + return -1; + cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e)); + return -1; + } + e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e)); + cudaFree(*out); + *out = NULL; + return -1; + } + return 0; +} + +static int load_qkv_bias(float **out, const embed_layer_t *src, const embed_config_t *c) { + int n = c->q_dim + 2 * c->kv_dim; + float *host = (float *)malloc((size_t)n * sizeof(float)); + if (!host) + return -1; + memcpy(host, src->q_bias, (size_t)c->q_dim * sizeof(float)); + memcpy(host + c->q_dim, src->k_bias, (size_t)c->kv_dim * sizeof(float)); + memcpy(host + c->q_dim + c->kv_dim, src->v_bias, (size_t)c->kv_dim * sizeof(float)); + int rc = load_vector(out, host, n); + free(host); + return rc; +} + +__global__ static void add_row_bias_kernel(float *x, const float *bias, int rows, int dim) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int count = rows * dim; + if (idx < count) + x[idx] += bias[idx % dim]; +} + +__global__ static void embed_lookup_kernel( + float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int count = total * hidden; + if (idx >= count) + return; + int tok = idx / hidden; + int d = idx - tok * hidden; + int id = ids[tok]; + if (id < 0 || id >= vocab) { + x[idx] = 0.0f; + return; + } + size_t off = (size_t)id * hidden + d; + x[idx] = + emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) : ((const float *)emb)[off]; +} + +// Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs). +__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, const float *in, size_t n) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + out[i] = __float2bfloat16(in[i]); +} + +// Widen n BF16 values to F32 (used to run an exact F32 GEMM on BF16-stored +// weights - memory-only BF16, no precision loss from the operands). +__global__ static void cast_bf16_to_f32_kernel(float *out, const __nv_bfloat16 *in, size_t n) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + out[i] = __bfloat162float(in[i]); +} + +__device__ static float block_sum(float v) { + __shared__ float partial[8]; + __shared__ float total; + int lane = threadIdx.x & 31; + int warp = threadIdx.x >> 5; + int warps = blockDim.x >> 5; + + for (int offset = 16; offset > 0; offset >>= 1) + v += __shfl_down_sync(0xffffffff, v, offset); + if (lane == 0) + partial[warp] = v; + __syncthreads(); + + if (warp == 0) { + v = lane < warps ? partial[lane] : 0.0f; + for (int offset = 16; offset > 0; offset >>= 1) + v += __shfl_down_sync(0xffffffff, v, offset); + if (lane == 0) + total = v; + } + __syncthreads(); + return total; +} + +__device__ static inline void store_act(float *out, size_t i, float v) { out[i] = v; } + +__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) { + out[i] = __float2bfloat16(v); +} + +// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight +// projection GEMM directly (same rounding as a separate cast kernel). +template <typename OUT_T> +__global__ static void +rms_norm_kernel(OUT_T *out, const float *x, const float *weight, int rows, int dim, float eps) { + int row = blockIdx.x; + int tid = threadIdx.x; + float sum = 0.0f; + for (int d = tid; d < dim; d += blockDim.x) { + float v = x[(size_t)row * dim + d]; + sum += v * v; + } + float total = block_sum(sum); + float inv = rsqrtf(total / (float)dim + eps); + for (int d = tid; d < dim; d += blockDim.x) + store_act(out, (size_t)row * dim + d, x[(size_t)row * dim + d] * inv * weight[d]); +} + +// One grid covers Q and K: blockIdx.y < n_heads selects the Q head with +// q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset. +__global__ static void rms_norm_rope_qk_kernel(float *x, + const float *q_norm, + const float *k_norm, + const int *positions, + const float *cosv, + const float *sinv, + int rows, + int n_heads, + int n_kv_heads, + int head_dim, + int row_stride, + int q_offset, + int k_offset, + int qk_norm, + float eps) { + __shared__ float partial[4]; + __shared__ float total; + int row = blockIdx.x; + int head = blockIdx.y; + int tid = threadIdx.x; + int lane = tid & 31; + int warp = tid >> 5; + int half = head_dim / 2; + int pair_d = tid < half ? tid + half : tid - half; + float sign = tid < half ? -1.0f : 1.0f; + const float *weight = head < n_heads ? q_norm : k_norm; + int base_offset = head < n_heads ? q_offset : k_offset; + if (head >= n_heads) + head -= n_heads; + size_t base = (size_t)row * row_stride + base_offset + head * head_dim; + float v = tid < head_dim ? x[base + tid] : 0.0f; + float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f; + if (qk_norm) { + float sum = v * v; + for (int offset = 16; offset > 0; offset >>= 1) + sum += __shfl_down_sync(0xffffffff, sum, offset); + if (lane == 0) + partial[warp] = sum; + __syncthreads(); + if (warp == 0) { + sum = lane < 4 ? partial[lane] : 0.0f; + for (int offset = 16; offset > 0; offset >>= 1) + sum += __shfl_down_sync(0xffffffff, sum, offset); + if (lane == 0) + total = sum; + } + __syncthreads(); + } + + if (tid < head_dim) { + int pos = positions[row]; + float c = cosv[(size_t)pos * head_dim + tid]; + float s = sinv[(size_t)pos * head_dim + tid]; + float a = v; + float b = pair_v; + if (qk_norm) { + float inv = rsqrtf(total / (float)head_dim + eps); + a *= inv * weight[tid]; + b *= inv * weight[pair_d]; + } + x[base + tid] = a * c + sign * b * s; + } +} + +// Writes SiLU(gate)*up to out with out_stride. The F32 path writes in place +// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16 +// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM. +template <typename OUT_T> +__global__ static void silu_mul_packed_kernel( + OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + int n = rows * intermediate; + if (i < n) { + int row = i / intermediate; + int d = i - row * intermediate; + size_t base = (size_t)row * (2 * intermediate); + float g = gate_up[base + d]; + float u = gate_up[base + intermediate + d]; + store_act(out, (size_t)row * out_stride + d, (g / (1.0f + expf(-g))) * u); + } +} + +// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim] +// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times) +// so the attention GEMMs can stride uniformly over query heads. +// V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly. +template <typename V_T> +__global__ static void attn_expand_kv_kernel(float *kexp, + V_T *vexp, + const float *qkv, + int total, + int n_heads, + int n_kv_heads, + int head_dim, + int qkv_dim, + int k_offset, + int v_offset) { + int q_dim = n_heads * head_dim; + size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + size_t count = (size_t)total * q_dim; + if (idx >= count) + return; + int t = (int)(idx / q_dim); + int rem = (int)(idx - (size_t)t * q_dim); + int h = rem / head_dim; + int e = rem - h * head_dim; + int kv = h / (n_heads / n_kv_heads); + const float *base = qkv + (size_t)t * qkv_dim; + kexp[idx] = base[k_offset + kv * head_dim + e]; + store_act(vexp, idx, base[v_offset + kv * head_dim + e]); +} + +// Row softmax over the key dimension for one sequence's score tensor laid out +// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i). +// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM. +// P_T is float with probs == scores (in place), or __nv_bfloat16 when the +// probabilities feed a BF16 @V GEMM. +template <typename P_T> +__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) { + float *s = scores + (size_t)blockIdx.x * L; + P_T *p = probs + (size_t)blockIdx.x * L; + int tid = threadIdx.x; + __shared__ float red[256]; + + float m = -3.402823466e+38F; + for (int j = tid; j < L; j += blockDim.x) + m = fmaxf(m, s[j]); + red[tid] = m; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) + red[tid] = fmaxf(red[tid], red[tid + o]); + __syncthreads(); + } + m = red[0]; + __syncthreads(); + + float sum = 0.0f; + for (int j = tid; j < L; j += blockDim.x) { + float e = __expf(s[j] - m); + s[j] = e; + sum += e; + } + red[tid] = sum; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) + red[tid] += red[tid + o]; + __syncthreads(); + } + float inv = 1.0f / red[0]; + for (int j = tid; j < L; j += blockDim.x) + store_act(p, (size_t)j, s[j] * inv); +} + +template <typename P_T> +__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, int L) { + float *s = scores + (size_t)blockIdx.x * L; + P_T *p = probs + (size_t)blockIdx.x * L; + int query = blockIdx.x % L; + int keys = query + 1; + int tid = threadIdx.x; + __shared__ float red[256]; + + float m = -3.402823466e+38F; + for (int j = tid; j < keys; j += blockDim.x) + m = fmaxf(m, s[j]); + red[tid] = m; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) + red[tid] = fmaxf(red[tid], red[tid + o]); + __syncthreads(); + } + m = red[0]; + __syncthreads(); + + float sum = 0.0f; + for (int j = tid; j < keys; j += blockDim.x) { + float e = __expf(s[j] - m); + s[j] = e; + sum += e; + } + red[tid] = sum; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) + red[tid] += red[tid + o]; + __syncthreads(); + } + float inv = 1.0f / red[0]; + for (int j = tid; j < L; j += blockDim.x) + store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f); +} + +__global__ static void +mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) { + int b = blockIdx.x; + int tid = threadIdx.x; + int start = offsets[b]; + int end = offsets[b + 1]; + int len = end - start; + for (int d = tid; d < hidden; d += blockDim.x) { + float sum = 0.0f; + for (int t = start; t < end; t++) + sum += x[(size_t)t * hidden + d]; + out[(size_t)b * hidden + d] = sum / (float)len; + } +} + +__global__ static void +last_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) { + int b = blockIdx.x; + int tid = threadIdx.x; + int last = offsets[b + 1] - 1; + for (int d = tid; d < hidden; d += blockDim.x) + out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d]; +} + +/* CLS pooling: gather the first token of each packed sequence. */ +__global__ static void +first_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) { + int b = blockIdx.x; + int tid = threadIdx.x; + int first = offsets[b]; + for (int d = tid; d < hidden; d += blockDim.x) + out[(size_t)b * hidden + d] = x[(size_t)first * hidden + d]; +} + +__global__ static void span_pool_kernel( + float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) { + int s = blockIdx.x; + int tid = threadIdx.x; + int start = starts[s]; + int len = lens[s]; + for (int d = tid; d < hidden; d += blockDim.x) { + float sum = 0.0f; + for (int t = 0; t < len; t++) + sum += x[(size_t)(start + t) * hidden + d]; + out[(size_t)s * hidden + d] = sum / (float)len; + } +} + +// BERT LayerNorm: mean-subtract, biased variance (/dim), scale by gamma, shift +// by beta. One block per row, two block reductions before any write, so it is +// safe in place (out == x). Matches the CPU embed_layer_norm. +__global__ static void layer_norm_kernel(float *out, + const float *x, + const float *gamma, + const float *beta, + int rows, + int dim, + float eps) { + int row = blockIdx.x; + int tid = threadIdx.x; + const float *xr = x + (size_t)row * dim; + float s = 0.0f; + for (int d = tid; d < dim; d += blockDim.x) + s += xr[d]; + float mean = block_sum(s) / (float)dim; + float vs = 0.0f; + for (int d = tid; d < dim; d += blockDim.x) { + float c = xr[d] - mean; + vs += c * c; + } + float inv = rsqrtf(block_sum(vs) / (float)dim + eps); + float *o = out + (size_t)row * dim; + for (int d = tid; d < dim; d += blockDim.x) + o[d] = (xr[d] - mean) * inv * gamma[d] + beta[d]; +} + +// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). The released encoders +// use this; the tanh approximation below is for gelu_new / gelu_pytorch_tanh. +__global__ static void gelu_kernel(float *x, size_t n) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) { + float v = x[i]; + x[i] = 0.5f * v * (1.0f + erff(v * 0.70710678118654752f)); + } +} + +// Tanh-approximation GeLU in place: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))). +__global__ static void gelu_tanh_kernel(float *x, size_t n) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) { + float v = x[i]; + float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); + x[i] = 0.5f * v * (1.0f + tanhf(inner)); + } +} + +// Add the learned absolute position embedding (indexed by the within-sequence +// position) and the token-type[0] embedding to the token embeddings already in +// x. token_type holds row 0 only; every input token has segment id 0. +__global__ static void bert_pos_type_add_kernel(float *x, + const int *positions, + const float *pos_emb, + const float *token_type, + int total, + int hidden) { + size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + size_t count = (size_t)total * hidden; + if (idx >= count) + return; + int tok = (int)(idx / hidden); + int d = (int)(idx - (size_t)tok * hidden); + int pos = positions[tok]; + x[idx] += pos_emb[(size_t)pos * hidden + d] + token_type[d]; +} + +static int launch_check(void) { + cudaError_t e = cudaGetLastError(); + if (e != cudaSuccess) { + fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e)); + return -1; + } + return 0; +} + +// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm). +// EMBED_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core +// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands +// internally. +static cublasComputeType_t gemm_compute(void) { + static int env_init = 0; + if (!g_gemm_compute_set && !env_init) { + const char *e = getenv("EMBED_CUDA_GEMM_MODE"); + if (e && parse_gemm_compute(e, &g_gemm_compute) == 0) + g_gemm_compute_set = 1; + env_init = 1; + } + return g_gemm_compute; +} + +// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only +// BF16 path materializes one F32 weight at a time into this buffer. +static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) { + if (ctx->weight_f32_elems >= elems) + return 0; + cudaFree(ctx->weight_f32); + ctx->weight_f32 = NULL; + ctx->weight_f32_elems = 0; + CUDA_CHECK(cudaMalloc((void **)&ctx->weight_f32, elems * sizeof(float))); + ctx->weight_f32_elems = elems; + return 0; +} + +// y = w @ x (+ beta*y). With BF16 weights and an already-BF16 activation +// (x_is_bf16 - the Qwen fused path) both inputs are BF16 with F32 accumulation. +// For the BERT family the activation stays F32 and the BF16 weight is widened +// back to F32 here, so the GEMM is exact (honoring gemm_compute): BF16 is then a +// storage choice that costs no quality, which the post-norm LayerNorm needs. +// With F32 weights this is the exact path. +static int linear_ex(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const void *x, + int x_is_bf16, + float *y, + int rows, + int in_dim, + int out_dim, + int x_stride, + float beta) { + const float alpha = 1.0f; + if (w->bf16) { + if (!x_is_bf16 && ctx->config.family == EMBED_FAMILY_BERT) { + size_t welems = (size_t)out_dim * (size_t)in_dim; + if (ensure_weight_f32(ctx, welems) != 0) + return -1; + int wt = 256; + cast_bf16_to_f32_kernel<<<(welems + wt - 1) / wt, wt, 0, ctx->stream>>>( + ctx->weight_f32, (const __nv_bfloat16 *)w->d, welems); + if (launch_check() != 0) + return -1; + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, + &alpha, ctx->weight_f32, CUDA_R_32F, in_dim, x, CUDA_R_32F, + x_stride, &beta, y, CUDA_R_32F, out_dim, gemm_compute(), + CUBLAS_GEMM_DEFAULT)); + return 0; + } + const void *xb = x; + if (!x_is_bf16) { + size_t n = (size_t)rows * (size_t)x_stride; + int threads = 256; + cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, ctx->stream>>>( + (__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n); + if (launch_check() != 0) + return -1; + xb = ctx->act_bf16; + } + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, + &alpha, w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride, + &beta, y, CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F, + CUBLAS_GEMM_DEFAULT)); + return 0; + } + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha, + w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y, + CUDA_R_32F, out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT)); + return 0; +} + +static int linear(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const float *x, + float *y, + int rows, + int in_dim, + int out_dim) { + return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f); +} + +// x points at activations already stored as BF16 (e.g. ctx->act_bf16). +static int linear_bf16x(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const void *x, + float *y, + int rows, + int in_dim, + int out_dim, + int x_stride, + float beta) { + return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta); +} + +static int linear_accum(embed_cuda_ctx_t *ctx, + const cuda_matrix_t *w, + const float *x, + float *y, + int rows, + int in_dim, + int out_dim) { + return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f); +} + +static int gemm_strided_batched(cublasHandle_t blas, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, + const float *A, + int lda, + long long strideA, + const float *B, + int ldb, + long long strideB, + const float *beta, + float *C, + int ldc, + long long strideC, + int batch_count) { + cublasComputeType_t ct = gemm_compute(); + if (ct == CUBLAS_COMPUTE_32F) { + CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda, + strideA, B, ldb, strideB, beta, C, ldc, strideC, + batch_count)); + } else { + CUBLAS_CHECK(cublasGemmStridedBatchedEx( + blas, transa, transb, m, n, k, alpha, A, CUDA_R_32F, lda, strideA, B, CUDA_R_32F, ldb, + strideB, beta, C, CUDA_R_32F, ldc, strideC, batch_count, ct, CUBLAS_GEMM_DEFAULT)); + } + return 0; +} + +// Attention for one micro-batch. Expands K/V to the per-query-head layout, +// then for each sequence runs batched-over-heads Q@K^T (scaled) -> row softmax +// -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and +// ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets +// is the host copy of the packed sequence boundaries. +// Prepare the equal-length batched-attention path for one forward pass: +// grow the score/probability scratch to hold attn_G sequences at once and +// upload the pointer arrays for the two batched GEMMs (buffer addresses are +// layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged +// batches, single sequences, or when one sequence's scores exceed the +// budget. +static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) { + const embed_config_t *c = &ctx->config; + ctx->attn_G = 0; + if (batch < 2) + return 0; + int L = ctx->offsets_host[1] - ctx->offsets_host[0]; + if (L > CUDA_ATTN_BATCHED_MAX_LEN) + return 0; + for (int b = 1; b < batch; b++) + if (ctx->offsets_host[b + 1] - ctx->offsets_host[b] != L) + return 0; + int H = c->n_heads, hd = c->head_dim, q_dim = c->q_dim; + long long per = (long long)H * L * L; + if (per <= 0 || per > CUDA_ATTN_SCORES_BUDGET / 2) + return 0; + int G = (int)(CUDA_ATTN_SCORES_BUDGET / per); + if (G > batch) + G = batch; + if (G < 2) + return 0; + + int bf16 = g_weights_bf16; + long long want = (long long)G * per; + if (want > ctx->attn_scores_elems) { + cudaFree(ctx->attn_scores); + ctx->attn_scores = NULL; + ctx->attn_scores_elems = 0; + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)want * sizeof(float))); + ctx->attn_scores_elems = want; + } + if (bf16 && want > ctx->attn_probs_elems) { + cudaFree(ctx->attn_probs); + ctx->attn_probs = NULL; + ctx->attn_probs_elems = 0; + CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)want * sizeof(__nv_bfloat16))); + ctx->attn_probs_elems = want; + } + + int entries = 6 * batch * H; + if (entries > ctx->attn_ptrs_cap) { + cudaFree((void *)ctx->attn_ptrs); + free((void *)ctx->attn_ptrs_host); + ctx->attn_ptrs = NULL; + ctx->attn_ptrs_cap = 0; + ctx->attn_ptrs_host = (const void **)malloc((size_t)entries * sizeof(void *)); + if (!ctx->attn_ptrs_host) + return -1; + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_ptrs, (size_t)entries * sizeof(void *))); + ctx->attn_ptrs_cap = entries; + } + + const void **K = ctx->attn_ptrs_host; + const void **Q = K + (size_t)batch * H; + const void **S = Q + (size_t)batch * H; + const void **V = S + (size_t)batch * H; + const void **P = V + (size_t)batch * H; + const void **O = P + (size_t)batch * H; + const __nv_bfloat16 *vb = (const __nv_bfloat16 *)ctx->vexp; + const __nv_bfloat16 *pb = (const __nv_bfloat16 *)ctx->attn_probs; + const __nv_bfloat16 *ab = (const __nv_bfloat16 *)ctx->act_bf16; + for (int b = 0; b < batch; b++) { + // Score/probability rows cycle every G sequences (chunked reuse). + long long srow = (long long)(b % G) * per; + size_t tok = (size_t)b * L; + for (int h = 0; h < H; h++) { + size_t i = (size_t)b * H + h; + K[i] = ctx->kexp + tok * q_dim + (size_t)h * hd; + Q[i] = ctx->qkv + tok * qkv_dim + q_offset + (size_t)h * hd; + S[i] = ctx->attn_scores + srow + (long long)h * L * L; + if (bf16) { + V[i] = vb + tok * q_dim + (size_t)h * hd; + P[i] = pb + srow + (long long)h * L * L; + O[i] = ab + tok * q_dim + (size_t)h * hd; + } else { + V[i] = ctx->vexp + tok * q_dim + (size_t)h * hd; + P[i] = S[i]; + O[i] = ctx->attn_out + tok * q_dim + (size_t)h * hd; + } + } + } + CUDA_CHECK(cudaMemcpy((void *)ctx->attn_ptrs, ctx->attn_ptrs_host, + (size_t)entries * sizeof(void *), cudaMemcpyHostToDevice)); + ctx->attn_G = G; + ctx->attn_L = L; + return 0; +} + +// out_bf16 != NULL selects the BF16-weight layout: V and the softmax +// probabilities are stored BF16 and the @V GEMM emits BF16 attention output +// at out_bf16 (stride q_dim), ready for the wo projection without a cast. +static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, + const int *h_offsets, + int batch, + int q_offset, + int k_offset, + int v_offset, + int qkv_dim, + float scale, + __nv_bfloat16 *out_bf16) { + const embed_config_t *c = &ctx->config; + int q_dim = c->q_dim; + int hd = c->head_dim; + int H = c->n_heads; + int total = h_offsets[batch]; + int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; + const float alpha = scale, beta = 0.0f, one = 1.0f; + + int threads = 256; + size_t exp_count = (size_t)total * q_dim; + if (out_bf16) { + attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, + k_offset, v_offset); + } else { + attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset, + v_offset); + } + if (launch_check() != 0) + return -1; + + if (ctx->attn_G >= 2) { + int G = ctx->attn_G, L = ctx->attn_L; + const void *const *K = (const void *const *)ctx->attn_ptrs; + const void *const *Q = K + (size_t)batch * H; + void *const *S = (void *const *)(Q + (size_t)batch * H); + const void *const *V = (const void *const *)S + (size_t)batch * H; + const void *const *P = V + (size_t)batch * H; + void *const *O = (void *const *)(P + (size_t)batch * H); + cudaDataType pv = out_bf16 ? CUDA_R_16BF : CUDA_R_32F; + cublasComputeType_t av_ct = out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute(); + for (int s = 0; s < batch; s += G) { + int g = batch - s < G ? batch - s : G; + int n = g * H; + size_t off = (size_t)s * H; + /* scores = scale * Q @ K^T, batched over (seq, head) */ + CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, + K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F, + qkv_dim, &beta, S + off, CUDA_R_32F, L, n, + gemm_compute(), CUBLAS_GEMM_DEFAULT)); + if (out_bf16) { + if (causal) + attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L); + else + attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( + ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L); + } else { + if (causal) + attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores, + ctx->attn_scores, L); + else + attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores, + ctx->attn_scores, L); + } + if (launch_check() != 0) + return -1; + /* O = P @ V, batched over (seq, head) */ + CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, + V + off, pv, q_dim, P + off, pv, L, &beta, O + off, pv, + q_dim, n, av_ct, CUBLAS_GEMM_DEFAULT)); + } + return 0; + } + + for (int b = 0; b < batch; b++) { + int start = h_offsets[b]; + int L = h_offsets[b + 1] - start; + const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset; + const float *K = ctx->kexp + (size_t)start * q_dim; + /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */ + if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim, + hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L, + H) != 0) + return -1; + if (out_bf16) { + __nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs; + const __nv_bfloat16 *V = (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim; + __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim; + if (causal) + attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L); + else + attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L); + if (launch_check() != 0) + return -1; + /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ + CUBLAS_CHECK(cublasGemmStridedBatchedEx( + ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, CUDA_R_16BF, q_dim, hd, P, + CUDA_R_16BF, L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, hd, H, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } else { + const float *V = ctx->vexp + (size_t)start * q_dim; + float *O = ctx->attn_out + (size_t)start * q_dim; + if (causal) + attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, + ctx->attn_scores, L); + else + attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, + ctx->attn_scores, L); + if (launch_check() != 0) + return -1; + /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ + if (gemm_strided_batched(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, q_dim, + hd, ctx->attn_scores, L, (long long)L * L, &beta, O, q_dim, hd, + H) != 0) + return -1; + } + } + return 0; +} + +static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) { + const embed_config_t *c = &ctx->config; + if (total > ctx->seq_cap) { + cudaFree(ctx->x); + cudaFree(ctx->x_norm); + cudaFree(ctx->qkv); + cudaFree(ctx->attn_out); + cudaFree(ctx->ffn_gate_up); + cudaFree(ctx->token_ids); + cudaFree(ctx->positions); + cudaFree(ctx->kexp); + cudaFree(ctx->vexp); + cudaFree(ctx->act_bf16); + ctx->x = ctx->x_norm = ctx->qkv = NULL; + ctx->attn_out = ctx->ffn_gate_up = NULL; + ctx->token_ids = ctx->positions = NULL; + ctx->kexp = ctx->vexp = NULL; + ctx->act_bf16 = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, + (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up, + (size_t)total * 2 * c->intermediate_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float))); + if (g_weights_bf16) + CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * 2 * c->intermediate_size * + sizeof(__nv_bfloat16))); + ctx->seq_cap = total; + } + if (batch + 1 > ctx->batch_cap) { + cudaFree(ctx->offsets); + free(ctx->offsets_host); + CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int))); + ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int)); + if (!ctx->offsets_host) + return -1; + ctx->batch_cap = batch + 1; + } + if (max_seq > ctx->max_seq_cap) { + cudaFree(ctx->rope_cos); + cudaFree(ctx->rope_sin); + long long score_elems = (long long)c->n_heads * max_seq * max_seq; + if (score_elems > ctx->attn_scores_elems) { + cudaFree(ctx->attn_scores); + ctx->attn_scores = NULL; + ctx->attn_scores_elems = 0; + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)score_elems * sizeof(float))); + ctx->attn_scores_elems = score_elems; + } + if (g_weights_bf16 && score_elems > ctx->attn_probs_elems) { + cudaFree(ctx->attn_probs); + ctx->attn_probs = NULL; + ctx->attn_probs_elems = 0; + CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)score_elems * sizeof(__nv_bfloat16))); + ctx->attn_probs_elems = score_elems; + } + float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); + float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); + if (!hcos || !hsin) { + free(hcos); + free(hsin); + return -1; + } + int half = c->head_dim / 2; + for (int pos = 0; pos < max_seq; pos++) { + for (int d = 0; d < half; d++) { + float inv = 1.0f / powf(c->rope_theta, (float)(2 * d) / (float)c->head_dim); + float angle = (float)pos * inv; + float cv = cosf(angle); + float sv = sinf(angle); + hcos[(size_t)pos * c->head_dim + d] = cv; + hcos[(size_t)pos * c->head_dim + half + d] = cv; + hsin[(size_t)pos * c->head_dim + d] = sv; + hsin[(size_t)pos * c->head_dim + half + d] = sv; + } + } + CUDA_CHECK( + cudaMalloc((void **)&ctx->rope_cos, (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void **)&ctx->rope_sin, (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, + (size_t)max_seq * c->head_dim * sizeof(float), + cudaMemcpyHostToDevice, ctx->stream)); + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin, + (size_t)max_seq * c->head_dim * sizeof(float), + cudaMemcpyHostToDevice, ctx->stream)); + free(hcos); + free(hsin); + ctx->max_seq_cap = max_seq; + } + return 0; +} + +static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) { + if (rows <= ctx->pooled_rows_cap) + return 0; + const embed_config_t *c = &ctx->config; + cudaFree(ctx->pooled_out); + ctx->pooled_out = NULL; + CUDA_CHECK( + cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float))); + ctx->pooled_rows_cap = rows; + return 0; +} + +static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) { + if (n_spans <= ctx->span_cap) + return 0; + cudaFree(ctx->span_starts); + cudaFree(ctx->span_lens); + ctx->span_starts = NULL; + ctx->span_lens = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, (size_t)n_spans * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, (size_t)n_spans * sizeof(int))); + ctx->span_cap = n_spans; + return 0; +} + +static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) { + return load_qkv_matrix(&dst->qkv, src, c) || + load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || + (c->qkv_bias && load_qkv_bias(&dst->qkv_bias, src, c)) || + (c->qk_norm && load_vector(&dst->q_norm, src->q_norm, c->head_dim)) || + (c->qk_norm && load_vector(&dst->k_norm, src->k_norm, c->head_dim)) || + load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || + load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || + load_gate_up_matrix(&dst->gate_up_proj, src, c) || + load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size); +} + +/* BERT layer: attention reuses the fused qkv matrix and qkv_bias (q_dim == + * kv_dim == hidden); the two block LayerNorm weights load into input_norm + * (post-attention) and post_attn_norm (post-FFN); gate_up_proj holds the single + * up_proj. The added fields carry the biases the Qwen slots lack. */ +static int load_bert_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) { + return load_qkv_matrix(&dst->qkv, src, c) || + load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || + load_qkv_bias(&dst->qkv_bias, src, c) || + load_vector(&dst->o_bias, src->o_bias, c->hidden_size) || + load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || + load_vector(&dst->attn_ln_bias, src->attn_ln_bias, c->hidden_size) || + load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || + load_vector(&dst->ffn_ln_bias, src->ffn_ln_bias, c->hidden_size) || + load_matrix(&dst->gate_up_proj, &src->up_proj, c->intermediate_size, c->hidden_size) || + load_vector(&dst->ffn_inter_bias, src->ffn_inter_bias, c->intermediate_size) || + load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size) || + load_vector(&dst->ffn_out_bias, src->ffn_out_bias, c->hidden_size); +} + +static void free_layer(cuda_layer_t *l) { + cuda_matrix_free(&l->qkv); + cuda_matrix_free(&l->wo); + cudaFree(l->qkv_bias); + cudaFree(l->q_norm); + cudaFree(l->k_norm); + cudaFree(l->input_norm); + cudaFree(l->post_attn_norm); + cuda_matrix_free(&l->gate_up_proj); + cuda_matrix_free(&l->down_proj); + cudaFree(l->o_bias); + cudaFree(l->attn_ln_bias); + cudaFree(l->ffn_inter_bias); + cudaFree(l->ffn_out_bias); + cudaFree(l->ffn_ln_bias); +} + +/* Build a device context from an already-loaded CPU model. own_cpu records + * whether embed_cuda_free should release that CPU model: embed_cuda_load owns + * the model it loaded; the late path passes a model owned by its CPU late + * model and sets own_cpu = 0. */ +static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) { + if (!cpu) + return NULL; + + embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx)); + if (!ctx) { + if (own_cpu) + embed_model_free(cpu); + return NULL; + } + ctx->cpu = cpu; + ctx->own_cpu = own_cpu; + ctx->config = cpu->config; + + /* Without an explicit --cuda-weight-dtype, store weights in the model + * file's dtype: BF16 snapshots load as BF16 (bit-exact pass-through), + * F32 snapshots keep the exact F32 default. */ + if (!g_weights_bf16_set) { + g_weights_bf16 = cpu->weights.embed_tokens.dtype == DTYPE_BF16; + if (g_weights_bf16) + fprintf(stderr, "cuda: BF16 model file; storing weights as BF16 " + "(use --cuda-weight-dtype f32 to override)\n"); + } + + if (cudaStreamCreate(&ctx->stream) != cudaSuccess || + cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS || + cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) { + embed_cuda_free(ctx); + return NULL; + } + + const embed_config_t *c = &ctx->config; + ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers)); + if (!ctx->layers) { + embed_cuda_free(ctx); + return NULL; + } + if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, c->vocab_size, + c->hidden_size) != 0) { + embed_cuda_free(ctx); + return NULL; + } + if (c->family == EMBED_FAMILY_BERT) { + /* BERT has no final norm; it adds learned position + token-type[0] + * embeddings and an embedding LayerNorm (only row 0 of token-type is + * ever used, so load just that row). */ + const embed_weights_t *w = &cpu->weights; + if (load_vector(&ctx->position_embeddings, w->position_embeddings, + c->max_position_embeddings * c->hidden_size) != 0 || + load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) != + 0 || + load_vector(&ctx->embed_ln_w, w->embed_ln_w, c->hidden_size) != 0 || + load_vector(&ctx->embed_ln_b, w->embed_ln_b, c->hidden_size) != 0) { + embed_cuda_free(ctx); + return NULL; + } + } else if (load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) { + embed_cuda_free(ctx); + return NULL; + } + for (int i = 0; i < c->n_layers; i++) { + int rc = c->family == EMBED_FAMILY_BERT + ? load_bert_layer(&ctx->layers[i], &cpu->weights.layers[i], c) + : load_layer(&ctx->layers[i], &cpu->weights.layers[i], c); + if (rc != 0) { + fprintf(stderr, "cuda: failed to load layer %d\n", i); + embed_cuda_free(ctx); + return NULL; + } + } + cudaStreamSynchronize(ctx->stream); + return ctx; +} + +embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) { + return cuda_ctx_from_model(embed_model_load(model_dir), 1); +} + +void embed_cuda_free(embed_cuda_ctx_t *ctx) { + if (!ctx) + return; + cuda_matrix_free(&ctx->embed_tokens); + if (ctx->layers) { + for (int i = 0; i < ctx->config.n_layers; i++) + free_layer(&ctx->layers[i]); + free(ctx->layers); + } + cudaFree(ctx->norm); + cudaFree(ctx->position_embeddings); + cudaFree(ctx->token_type_embedding); + cudaFree(ctx->embed_ln_w); + cudaFree(ctx->embed_ln_b); + cudaFree(ctx->x); + cudaFree(ctx->x_norm); + cudaFree(ctx->qkv); + cudaFree(ctx->attn_out); + cudaFree(ctx->ffn_gate_up); + cudaFree(ctx->pooled_out); + cudaFree(ctx->rope_cos); + cudaFree(ctx->rope_sin); + cudaFree(ctx->token_ids); + cudaFree(ctx->offsets); + free(ctx->offsets_host); + cudaFree(ctx->positions); + cudaFree(ctx->kexp); + cudaFree(ctx->vexp); + cudaFree(ctx->act_bf16); + cudaFree(ctx->weight_f32); + cudaFree(ctx->attn_scores); + cudaFree(ctx->attn_probs); + cudaFree((void *)ctx->attn_ptrs); + free((void *)ctx->attn_ptrs_host); + cudaFree(ctx->span_starts); + cudaFree(ctx->span_lens); + if (ctx->blas) + cublasDestroy(ctx->blas); + if (ctx->stream) + cudaStreamDestroy(ctx->stream); + if (ctx->own_cpu) + embed_model_free(ctx->cpu); + free(ctx); +} + +const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) { + return ctx ? &ctx->config : NULL; +} + +// Pack inputs into the device scratch every forward shares: token ids, the +// per-sequence positions 0..L-1, and the batch offsets. Grows scratch via +// ensure_buffers. Returns the packed token count in *out_total, or -1 on error. +static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + int *out_total) { + int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); + if (!h_offsets) + return -1; + int total = 0, max_seq = 0; + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) { + free(h_offsets); + return -1; + } + h_offsets[b] = total; + total += inputs[b].n_tokens; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; + } + h_offsets[batch] = total; + int *h_ids = (int *)malloc((size_t)total * sizeof(int)); + int *h_pos = (int *)malloc((size_t)total * sizeof(int)); + if (!h_ids || !h_pos) { + free(h_offsets); + free(h_ids); + free(h_pos); + return -1; + } + for (int b = 0; b < batch; b++) { + int off = h_offsets[b]; + memcpy(h_ids + off, inputs[b].ids, (size_t)inputs[b].n_tokens * sizeof(int)); + for (int i = 0; i < inputs[b].n_tokens; i++) + h_pos[off + i] = i; + } + if (ensure_buffers(ctx, total, batch, max_seq) != 0) { + free(h_offsets); + free(h_ids); + free(h_pos); + return -1; + } + memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int)); + cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, (size_t)total * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->positions, h_pos, (size_t)total * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + free(h_offsets); + free(h_ids); + free(h_pos); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce)); + return -1; + } + *out_total = total; + return 0; +} + +// BERT-family forward (post-norm encoder), mirroring the CPU forward_packed_bert. +// Embeddings: token + learned position + token-type[0], then the embedding +// LayerNorm. Each layer: QKV(+bias) from x directly -> bidirectional attention +// -> output dense(+bias) -> residual -> LayerNorm -> dense(+bias) -> GeLU -> +// dense(+bias) -> residual -> LayerNorm. No RoPE, no qk-norm, no final norm. +// F32 activations: BERT checkpoints are F32, so the linear/linear_accum F32 +// entry points are used throughout (they still handle bf16 weights via an +// internal cast, which keeps a future bf16 BERT correct if not yet optimized). +static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) { + const embed_config_t *c = &ctx->config; + int total = 0; + if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0) + return -1; + + int hidden = c->hidden_size; + int inter = c->intermediate_size; + int qkv_dim = c->q_dim + 2 * c->kv_dim; + int q_offset = 0, k_offset = c->q_dim, v_offset = c->q_dim + c->kv_dim; + float scale = 1.0f / sqrtf((float)c->head_dim); + float eps = c->layer_norm_eps; + int threads = 256; + int blocks_hidden = (total * hidden + threads - 1) / threads; + + embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, hidden, + c->vocab_size); + if (launch_check() != 0) + return -1; + bert_pos_type_add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->positions, ctx->position_embeddings, ctx->token_type_embedding, total, hidden); + if (launch_check() != 0) + return -1; + layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->embed_ln_w, + ctx->embed_ln_b, total, hidden, eps); + if (launch_check() != 0) + return -1; + + if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0) + return -1; + + for (int layer = 0; layer < c->n_layers; layer++) { + cuda_layer_t *l = &ctx->layers[layer]; + + // Self-attention (post-norm): QKV read x directly, then +bias. + if (linear(ctx, &l->qkv, ctx->x, ctx->qkv, total, hidden, qkv_dim) != 0) + return -1; + int qkv_count = total * qkv_dim; + add_row_bias_kernel<<<(qkv_count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias, + total, qkv_dim); + if (launch_check() != 0) + return -1; + if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, + qkv_dim, scale, NULL) != 0) + return -1; + + // Output dense(+bias), residual into x, then post-attention LayerNorm. + if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, hidden) != 0) + return -1; + add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>( + ctx->x, l->o_bias, total, hidden); + if (launch_check() != 0) + return -1; + layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->input_norm, + l->attn_ln_bias, total, hidden, eps); + if (launch_check() != 0) + return -1; + + // Feed-forward (post-norm): dense(+bias) -> GeLU -> dense(+bias). + if (linear(ctx, &l->gate_up_proj, ctx->x, ctx->ffn_gate_up, total, hidden, inter) != 0) + return -1; + int inter_count = total * inter; + add_row_bias_kernel<<<(inter_count + 255) / 256, 256, 0, ctx->stream>>>( + ctx->ffn_gate_up, l->ffn_inter_bias, total, inter); + if (launch_check() != 0) + return -1; + if (c->ffn_act == EMBED_ACT_GELU_TANH) + gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->ffn_gate_up, (size_t)inter_count); + else + gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>( + ctx->ffn_gate_up, (size_t)inter_count); + if (launch_check() != 0) + return -1; + if (linear_accum(ctx, &l->down_proj, ctx->ffn_gate_up, ctx->x, total, inter, hidden) != 0) + return -1; + add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>( + ctx->x, l->ffn_out_bias, total, hidden); + if (launch_check() != 0) + return -1; + layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->post_attn_norm, + l->ffn_ln_bias, total, hidden, eps); + if (launch_check() != 0) + return -1; + } + // No final norm; embed_cuda_encode_batch pools ctx->x directly. + return 0; +} + +static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) { + if (!ctx || !inputs || batch <= 0) + return -1; + const embed_config_t *c = &ctx->config; + if (c->family == EMBED_FAMILY_BERT) + return cuda_forward_batch_bert(ctx, inputs, batch); + int total = 0; + if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0) + return -1; + + int threads = 256; + int blocks_hidden = (total * c->hidden_size + threads - 1) / threads; + embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, c->hidden_size, + c->vocab_size); + if (launch_check() != 0) + return -1; + + float scale = 1.0f / sqrtf((float)c->head_dim); + int q_offset = 0; + int k_offset = c->q_dim; + int v_offset = c->q_dim + c->kv_dim; + int qkv_dim = c->q_dim + 2 * c->kv_dim; + // With BF16 weights every producer (norm, SiLU, attention @V GEMM) + // stores its result as BF16 directly into ctx->act_bf16, which the next + // projection GEMM consumes before any later producer reuses the buffer + // (single stream). No cast launches remain in the layer loop. + __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16; + if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0) + return -1; + for (int layer = 0; layer < c->n_layers; layer++) { + cuda_layer_t *l = &ctx->layers[layer]; + int wbf16 = l->qkv.bf16; + if (wbf16) { + rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->input_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, total, c->hidden_size, qkv_dim, + c->hidden_size, 0.0f) != 0) + return -1; + } else { + rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm, + total, c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, total, c->hidden_size, qkv_dim) != 0) + return -1; + } + + if (c->qkv_bias) { + int count = total * qkv_dim; + add_row_bias_kernel<<<(count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias, + total, qkv_dim); + if (launch_check() != 0) + return -1; + } + + rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>( + ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total, + c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->qk_norm, + c->rms_norm_eps); + if (launch_check() != 0) + return -1; + + if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, + qkv_dim, scale, wbf16 ? act : NULL) != 0) + return -1; + + if (wbf16) { + if (linear_bf16x(ctx, &l->wo, act, ctx->x, total, c->q_dim, c->hidden_size, c->q_dim, + 1.0f) != 0) + return -1; + } else { + if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, c->hidden_size) != + 0) + return -1; + } + + int inter_count = total * c->intermediate_size; + if (wbf16) { + rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->post_attn_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, total, c->hidden_size, + 2 * c->intermediate_size, c->hidden_size, 0.0f) != 0) + return -1; + silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0, + ctx->stream>>>(act, c->intermediate_size, ctx->ffn_gate_up, + total, c->intermediate_size); + if (launch_check() != 0) + return -1; + if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, total, c->intermediate_size, + c->hidden_size, c->intermediate_size, 1.0f) != 0) + return -1; + } else { + rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->post_attn_norm, + total, c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + if (linear(ctx, &l->gate_up_proj, ctx->x_norm, ctx->ffn_gate_up, total, c->hidden_size, + 2 * c->intermediate_size) != 0) + return -1; + silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0, + ctx->stream>>>(ctx->ffn_gate_up, 2 * c->intermediate_size, + ctx->ffn_gate_up, total, c->intermediate_size); + if (launch_check() != 0) + return -1; + if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, total, + c->intermediate_size, c->hidden_size, 2 * c->intermediate_size, + 1.0f) != 0) + return -1; + } + } + + rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) + return -1; + + return 0; +} + +int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; + const embed_config_t *c = &ctx->config; + if (cuda_forward_batch(ctx, inputs, batch) != 0) + return -1; + if (ensure_pooled_rows(ctx, batch) != 0) + return -1; + if (c->pooling_mode == EMBED_POOL_LAST_TOKEN) + last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, + batch, c->hidden_size); + else if (c->pooling_mode == EMBED_POOL_CLS) + first_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, + batch, c->hidden_size); + else + mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, + batch, c->hidden_size); + if (launch_check() != 0) + return -1; + cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, + (size_t)batch * c->hidden_size * sizeof(float), + cudaMemcpyDeviceToHost, ctx->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce)); + return -1; + } + if (c->normalize_embeddings) { + for (int b = 0; b < batch; b++) { + if (embed_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) != + 0) + return -1; + } + } + return 0; +} + +int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; + const embed_config_t *c = &ctx->config; + int total_spans = 0; + int total_tokens = 0; + for (int b = 0; b < batch; b++) { + const embed_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || + total_spans > INT_MAX - input->n_spans || total_tokens > INT_MAX - n_tokens) + return -1; + for (int s = 0; s < input->n_spans; s++) { + int start = input->spans[s].start; + int len = input->spans[s].n_tokens; + if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) + return -1; + } + total_spans += input->n_spans; + total_tokens += n_tokens; + } + + embed_input_t *packed = (embed_input_t *)malloc((size_t)batch * sizeof(*packed)); + int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int)); + int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int)); + if (!packed || !h_starts || !h_lens) { + free(packed); + free(h_starts); + free(h_lens); + return -1; + } + + int token_offset = 0; + int span_offset = 0; + for (int b = 0; b < batch; b++) { + const embed_context_input_t *input = &inputs[b]; + packed[b] = input->input; + for (int s = 0; s < input->n_spans; s++) { + h_starts[span_offset] = token_offset + input->spans[s].start; + h_lens[span_offset] = input->spans[s].n_tokens; + span_offset++; + } + token_offset += input->input.n_tokens; + } + + int rc = -1; + cudaError_t ce; + if (cuda_forward_batch(ctx, packed, batch) != 0) + goto cleanup; + if (ensure_pooled_rows(ctx, total_spans) != 0 || ensure_span_buffers(ctx, total_spans) != 0) + goto cleanup; + ce = cudaMemcpyAsync(ctx->span_starts, h_starts, (size_t)total_spans * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->span_lens, h_lens, (size_t)total_spans * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce)); + goto cleanup; + } + span_pool_kernel<<<total_spans, 256, 0, ctx->stream>>>( + ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, total_spans, c->hidden_size); + if (launch_check() != 0) + goto cleanup; + ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, + (size_t)total_spans * c->hidden_size * sizeof(float), + cudaMemcpyDeviceToHost, ctx->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda span output copy failed: %s\n", cudaGetErrorString(ce)); + goto cleanup; + } + rc = 0; + +cleanup: + free(h_lens); + free(h_starts); + free(packed); + return rc; +} + +int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding) { + embed_input_t input = {token_ids, n_tokens}; + return embed_cuda_encode_batch(ctx, &input, 1, out_embedding); +} + +float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) { + if (!ctx) + return NULL; + int hidden = ctx->config.hidden_size; + float *out = (float *)malloc((size_t)hidden * sizeof(float)); + if (!out) + return NULL; + if (embed_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; + } + return out; +} + +/* ======================================================================== + * Late-interaction CUDA path + * + * The transformer forward and the 1_Dense projection run on the GPU; MaxSim + * scoring stays on the host, where the grouped-GEMM scorer is far faster than a + * device graph for this small op. So the encoders copy the projected token + * vectors back to host memory and the caller runs embed_late_maxsim_batch there. + * ======================================================================== */ + +struct embed_cuda_late_ctx { + embed_cuda_ctx_t *base; // device base model; does not own its CPU model + embed_late_model_t *cpu_late; // owns the CPU base model + projection host data + cuda_matrix_t projection; // device [token_dim, hidden] + float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand + size_t proj_cap; // capacity of proj_dev in floats + int token_dim; +}; + +embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir) { + embed_late_model_t *cpu_late = embed_late_model_load(model_dir); + if (!cpu_late) + return NULL; + + embed_cuda_late_ctx_t *ctx = (embed_cuda_late_ctx_t *)calloc(1, sizeof(*ctx)); + if (!ctx) { + embed_late_model_free(cpu_late); + return NULL; + } + ctx->cpu_late = cpu_late; + ctx->token_dim = embed_late_model_token_dim(cpu_late); + + // Upload the base transformer weights; the CPU late model keeps ownership. + ctx->base = cuda_ctx_from_model(embed_late_model_base(cpu_late), 0); + if (!ctx->base) { + embed_cuda_late_free(ctx); + return NULL; + } + + // Upload the 1_Dense projection [token_dim, hidden]. + const embed_weight_ref_t *proj = embed_late_model_projection(cpu_late); + if (!proj || ctx->token_dim <= 0 || + load_matrix(&ctx->projection, proj, ctx->token_dim, ctx->base->config.hidden_size) != 0) { + embed_cuda_late_free(ctx); + return NULL; + } + return ctx; +} + +void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx) { + if (!ctx) + return; + cuda_matrix_free(&ctx->projection); + cudaFree(ctx->proj_dev); + embed_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model + embed_late_model_free(ctx->cpu_late); // frees the CPU base model + projection + free(ctx); +} + +const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx) { + return ctx ? &ctx->base->config : NULL; +} + +int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } + +static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) { + size_t need = (size_t)total * (size_t)ctx->token_dim; + if (need <= ctx->proj_cap) + return 0; + cudaFree(ctx->proj_dev); + ctx->proj_dev = NULL; + ctx->proj_cap = 0; + if (cudaMalloc((void **)&ctx->proj_dev, need * sizeof(float)) != cudaSuccess) + return -1; + ctx->proj_cap = need; + return 0; +} + +/* Forward all inputs, project every token to token_dim, and copy the packed + * [total, token_dim] F32 result to a freshly malloc'd host buffer (caller frees). + * On success *proj_host and *total_out are set; base->offsets_host holds the + * packed per-document token offsets. */ +static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx, + const embed_input_t *inputs, + int n_docs, + float **proj_host, + int *total_out) { + embed_cuda_ctx_t *base = ctx->base; + const embed_config_t *c = &base->config; + if (cuda_forward_batch(base, inputs, n_docs) != 0) + return -1; + int total = base->offsets_host[n_docs]; + if (total <= 0 || ensure_late_proj(ctx, total) != 0) + return -1; + if (linear(base, &ctx->projection, base->x, ctx->proj_dev, total, c->hidden_size, + ctx->token_dim) != 0) + return -1; + float *host = (float *)malloc((size_t)total * (size_t)ctx->token_dim * sizeof(float)); + if (!host) + return -1; + cudaError_t ce = + cudaMemcpyAsync(host, ctx->proj_dev, (size_t)total * (size_t)ctx->token_dim * sizeof(float), + cudaMemcpyDeviceToHost, base->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(base->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda late output copy failed: %s\n", cudaGetErrorString(ce)); + free(host); + return -1; + } + *proj_host = host; + *total_out = total; + return 0; +} + +int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { + if (!ctx || !token_ids || n_tokens <= 0 || !out_vectors || ctx->token_dim <= 0) + return -1; + embed_input_t input = {token_ids, n_tokens}; + float *proj_host = NULL; + int total = 0; + if (cuda_late_forward_project(ctx, &input, 1, &proj_host, &total) != 0) + return -1; + memcpy(out_vectors, proj_host, (size_t)total * (size_t)ctx->token_dim * sizeof(float)); + free(proj_host); + if (normalize) { + for (int i = 0; i < total; i++) + if (embed_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0) + return -1; + } + return 0; +} + +int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets) { + if (!ctx || !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors || + !out_offsets || ctx->token_dim <= 0) + return -1; + int dim = ctx->token_dim; + + embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); + if (!inputs) + return -1; + + int total_keep = 0; + out_offsets[0] = 0; + for (int d = 0; d < n_docs; d++) { + if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 || + n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) { + free(inputs); + return -1; + } + for (int k = 0; k < n_keep[d]; k++) { + if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) { + free(inputs); + return -1; + } + } + inputs[d].ids = doc_ids[d]; + inputs[d].n_tokens = n_tokens[d]; + total_keep += n_keep[d]; + out_offsets[d + 1] = total_keep; + } + + float *proj_host = NULL; + int total = 0; + if (cuda_late_forward_project(ctx, inputs, n_docs, &proj_host, &total) != 0) { + free(inputs); + return -1; + } + + /* Gather each document's kept token rows (in packed forward order) into the + * output, the layout MaxSim consumes - matching the CPU/MLX paths. */ + const int *off = ctx->base->offsets_host; + int pos = 0; + for (int d = 0; d < n_docs; d++) { + int start = off[d]; + for (int k = 0; k < n_keep[d]; k++) { + memcpy(out_vectors + (size_t)pos * dim, proj_host + (size_t)(start + keep[d][k]) * dim, + (size_t)dim * sizeof(float)); + pos++; + } + } + free(proj_host); + free(inputs); + + if (normalize) { + for (int i = 0; i < total_keep; i++) + if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + return 0; +} |
