From 2c37727e653d2340d8977dd5db4addc767dd81ea Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 10 Jun 2026 08:24:39 +0200 Subject: gitignore: track new binary and library names --- .gitignore | 5 +- embed.c | 1483 ++++++++++++++++++++++++++++++ embed.h | 293 ++++++ embed_cli.c | 825 +++++++++++++++++ embed_cuda.cu | 1204 ++++++++++++++++++++++++ embed_cuda.h | 30 + embed_distributed.c | 888 ++++++++++++++++++ embed_distributed.h | 47 + embed_internal.h | 64 ++ embed_mlx.c | 2296 ++++++++++++++++++++++++++++++++++++++++++++++ embed_mlx.h | 149 +++ embed_server.c | 2431 +++++++++++++++++++++++++++++++++++++++++++++++++ embed_server.h | 32 + main.c | 825 ----------------- pplx_distributed.c | 888 ------------------ pplx_distributed.h | 47 - pplx_embed.c | 1483 ------------------------------ pplx_embed.h | 293 ------ pplx_embed_cuda.cu | 1204 ------------------------ pplx_embed_cuda.h | 30 - pplx_embed_internal.h | 64 -- pplx_embed_mlx.c | 2296 ---------------------------------------------- pplx_embed_mlx.h | 149 --- pplx_server.c | 2431 ------------------------------------------------- pplx_server.h | 32 - server_main.c | 340 ------- 26 files changed, 9746 insertions(+), 10083 deletions(-) create mode 100644 embed.c create mode 100644 embed.h create mode 100644 embed_cli.c create mode 100644 embed_cuda.cu create mode 100644 embed_cuda.h create mode 100644 embed_distributed.c create mode 100644 embed_distributed.h create mode 100644 embed_internal.h create mode 100644 embed_mlx.c create mode 100644 embed_mlx.h create mode 100644 embed_server.c create mode 100644 embed_server.h delete mode 100644 main.c delete mode 100644 pplx_distributed.c delete mode 100644 pplx_distributed.h delete mode 100644 pplx_embed.c delete mode 100644 pplx_embed.h delete mode 100644 pplx_embed_cuda.cu delete mode 100644 pplx_embed_cuda.h delete mode 100644 pplx_embed_internal.h delete mode 100644 pplx_embed_mlx.c delete mode 100644 pplx_embed_mlx.h delete mode 100644 pplx_server.c delete mode 100644 pplx_server.h delete mode 100644 server_main.c diff --git a/.gitignore b/.gitignore index d4750c8..987e7be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ *.o -pplx_embed +pplx-embed +pplx-embed-server +libpplxembed.* +tests/test_kernels_* book/ .venv/ __pycache__/ diff --git a/embed.c b/embed.c new file mode 100644 index 0000000..996be9c --- /dev/null +++ b/embed.c @@ -0,0 +1,1483 @@ +#include "pplx_embed.h" +#include "pplx_embed_internal.h" +#include "qwen_kernels.h" +#include "qwen_safetensors.h" + +#include +#include +#include +#include +#include +#include + +struct pplx_late_model { + pplx_model_t *base; + safetensors_file_t *dense_sf; + pplx_weight_ref_t projection; /* [token_dim, hidden] */ + int token_dim; +}; + +struct pplx_late_workspace { + const pplx_late_model_t *model; + pplx_workspace_t *base_ws; + float *states; + int states_seq_cap; +}; + +/* ======================================================================== + * Globals + * ======================================================================== */ + +int pplx_verbose = 0; +int qwen_verbose = 0; + +#define PPLX_BF16_QKV_FUSE_MAX_SEQ 16 +#define PPLX_BF16_PAIR_FUSE_MAX_SEQ 4 +#define PPLX_MIN_WORKSPACE_SEQ_CAP 16 +#define PPLX_WORKSPACE_SEQ_GRANULARITY 16 + +/* ======================================================================== + * Size helpers + * ======================================================================== */ + +static int mul_size(size_t a, size_t b, size_t *out) +{ + if (a != 0 && b > SIZE_MAX / a) return -1; + *out = a * b; + return 0; +} + +static int add_size(size_t a, size_t b, size_t *out) +{ + if (b > SIZE_MAX - a) return -1; + *out = a + b; + return 0; +} + +static int grow_cap(int current, int needed, int *out) +{ + if (needed <= 0) return -1; + + int cap = needed; + if (cap < PPLX_MIN_WORKSPACE_SEQ_CAP) + cap = PPLX_MIN_WORKSPACE_SEQ_CAP; + + int rem = cap % PPLX_WORKSPACE_SEQ_GRANULARITY; + if (rem != 0) { + int add = PPLX_WORKSPACE_SEQ_GRANULARITY - rem; + if (cap > INT_MAX - add) return -1; + cap += add; + } + + if (cap < current) + cap = current; + + *out = cap; + return 0; +} + +static int realloc_floats(float **ptr, size_t count) +{ + size_t bytes; + if (mul_size(count, sizeof(float), &bytes) != 0) return -1; + + void *p = realloc(*ptr, bytes); + if (!p && bytes != 0) return -1; + *ptr = (float *)p; + return 0; +} + +static float *malloc_floats(size_t count) +{ + size_t bytes; + if (mul_size(count, sizeof(float), &bytes) != 0) return NULL; + return (float *)malloc(bytes); +} + +static int model_dir_has_file(const char *model_dir, const char *rel) +{ + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); + if (n < 0 || (size_t)n >= sizeof(path)) return 0; + FILE *f = fopen(path, "rb"); + if (!f) return 0; + fclose(f); + return 1; +} + +static int model_dir_has_late_projection(const char *model_dir) +{ + return model_dir_has_file(model_dir, "1_Dense/config.json") && + model_dir_has_file(model_dir, "1_Dense/model.safetensors"); +} + +static int realloc_floats_2d(float **ptr, int rows, int cols) +{ + if (rows < 0 || cols < 0) return -1; + + size_t count; + if (mul_size((size_t)rows, (size_t)cols, &count) != 0) return -1; + return realloc_floats(ptr, count); +} + +static int ensure_late_state_buffer(pplx_late_workspace_t *ws, int n_tokens) +{ + if (!ws || !ws->model || n_tokens <= 0) return -1; + if (n_tokens <= ws->states_seq_cap) return 0; + int hidden = ws->model->base->config.hidden_size; + if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0) + return -1; + ws->states_seq_cap = n_tokens; + return 0; +} + +static int realloc_ints(int **ptr, size_t count) +{ + size_t bytes; + if (mul_size(count, sizeof(int), &bytes) != 0) return -1; + + void *p = realloc(*ptr, bytes); + if (!p && bytes != 0) return -1; + *ptr = (int *)p; + return 0; +} + +/* ======================================================================== + * Minimal JSON helpers for config.json parsing + * ======================================================================== */ + +static const char *skip_ws(const char *p) +{ + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + return p; +} + +/* Find "key": in a flat JSON object. Returns pointer to value start. */ +static const char *json_find_key(const char *json, const char *key) +{ + size_t klen = strlen(key); + const char *p = json; + while ((p = strstr(p, key)) != NULL) { + /* Check it's actually a quoted key: "key" */ + if (p > json && *(p - 1) == '"') { + const char *after = p + klen; + if (*after == '"') { + after = skip_ws(after + 1); + if (*after == ':') return skip_ws(after + 1); + } + } + p += klen; + } + return NULL; +} + +static int json_get_int(const char *json, const char *key, int fallback) +{ + const char *v = json_find_key(json, key); + if (!v) return fallback; + return atoi(v); +} + +static double json_get_double(const char *json, const char *key, double fallback) +{ + const char *v = json_find_key(json, key); + if (!v) return fallback; + return atof(v); +} + +static int parse_config(pplx_config_t *cfg, const char *model_dir) +{ + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "pplx_model_load: model path too long\n"); + return -1; + } + + FILE *f = fopen(path, "rb"); + if (!f) { + fprintf(stderr, "pplx_model_load: cannot open %s\n", path); + return -1; + } + + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + fclose(f); free(buf); return -1; + } + fclose(f); + buf[sz] = '\0'; + + cfg->hidden_size = json_get_int(buf, "hidden_size", 0); + cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0); + cfg->n_heads = json_get_int(buf, "num_attention_heads", 0); + cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0); + cfg->head_dim = json_get_int(buf, "head_dim", PPLX_HEAD_DIM); + cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0); + cfg->vocab_size = json_get_int(buf, "vocab_size", PPLX_VOCAB_SIZE); + cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); + cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); + + cfg->q_dim = 0; + cfg->kv_dim = 0; + if (cfg->n_heads > 0 && cfg->head_dim > 0 && + cfg->n_heads <= INT_MAX / cfg->head_dim) + cfg->q_dim = cfg->n_heads * cfg->head_dim; + if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && + cfg->n_kv_heads <= INT_MAX / cfg->head_dim) + cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; + + free(buf); + + /* Sanity checks */ + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || + cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || + cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || + cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || + (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || + !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || + !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { + fprintf(stderr, "pplx_model_load: invalid config in %s " + "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", + path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, + cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); + return -1; + } + if (cfg->n_layers > PPLX_MAX_LAYERS) { + fprintf(stderr, "pplx_model_load: too many layers (%d > %d)\n", + cfg->n_layers, PPLX_MAX_LAYERS); + return -1; + } + + if (pplx_verbose >= 1) + fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, " + "inter=%d, head_dim=%d\n", + cfg->hidden_size, cfg->n_layers, + cfg->n_heads, cfg->n_kv_heads, + cfg->intermediate_size, cfg->head_dim); + + return 0; +} + +/* ======================================================================== + * Weight loading (direct mmap pointers into safetensors) + * ======================================================================== */ + +static size_t dtype_size(safetensor_dtype_t dtype) +{ + switch (dtype) { + case DTYPE_F32: return sizeof(float); + case DTYPE_BF16: return sizeof(uint16_t); + default: return 0; + } +} + +static const char *dtype_name(safetensor_dtype_t dtype) +{ + switch (dtype) { + case DTYPE_F32: return "F32"; + case DTYPE_F16: return "F16"; + case DTYPE_BF16: return "BF16"; + case DTYPE_I32: return "I32"; + case DTYPE_I64: return "I64"; + case DTYPE_BOOL: return "BOOL"; + default: return "UNKNOWN"; + } +} + +static int tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, const char *name, + const int64_t *shape, int ndim, + int bf16_ok) +{ + if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) { + fprintf(stderr, "pplx: unsupported dtype for %s: got %s, expected %s\n", + name, dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); + return 0; + } + if (t->ndim != ndim) { + fprintf(stderr, "pplx: bad rank for %s: got %d, expected %d\n", + name, t->ndim, ndim); + return 0; + } + + size_t numel = 1; + for (int i = 0; i < ndim; i++) { + if (t->shape[i] != shape[i]) { + fprintf(stderr, "pplx: bad shape for %s at dim %d: " + "got %lld, expected %lld\n", + name, i, (long long)t->shape[i], (long long)shape[i]); + return 0; + } + if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { + fprintf(stderr, "pplx: shape too large for %s\n", name); + return 0; + } + numel *= (size_t)shape[i]; + } + + size_t elem_size = dtype_size(t->dtype); + if (elem_size == 0) { + fprintf(stderr, "pplx: unsupported dtype for %s: got %s\n", + name, dtype_name(t->dtype)); + return 0; + } + + size_t bytes; + if (mul_size(numel, elem_size, &bytes) != 0) { + fprintf(stderr, "pplx: tensor data too large for %s\n", name); + return 0; + } + if (t->data_size != bytes) { + fprintf(stderr, "pplx: bad data size for %s: got %zu, expected %zu\n", + name, t->data_size, bytes); + return 0; + } + + size_t data_start = 8 + sf->header_size; + if (data_start > sf->file_size || + t->data_offset > sf->file_size - data_start || + t->data_size > sf->file_size - data_start - t->data_offset) { + fprintf(stderr, "pplx: tensor data out of bounds for %s\n", name); + return 0; + } + return 1; +} + +static pplx_weight_ref_t load_weight_direct(multi_safetensors_t *ms, + const char *name, + const int64_t *shape, int ndim) +{ + pplx_weight_ref_t ref = { DTYPE_UNKNOWN, NULL }; + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "pplx: tensor not found: %s\n", name); + return ref; + } + if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) + return ref; + + ref.dtype = t->dtype; + ref.data = safetensors_data(sf, t); + return ref; +} + +static float *load_norm_f32(multi_safetensors_t *ms, const char *name, + const int64_t *shape, int ndim) +{ + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "pplx: tensor not found: %s\n", name); + return NULL; + } + if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) + return NULL; + return safetensors_get_f32(sf, t); +} + +static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) +{ + for (int i = 0; i < n; i++) { + uint32_t u = ((uint32_t)src[i]) << 16; + memcpy(dst + i, &u, sizeof(float)); + } +} + +static void copy_weight_row(float *dst, const pplx_weight_ref_t *w, + size_t row, int row_width) +{ + if (w->dtype == DTYPE_F32) { + const float *src = (const float *)w->data + row * (size_t)row_width; + memcpy(dst, src, (size_t)row_width * sizeof(float)); + } else { + const uint16_t *src = (const uint16_t *)w->data + row * (size_t)row_width; + bf16_row_to_f32(dst, src, row_width); + } +} + +static void linear_nobias_weight(float *y, const float *x, + const pplx_weight_ref_t *w, + int seq_len, int in_dim, int out_dim) +{ + if (w->dtype == DTYPE_F32) { + qwen_linear_nobias(y, x, (const float *)w->data, + seq_len, in_dim, out_dim); + } else { + qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, + seq_len, in_dim, out_dim); + } +} + +static void linear_qkv_weight(float *q, float *k, float *v, const float *x, + const pplx_weight_ref_t *wq, + const pplx_weight_ref_t *wk, + const pplx_weight_ref_t *wv, + int seq_len, int in_dim, int q_dim, int kv_dim) +{ + if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && + wv->dtype == DTYPE_BF16 && + seq_len <= PPLX_BF16_QKV_FUSE_MAX_SEQ) { + qwen_linear_nobias_bf16_qkv(q, k, v, x, + (const uint16_t *)wq->data, + (const uint16_t *)wk->data, + (const uint16_t *)wv->data, + seq_len, in_dim, q_dim, kv_dim); + return; + } + + linear_nobias_weight(q, x, wq, seq_len, in_dim, q_dim); + linear_nobias_weight(k, x, wk, seq_len, in_dim, kv_dim); + linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim); +} + +static void linear_pair_weight(float *a, float *b, const float *x, + const pplx_weight_ref_t *wa, + const pplx_weight_ref_t *wb, + int seq_len, int in_dim, int a_dim, int b_dim) +{ + if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && + seq_len <= PPLX_BF16_PAIR_FUSE_MAX_SEQ) { + qwen_linear_nobias_bf16_pair(a, b, x, + (const uint16_t *)wa->data, + (const uint16_t *)wb->data, + seq_len, in_dim, a_dim, b_dim); + return; + } + + linear_nobias_weight(a, x, wa, seq_len, in_dim, a_dim); + linear_nobias_weight(b, x, wb, seq_len, in_dim, b_dim); +} + +/* ======================================================================== + * Working buffer management + * ======================================================================== */ + +static int ensure_buffers(pplx_workspace_t *ws, const pplx_config_t *c, int seq) +{ + if (seq <= ws->buf_seq_cap) return 0; + + int cap; + if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) return -1; + +#define R(ptr, n) do { \ + if (realloc_floats_2d(&(ptr), cap, (n)) != 0) return -1; \ +} while (0) + + R(ws->x, c->hidden_size); + R(ws->x_norm, c->hidden_size); + R(ws->q, c->q_dim); + R(ws->k, c->kv_dim); + R(ws->v, c->kv_dim); + R(ws->attn_out, c->q_dim); + R(ws->proj_out, c->hidden_size); + R(ws->ffn_gate, c->intermediate_size); + R(ws->ffn_up, c->intermediate_size); + +#undef R + + ws->buf_seq_cap = cap; + return 0; +} + +static void ensure_attention_scores(pplx_workspace_t *ws, const int *offsets, + int batch) +{ + size_t bytes = + qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (bytes <= ws->attn_scores_bytes) return; + + void *p = realloc(ws->attn_scores, bytes); + if (!p) return; + ws->attn_scores = (float *)p; + ws->attn_scores_bytes = bytes; +} + +/* ======================================================================== + * RoPE cache + * ======================================================================== */ + +static int ensure_rope_cache(pplx_workspace_t *ws, const pplx_config_t *cfg, + int n_pos) +{ + if (n_pos <= ws->rope_cache_cap) return 0; + + int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; + while (cap < n_pos) { + if (cap > INT_MAX / 2) return -1; + cap *= 2; + } + + int head_dim = cfg->head_dim; + int half = head_dim / 2; + float theta = cfg->rope_theta; + + size_t count; + if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) + return -1; + if (realloc_floats(&ws->rope_cos, count) != 0) + return -1; + if (realloc_floats(&ws->rope_sin, count) != 0) + return -1; + + for (int pos = ws->rope_cache_cap; pos < cap; pos++) { + float fp = (float)pos; + float *c = ws->rope_cos + (size_t)pos * head_dim; + float *s = ws->rope_sin + (size_t)pos * head_dim; + for (int d = 0; d < half; d++) { + float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = fp * inv; + float cv = cosf(angle), sv = sinf(angle); + c[d] = cv; c[half + d] = cv; + s[d] = sv; s[half + d] = sv; + } + } + + ws->rope_cache_cap = cap; + return 0; +} + +static int ensure_offsets(pplx_workspace_t *ws, int batch) +{ + if (!ws || batch <= 0 || batch == INT_MAX) return -1; + + int needed = batch + 1; + if (needed <= ws->offsets_cap) return 0; + + if (realloc_ints(&ws->offsets, (size_t)needed) != 0) + return -1; + ws->offsets_cap = needed; + return 0; +} + +/* ======================================================================== + * Model / workspace load and free + * ======================================================================== */ + +static pplx_model_t *model_load_range_ex(const char *model_dir, + int layer_start, int layer_end, + int allow_late) +{ + if (!allow_late && model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "pplx_model_load: late-interaction models require " + "token-level MaxSim support and are not valid pooled " + "embedding models yet\n"); + return NULL; + } + + pplx_model_t *model = (pplx_model_t *)calloc(1, sizeof(pplx_model_t)); + if (!model) return NULL; + + /* Parse config.json */ + if (parse_config(&model->config, model_dir) != 0) { + free(model); + return NULL; + } + + const pplx_config_t *cfg = &model->config; + if (layer_end == -1) + layer_end = cfg->n_layers; + if (layer_start < 0 || layer_start >= layer_end || + layer_end > cfg->n_layers) { + fprintf(stderr, "pplx_model_load: invalid layer range [%d, %d) " + "for %d-layer model\n", layer_start, layer_end, cfg->n_layers); + free(model); + return NULL; + } + model->layer_start = layer_start; + model->layer_end = layer_end; + + /* Open safetensors (handles single file or multi-shard) */ + multi_safetensors_t *ms = multi_safetensors_open(model_dir); + if (!ms) { + fprintf(stderr, "pplx_model_load: failed to open safetensors in %s\n", model_dir); + free(model); + return NULL; + } + model->safetensors = ms; + + if (pplx_verbose >= 1) + fprintf(stderr, "pplx_model_load: loading weights from %s\n", model_dir); + + /* Only the first stage needs token embeddings. */ + pplx_weights_t *w = &model->weights; + if (layer_start == 0) { + const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size }; + w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", + embed_shape, 2); + if (!w->embed_tokens.data) goto fail; + } + + /* Allocate layer array */ + w->layers = (pplx_layer_t *)calloc(cfg->n_layers, sizeof(pplx_layer_t)); + if (!w->layers) goto fail; + + /* Per-layer weights */ + char name[256]; + for (int i = layer_start; i < layer_end; i++) { + pplx_layer_t *l = &w->layers[i]; + +#define LOAD_NORM(field, fmt, d0) do { \ + const int64_t expect[1] = { (d0) }; \ + snprintf(name, sizeof(name), fmt, i); \ + l->field = load_norm_f32(ms, name, expect, 1); \ + if (!l->field) goto fail; \ +} while (0) + +#define LOAD2(field, fmt, d0, d1) do { \ + const int64_t expect[2] = { (d0), (d1) }; \ + snprintf(name, sizeof(name), fmt, i); \ + l->field = load_weight_direct(ms, name, expect, 2); \ + if (!l->field.data) goto fail; \ +} while (0) + + LOAD2(wq, "layers.%d.self_attn.q_proj.weight", + cfg->q_dim, cfg->hidden_size); + LOAD2(wk, "layers.%d.self_attn.k_proj.weight", + cfg->kv_dim, cfg->hidden_size); + LOAD2(wv, "layers.%d.self_attn.v_proj.weight", + cfg->kv_dim, cfg->hidden_size); + LOAD2(wo, "layers.%d.self_attn.o_proj.weight", + cfg->hidden_size, cfg->q_dim); + LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight", + cfg->head_dim); + LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight", + cfg->head_dim); + LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight", + cfg->hidden_size); + LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight", + cfg->hidden_size); + LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight", + cfg->intermediate_size, cfg->hidden_size); + LOAD2(up_proj, "layers.%d.mlp.up_proj.weight", + cfg->intermediate_size, cfg->hidden_size); + LOAD2(down_proj, "layers.%d.mlp.down_proj.weight", + cfg->hidden_size, cfg->intermediate_size); + +#undef LOAD2 +#undef LOAD_NORM + + if (pplx_verbose >= 2) + fprintf(stderr, " layer %d loaded\n", i); + } + + /* Only the final stage applies the output RMSNorm. */ + if (layer_end == cfg->n_layers) { + const int64_t norm_shape[1] = { cfg->hidden_size }; + w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1); + if (!w->norm) goto fail; + } + + if (pplx_verbose >= 1) + fprintf(stderr, "pplx_model_load: layers [%d, %d) loaded " + "(%d-dim embeddings)\n", + layer_start, layer_end, cfg->hidden_size); + + return model; + +fail: + pplx_model_free(model); + return NULL; +} + +pplx_model_t *pplx_model_load(const char *model_dir) +{ + return model_load_range_ex(model_dir, 0, -1, 0); +} + +pplx_model_t *pplx_model_load_slice(const char *model_dir, + int layer_start, int layer_end) +{ + return model_load_range_ex(model_dir, layer_start, layer_end, 0); +} + +void pplx_model_free(pplx_model_t *model) +{ + if (!model) return; + if (model->weights.layers) { + for (int i = 0; i < model->config.n_layers; i++) { + pplx_layer_t *l = &model->weights.layers[i]; + free((void *)l->q_norm); + free((void *)l->k_norm); + free((void *)l->input_norm); + free((void *)l->post_attn_norm); + } + } + free((void *)model->weights.norm); + if (model->safetensors) + multi_safetensors_close((multi_safetensors_t *)model->safetensors); + free(model->weights.layers); + free(model); +} + +pplx_workspace_t *pplx_workspace_new(const pplx_model_t *model) +{ + if (!model) return NULL; + pplx_workspace_t *ws = (pplx_workspace_t *)calloc(1, sizeof(pplx_workspace_t)); + if (!ws) return NULL; + ws->model = model; + return ws; +} + +void pplx_workspace_free(pplx_workspace_t *ws) +{ + if (!ws) return; + free(ws->x); free(ws->x_norm); + free(ws->q); free(ws->k); free(ws->v); + free(ws->attn_out); free(ws->proj_out); + free(ws->ffn_gate); free(ws->ffn_up); + free(ws->attn_scores); + free(ws->offsets); + free(ws->rope_cos); free(ws->rope_sin); + free(ws); +} + +const pplx_config_t *pplx_model_config(const pplx_model_t *model) +{ + return model ? &model->config : NULL; +} + +static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, + const char *name) +{ + if (!sf || !name) return NULL; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) + return &sf->tensors[i]; + } + return NULL; +} + +pplx_late_model_t *pplx_late_model_load(const char *model_dir) +{ + if (!model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "pplx_late_model_load: missing 1_Dense projection\n"); + return NULL; + } + + pplx_late_model_t *late = (pplx_late_model_t *)calloc(1, sizeof(*late)); + if (!late) return NULL; + + late->base = model_load_range_ex(model_dir, 0, -1, 1); + if (!late->base) goto fail; + + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", + model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "pplx_late_model_load: model path too long\n"); + goto fail; + } + + late->dense_sf = safetensors_open(path); + if (!late->dense_sf) { + fprintf(stderr, "pplx_late_model_load: cannot open %s\n", path); + goto fail; + } + + const safetensor_t *t = find_single_safetensor(late->dense_sf, + "linear.weight"); + if (!t) { + fprintf(stderr, "pplx_late_model_load: missing linear.weight\n"); + goto fail; + } + if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) { + fprintf(stderr, "pplx_late_model_load: bad projection shape\n"); + goto fail; + } + + int64_t shape[2] = { t->shape[0], late->base->config.hidden_size }; + if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", + shape, 2, 1)) + goto fail; + + late->projection.dtype = t->dtype; + late->projection.data = safetensors_data(late->dense_sf, t); + late->token_dim = (int)t->shape[0]; + return late; + +fail: + pplx_late_model_free(late); + return NULL; +} + +void pplx_late_model_free(pplx_late_model_t *model) +{ + if (!model) return; + pplx_model_free(model->base); + safetensors_close(model->dense_sf); + free(model); +} + +pplx_late_workspace_t *pplx_late_workspace_new(const pplx_late_model_t *model) +{ + if (!model || !model->base) return NULL; + pplx_late_workspace_t *ws = (pplx_late_workspace_t *)calloc(1, sizeof(*ws)); + if (!ws) return NULL; + ws->model = model; + ws->base_ws = pplx_workspace_new(model->base); + if (!ws->base_ws) { + free(ws); + return NULL; + } + return ws; +} + +void pplx_late_workspace_free(pplx_late_workspace_t *ws) +{ + if (!ws) return; + pplx_workspace_free(ws->base_ws); + free(ws->states); + free(ws); +} + +const pplx_config_t *pplx_late_model_config(const pplx_late_model_t *model) +{ + return model && model->base ? &model->base->config : NULL; +} + +int pplx_late_model_token_dim(const pplx_late_model_t *model) +{ + return model ? model->token_dim : 0; +} + +int pplx_late_model_encode_tokens(const pplx_late_model_t *model, + pplx_late_workspace_t *ws, + const int *token_ids, int n_tokens, + int normalize, float *out_vectors) +{ + if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || + !out_vectors || model->token_dim <= 0) + return -1; + + int hidden = model->base->config.hidden_size; + int dim = model->token_dim; + if (ensure_late_state_buffer(ws, n_tokens) != 0) + return -1; + if (pplx_model_forward_into(model->base, ws->base_ws, token_ids, + n_tokens, ws->states) != 0) + return -1; + + linear_nobias_weight(out_vectors, ws->states, &model->projection, + n_tokens, hidden, dim); + if (normalize) { + for (int i = 0; i < n_tokens; i++) { + if (pplx_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + } + return 0; +} + +float pplx_late_maxsim(const float *query_vectors, int query_tokens, + const float *doc_vectors, int doc_tokens, + int dim) +{ + if (!query_vectors || !doc_vectors || query_tokens <= 0 || + doc_tokens <= 0 || dim <= 0) + return 0.0f; + + float score = 0.0f; + for (int qi = 0; qi < query_tokens; qi++) { + const float *q = query_vectors + (size_t)qi * dim; + float best = -FLT_MAX; + for (int di = 0; di < doc_tokens; di++) { + const float *d = doc_vectors + (size_t)di * dim; + float dot = qwen_dot_f32(q, d, dim); + if (dot > best) best = dot; + } + score += best; + } + return score; +} + +int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens, + const float *doc_vectors, + const int *doc_offsets, int docs, + int dim, float *scores) +{ + if (!query_vectors || !doc_vectors || !doc_offsets || !scores || + query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0) + return -1; + + for (int i = 0; i < docs; i++) { + int start = doc_offsets[i]; + int end = doc_offsets[i + 1]; + if (start < 0 || end <= start) + return -1; + scores[i] = pplx_late_maxsim(query_vectors, query_tokens, + doc_vectors + (size_t)start * dim, + end - start, dim); + } + return 0; +} + +size_t pplx_workspace_nbytes(const pplx_workspace_t *ws) +{ + if (!ws) return 0; + + size_t total = sizeof(*ws); + const pplx_model_t *model = ws->model; + if (!model) return total; + + const pplx_config_t *c = &model->config; + size_t row_floats = 0; + const int dims[] = { + c->hidden_size, /* x */ + c->hidden_size, /* x_norm */ + c->q_dim, /* q */ + c->kv_dim, /* k */ + c->kv_dim, /* v */ + c->q_dim, /* attn_out */ + c->hidden_size, /* proj_out */ + c->intermediate_size, /* ffn_gate */ + c->intermediate_size, /* ffn_up */ + }; + + for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { + if (dims[i] < 0 || + add_size(row_floats, (size_t)dims[i], &row_floats) != 0) + return SIZE_MAX; + } + + size_t buf_floats = 0; + if (ws->buf_seq_cap > 0 && + mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) + return SIZE_MAX; + + size_t rope_floats = 0; + if (ws->rope_cache_cap > 0) { + size_t rope_rows = 0; + if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, + &rope_rows) != 0 || + mul_size(rope_rows, 2, &rope_floats) != 0) + return SIZE_MAX; + } + + size_t all_floats = 0, bytes = 0; + if (add_size(buf_floats, rope_floats, &all_floats) != 0 || + mul_size(all_floats, sizeof(float), &bytes) != 0 || + add_size(total, bytes, &total) != 0) + return SIZE_MAX; + + size_t offset_bytes = 0; + if (ws->offsets_cap > 0 && + mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) + return SIZE_MAX; + if (add_size(total, offset_bytes, &total) != 0) + return SIZE_MAX; + if (add_size(total, ws->attn_scores_bytes, &total) != 0) + return SIZE_MAX; + + return total; +} + +/* ======================================================================== + * Packed forward pass + * ======================================================================== */ + +static int build_offsets(const pplx_input_t *inputs, int batch, int require_ids, + int *offsets, int *total_out, int *max_out) +{ + if (!inputs || batch <= 0 || !offsets || !total_out || !max_out) + return -1; + + int total = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + if ((require_ids && !inputs[b].ids) || inputs[b].n_tokens <= 0) + return -1; + if (total > INT32_MAX - inputs[b].n_tokens) + return -1; + offsets[b] = total; + total += inputs[b].n_tokens; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; + } + offsets[batch] = total; + *total_out = total; + *max_out = max_seq; + return 0; +} + +static int forward_packed_slice_inplace(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_input_t *inputs, + int batch, const int *offsets, + int total_seq, int max_seq, + const float *input_states, + int layer_start, int layer_end, + int apply_final_norm) +{ + if (!model || !ws || ws->model != model || !inputs || + batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0) + return -1; + + const pplx_config_t *cfg = &model->config; + const pplx_weights_t *w = &model->weights; + if (layer_start < 0 || layer_start > layer_end || + layer_end > cfg->n_layers || + layer_start < model->layer_start || layer_end > model->layer_end || + (layer_start == 0 && input_states) || + (layer_start != 0 && !input_states) || + (layer_start == 0 && !w->embed_tokens.data) || + (apply_final_norm && (layer_end != cfg->n_layers || !w->norm))) + return -1; + + int hidden = cfg->hidden_size; + int n_heads = cfg->n_heads; + int n_kv_heads = cfg->n_kv_heads; + int head_dim = cfg->head_dim; + int q_dim = cfg->q_dim; + int kv_dim = cfg->kv_dim; + int inter = cfg->intermediate_size; + float eps = cfg->rms_norm_eps; + + if (ensure_buffers(ws, cfg, total_seq) != 0) return -1; + if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1; + ensure_attention_scores(ws, offsets, batch); + + const float *rope_cos = ws->rope_cos; + const float *rope_sin = ws->rope_sin; + + float *x = ws->x; + float *x_norm = ws->x_norm; + float *q_buf = ws->q; + float *k_buf = ws->k; + float *v_buf = ws->v; + float *attn_out = ws->attn_out; + float *proj_out = ws->proj_out; + float *ffn_gate = ws->ffn_gate; + float *ffn_up = ws->ffn_up; + + /* A first stage performs embedding lookup. Later stages resume from a + * packed hidden-state tensor received from the preceding stage. */ + if (layer_start == 0) { + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + for (int i = 0; i < inputs[b].n_tokens; i++) { + int id = inputs[b].ids[i]; + if (id < 0 || id >= cfg->vocab_size) { + fprintf(stderr, "pplx: invalid token id %d at input %d position %d\n", + id, b, i); + return -1; + } + copy_weight_row(x + (size_t)(start + i) * hidden, + &w->embed_tokens, (size_t)id, hidden); + } + } + } else { + memcpy(x, input_states, (size_t)total_seq * hidden * sizeof(float)); + } + + float scale = 1.0f / sqrtf((float)head_dim); + + /* 2. Transformer layers. Linear/MLP ops run over all packed tokens; + * attention is explicitly block-diagonal by sequence offsets. */ + for (int layer = layer_start; layer < layer_end; layer++) { + const pplx_layer_t *l = &w->layers[layer]; + + qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); + + linear_qkv_weight(q_buf, k_buf, v_buf, x_norm, + &l->wq, &l->wk, &l->wv, + total_seq, hidden, q_dim, kv_dim); + + qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); + qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); + + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + int len = offsets[b + 1] - start; + qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, + rope_cos, rope_sin, len, n_heads, head_dim); + qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, + rope_cos, rope_sin, len, n_kv_heads, head_dim); + } + + qwen_bidirectional_gqa_attention_packed_with_scratch( + attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, + n_kv_heads, head_dim, scale, ws->attn_scores, + ws->attn_scores_bytes); + + linear_nobias_weight(proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); + qwen_add_inplace(x, proj_out, total_seq * hidden); + + qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); + + linear_pair_weight(ffn_gate, ffn_up, x_norm, + &l->gate_proj, &l->up_proj, + total_seq, hidden, inter, inter); + qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); + linear_nobias_weight(proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); + qwen_add_inplace(x, proj_out, total_seq * hidden); + } + + if (apply_final_norm) + qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps); + return 0; +} + +static int forward_packed_inplace(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_input_t *inputs, + int batch, const int *offsets, + int total_seq, int max_seq, + int apply_final_norm) +{ + return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, + total_seq, max_seq, NULL, 0, + model->config.n_layers, + apply_final_norm); +} + +static void pool_embeddings(const pplx_model_t *model, + const pplx_workspace_t *ws, + const int *offsets, int batch, + float *out_embeddings) +{ + const pplx_config_t *cfg = &model->config; + int hidden = cfg->hidden_size; + float eps = cfg->rms_norm_eps; + const float *norm_weight = model->weights.norm; + const float *x = ws->x; + + for (int b = 0; b < batch; b++) { + int start = offsets[b]; + int end = offsets[b + 1]; + int len = end - start; + float *emb = out_embeddings + (size_t)b * hidden; + + memset(emb, 0, (size_t)hidden * sizeof(float)); + for (int i = start; i < end; i++) { + const float *row = x + (size_t)i * hidden; + + float sum_sq = 0.0f; + for (int d = 0; d < hidden; d++) + sum_sq += row[d] * row[d]; + + float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); + for (int d = 0; d < hidden; d++) + emb[d] += row[d] * rms_inv * norm_weight[d]; + } + + float inv_len = 1.0f / (float)len; + for (int d = 0; d < hidden; d++) + emb[d] *= inv_len; + + } +} + +/* ======================================================================== + * Public forward/embed APIs + * ======================================================================== */ + +int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + float *out_states) +{ + if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) return -1; + + size_t count, bytes; + if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, + &count) != 0 || + mul_size(count, sizeof(float), &bytes) != 0) + return -1; + + pplx_input_t input = { token_ids, n_tokens }; + int offsets[2] = { 0, n_tokens }; + if (forward_packed_inplace(model, ws, &input, 1, offsets, + n_tokens, n_tokens, 1) != 0) + return -1; + + memcpy(out_states, ws->x, bytes); + return 0; +} + +int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings) +{ + if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || + !out_embeddings) + return -1; + + int hidden = cfg->hidden_size; + if (hidden <= 0) return -1; + + for (int s = 0; s < n_spans; s++) { + int start = spans[s].start; + int len = spans[s].n_tokens; + if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) + return -1; + + float *emb = out_embeddings + (size_t)s * hidden; + memset(emb, 0, (size_t)hidden * sizeof(float)); + + for (int t = 0; t < len; t++) { + const float *row = states + (size_t)(start + t) * hidden; + for (int d = 0; d < hidden; d++) + emb[d] += row[d]; + } + + float inv_len = 1.0f / (float)len; + for (int d = 0; d < hidden; d++) { + emb[d] *= inv_len; + } + } + return 0; +} + +int pplx_model_embed_spans_batch(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings) +{ + if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) + return -1; + + if (ensure_offsets(ws, batch) != 0) return -1; + if ((size_t)batch > SIZE_MAX / sizeof(pplx_input_t)) return -1; + pplx_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); + if (!packed_inputs) return -1; + int *offsets = ws->offsets; + int total_seq = 0; + int max_seq = 0; + int total_spans = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || + input->n_spans <= 0 || total_seq > INT_MAX - n_tokens || + total_spans > INT_MAX - input->n_spans) { + free(packed_inputs); + return -1; + } + packed_inputs[b] = input->input; + offsets[b] = total_seq; + total_seq += n_tokens; + total_spans += input->n_spans; + if (n_tokens > max_seq) max_seq = n_tokens; + for (int s = 0; s < input->n_spans; s++) { + int start = input->spans[s].start; + int len = input->spans[s].n_tokens; + if (start < 0 || len <= 0 || start > n_tokens || + len > n_tokens - start) { + free(packed_inputs); + return -1; + } + } + } + offsets[batch] = total_seq; + + if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, + total_seq, max_seq, 1) != 0) { + free(packed_inputs); + return -1; + } + free(packed_inputs); + + int span_offset = 0; + int hidden = model->config.hidden_size; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + if (pplx_pool_spans(&model->config, + ws->x + (size_t)offsets[b] * hidden, + input->input.n_tokens, input->spans, input->n_spans, + out_embeddings + (size_t)span_offset * hidden) != 0) + return -1; + span_offset += input->n_spans; + } + return 0; +} + +int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings) +{ + pplx_context_input_t input = { + .input = { token_ids, n_tokens }, + .spans = spans, + .n_spans = n_spans, + }; + return pplx_model_embed_spans_batch(model, ws, &input, 1, out_embeddings); +} + +float *pplx_model_forward(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens) +{ + if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; + + size_t n; + if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0) + return NULL; + float *out = malloc_floats(n); + if (!out) return NULL; + + if (pplx_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; + } + return out; +} + +int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws, + const pplx_input_t *inputs, int batch, + float *out_embeddings) +{ + if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; + + if (ensure_offsets(ws, batch) != 0) return -1; + int *offsets = ws->offsets; + + int total_seq = 0; + int max_seq = 0; + if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0) + return -1; + + int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, + total_seq, max_seq, 0); + if (rc == 0) + pool_embeddings(model, ws, offsets, batch, out_embeddings); + + return rc; +} + +int pplx_model_forward_slice_batch(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_input_t *inputs, int batch, + const float *input_states, + int layer_start, int layer_end, + int apply_final_norm, + float *out_states) +{ + if (!model || !ws || !inputs || batch <= 0 || !out_states || + layer_start < 0 || layer_start > layer_end || + layer_end > model->config.n_layers) + return -1; + + if (ensure_offsets(ws, batch) != 0) return -1; + int *offsets = ws->offsets; + + int total_seq = 0; + int max_seq = 0; + if (build_offsets(inputs, batch, layer_start == 0, offsets, + &total_seq, &max_seq) != 0) + return -1; + + int rc = forward_packed_slice_inplace(model, ws, inputs, batch, offsets, + total_seq, max_seq, input_states, + layer_start, layer_end, + apply_final_norm); + if (rc == 0) { + size_t count; + if (mul_size((size_t)total_seq, + (size_t)model->config.hidden_size, &count) != 0) + return -1; + memcpy(out_states, ws->x, count * sizeof(float)); + } + return rc; +} + +int pplx_pool_batch(const pplx_config_t *cfg, const float *states, + const int *seq_lengths, int batch, + float *out_embeddings) +{ + if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || + cfg->hidden_size <= 0) + return -1; + + int offset = 0; + for (int b = 0; b < batch; b++) { + int len = seq_lengths[b]; + if (len <= 0 || offset > INT_MAX - len) + return -1; + + float *emb = out_embeddings + (size_t)b * cfg->hidden_size; + memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); + for (int t = 0; t < len; t++) { + const float *row = states + (size_t)(offset + t) * cfg->hidden_size; + for (int d = 0; d < cfg->hidden_size; d++) + emb[d] += row[d]; + } + + float inv_len = 1.0f / (float)len; + for (int d = 0; d < cfg->hidden_size; d++) + emb[d] *= inv_len; + offset += len; + } + return 0; +} + +int pplx_model_embed_into(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + float *out_embedding) +{ + pplx_input_t input = { token_ids, n_tokens }; + return pplx_model_embed_batch(model, ws, &input, 1, out_embedding); +} + +float *pplx_model_embed(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens) +{ + if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; + + int hidden = model->config.hidden_size; + float *emb = malloc_floats((size_t)hidden); + if (!emb) return NULL; + + if (pplx_model_embed_into(model, ws, token_ids, n_tokens, emb) != 0) { + free(emb); + return NULL; + } + return emb; +} + +/* ======================================================================== + * Vector helpers + * ======================================================================== */ + +int pplx_l2_normalize(float *vec, int dim) +{ + if (!vec || dim <= 0) return -1; + + float norm_sq = 0.0f; + for (int i = 0; i < dim; i++) + norm_sq += vec[i] * vec[i]; + + if (!(norm_sq > 0.0f) || !isfinite(norm_sq)) return -1; + + float inv_norm = 1.0f / sqrtf(norm_sq); + for (int i = 0; i < dim; i++) + vec[i] *= inv_norm; + return 0; +} + +float pplx_cosine_similarity(const float *a, const float *b, int dim) +{ + if (!a || !b || dim <= 0) return 0.0f; + + float dot = 0.0f; + float norm_a_sq = 0.0f; + float norm_b_sq = 0.0f; + for (int i = 0; i < dim; i++) { + dot += a[i] * b[i]; + norm_a_sq += a[i] * a[i]; + norm_b_sq += b[i] * b[i]; + } + + if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || + !isfinite(norm_a_sq) || !isfinite(norm_b_sq)) + return 0.0f; + return dot / sqrtf(norm_a_sq * norm_b_sq); +} diff --git a/embed.h b/embed.h new file mode 100644 index 0000000..5a2501b --- /dev/null +++ b/embed.h @@ -0,0 +1,293 @@ +/* + * pplx_embed.h - Pure C inference for Perplexity AI's pplx-embed models. + * + * Supports all pplx-embed-v1 and pplx-embed-context-v1 variants (0.6B, 4B). + */ + +#ifndef PPLX_EMBED_H +#define PPLX_EMBED_H + +#include +#include + +/* ======================================================================== + * Shared constants (identical across all models) + * ======================================================================== */ + +#define PPLX_VOCAB_SIZE 151936 +#define PPLX_HEAD_DIM 128 +#define PPLX_MAX_LAYERS 64 /* upper bound for stack arrays */ +#define PPLX_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ + +/* ======================================================================== + * Model Configuration (populated from config.json at load time) + * ======================================================================== */ + +typedef struct { + int hidden_size; /* 0.6B: 1024 4B: 2560 */ + int n_layers; /* 0.6B: 28 4B: 36 */ + int n_heads; /* 0.6B: 16 4B: 32 */ + int n_kv_heads; /* 0.6B: 8 4B: 8 */ + int head_dim; /* always 128 */ + int q_dim; /* n_heads * head_dim */ + int kv_dim; /* n_kv_heads * head_dim */ + int intermediate_size; /* 0.6B: 3072 4B: 9728 */ + int vocab_size; /* 151936 */ + float rms_norm_eps; /* 1e-6 */ + float rope_theta; /* 1e6 */ +} pplx_config_t; + +/* ======================================================================== + * Model + workspace ownership + * ======================================================================== */ + +typedef struct pplx_model pplx_model_t; +typedef struct pplx_workspace pplx_workspace_t; +typedef struct pplx_late_model pplx_late_model_t; +typedef struct pplx_late_workspace pplx_late_workspace_t; + +typedef struct { + const int *ids; + int n_tokens; +} pplx_input_t; + +typedef struct { + int start; + int n_tokens; +} pplx_span_t; + +typedef struct { + pplx_input_t input; + const pplx_span_t *spans; + int n_spans; +} pplx_context_input_t; + +/* ======================================================================== + * API + * ======================================================================== */ + +/* + * Load immutable model data from a directory containing: + * config.json - model hyperparameters + * model.safetensors (or sharded) - F32 or BF16 weights + * vocab.json + merges.txt - BPE tokenizer + * + * Reads config.json to determine model size (0.6B vs 4B). + * Returns NULL on error. + */ +pplx_model_t *pplx_model_load(const char *model_dir); + +/* + * Load one contiguous transformer layer range for distributed execution. + * + * layer_start is inclusive and layer_end is exclusive. A range beginning at + * zero also loads the token embedding table. A range ending at n_layers also + * loads the final RMSNorm. Safetensors remain mmap-backed, so unselected + * weights are not touched. + */ +pplx_model_t *pplx_model_load_slice(const char *model_dir, + int layer_start, int layer_end); + +void pplx_model_free(pplx_model_t *model); + +/* + * Allocate/free mutable scratch buffers for a model. The model must outlive + * workspaces created from it. + */ +pplx_workspace_t *pplx_workspace_new(const pplx_model_t *model); +void pplx_workspace_free(pplx_workspace_t *ws); + +/* Config accessor for opaque model ownership. */ +const pplx_config_t *pplx_model_config(const pplx_model_t *model); + +/* + * Current mutable workspace memory owned by this workspace, in bytes. + * + * This includes the workspace struct, scratch buffers, and RoPE cache capacity. + * It does not include immutable mmap'd model weights. + */ +size_t pplx_workspace_nbytes(const pplx_workspace_t *ws); + +/* + * Compute embedding for a token sequence. + * + * 1. Token embedding lookup + * 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU) + * 3. Final RMSNorm + * 4. Mean pooling over all positions + * + * Perplexity embeddings are intentionally not L2-normalized. Use cosine + * similarity for comparisons, or normalize explicitly before storing in vector + * databases that only support inner product. + * + * Returns malloc'd float[hidden_size] (caller frees). NULL on error. + */ +float *pplx_model_embed(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens); + +/* + * Compute one embedding into caller-provided out[hidden_size]. + * Returns 0 on success, -1 on error. + */ +int pplx_model_embed_into(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + float *out_embedding); + +/* + * Compute embeddings for a true packed/ragged batch. + * + * out_embeddings is caller-provided [batch, hidden_size]. + * Each sequence attends only to tokens from the same input, and RoPE + * positions restart at zero for every input. + * + * Returns 0 on success, -1 on error. + */ +int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws, + const pplx_input_t *inputs, int batch, + float *out_embeddings); + +/* + * Run a contiguous transformer layer range for a packed/ragged batch. + * + * layer_start is inclusive and layer_end is exclusive. For a range beginning + * at layer 0, input_states must be NULL and every input must provide token + * ids. For a later range, input_states must contain the preceding range's + * packed [sum(n_tokens), hidden_size] output; token ids are not required. + * + * apply_final_norm is valid only for a range ending at the final transformer + * layer. out_states receives packed [sum(n_tokens), hidden_size] states. + * + * This is the core execution boundary used by layer-sharded inference. + */ +int pplx_model_forward_slice_batch(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_input_t *inputs, int batch, + const float *input_states, + int layer_start, int layer_end, + int apply_final_norm, + float *out_states); + +/* + * Mean-pool packed final hidden states for a ragged batch. + * + * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive + * token count per sequence. The states must already include final RMSNorm. + */ +int pplx_pool_batch(const pplx_config_t *cfg, const float *states, + const int *seq_lengths, int batch, + float *out_embeddings); + +/* + * Run the transformer forward pass WITHOUT pooling. + * Returns the full [n_tokens * hidden_size] output after final RMSNorm. + * Caller frees the returned buffer. NULL on error. + * + * Useful for contextual (late-chunking) models where you need + * per-token embeddings before splitting by separator positions. + */ +float *pplx_model_forward(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens); + +/* + * Run the transformer forward pass into caller-provided + * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error. + */ +int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + float *out_states); + +/* + * Pool embeddings from final hidden states. + * + * states is [n_tokens, hidden_size], normally produced by + * pplx_model_forward_into(). Each span is mean pooled into + * out_embeddings[n_spans, hidden_size] without L2 normalization. + */ +int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings); + +/* + * Run one contextual sequence and pool selected token spans. + * Returns 0 on success, -1 on error. + */ +int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings); + +/* + * Run a packed/ragged contextual document batch and pool every selected span. + * + * Documents attend independently, RoPE positions restart for each document, + * and out_embeddings contains spans in document order. + */ +int pplx_model_embed_spans_batch(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings); + +/* + * L2-normalize vec[dim] in place. + * + * Returns 0 on success. Returns -1 for invalid arguments or a zero-length + * vector, which cannot be normalized. + */ +int pplx_l2_normalize(float *vec, int dim); + +/* + * Cosine similarity between two vectors. + * Returns 0 for invalid arguments or if either vector has zero length. + */ +float pplx_cosine_similarity(const float *a, const float *b, int dim); + +/* + * Late-interaction model API. + * + * A late model produces one 128D vector per retained token. The vectors are + * projected by the snapshot's 1_Dense head and L2-normalized per token by + * default, matching ColBERT/MaxSim scoring semantics. + */ +pplx_late_model_t *pplx_late_model_load(const char *model_dir); +void pplx_late_model_free(pplx_late_model_t *model); + +pplx_late_workspace_t *pplx_late_workspace_new(const pplx_late_model_t *model); +void pplx_late_workspace_free(pplx_late_workspace_t *ws); + +const pplx_config_t *pplx_late_model_config(const pplx_late_model_t *model); +int pplx_late_model_token_dim(const pplx_late_model_t *model); + +/* + * Encode token_ids into out_vectors[n_tokens, token_dim]. + * + * If normalize is non-zero, every non-zero token vector is L2-normalized. + */ +int pplx_late_model_encode_tokens(const pplx_late_model_t *model, + pplx_late_workspace_t *ws, + const int *token_ids, int n_tokens, + int normalize, float *out_vectors); + +/* + * Sum over query tokens of max dot product against document tokens. + */ +float pplx_late_maxsim(const float *query_vectors, int query_tokens, + const float *doc_vectors, int doc_tokens, + int dim); + +/* + * Score one query against a packed/ragged batch of documents. + * + * doc_offsets has docs + 1 entries. Document i is stored in + * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim). + * scores is caller-provided [docs]. + */ +int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens, + const float *doc_vectors, + const int *doc_offsets, int docs, + int dim, float *scores); + +/* Verbose level: 0=quiet, 1=info, 2=debug */ +extern int pplx_verbose; +extern int qwen_verbose; + +#endif /* PPLX_EMBED_H */ diff --git a/embed_cli.c b/embed_cli.c new file mode 100644 index 0000000..140494b --- /dev/null +++ b/embed_cli.c @@ -0,0 +1,825 @@ +/* main.c - pplx_embed command-line tool */ + +#include "pplx_embed.h" +#include "pplx_distributed.h" +#include "qwen_kernels.h" +#include "qwen_tokenizer.h" + +#ifdef USE_MLX +#include "pplx_embed_mlx.h" +#endif +#ifdef USE_CUDA +#include "pplx_embed_cuda.h" +#endif + +#include +#include +#include +#include +#include + +static double now_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6; +} + +static void print_usage(const char *prog) +{ + fprintf(stderr, + "Usage: %s -d [options] [text...]\n" + "\n" + "Options:\n" + " -d Model directory (required)\n" + " --mlx Use Apple MLX GPU backend\n" + " --cuda Use CUDA/cuBLAS GPU backend\n" + " --stream Read lines from stdin, write one JSON embedding per line\n" + " --dist-worker\n" + " Serve one final distributed layer shard\n" + " --dist-remote HOST:PORT\n" + " Use one remote final shard for CLI or --stream inference\n" + " --layers A:B Distributed half-open layer range, for example 0:14\n" + " --dist-activation-bits N\n" + " Distributed activation transport width: 32 or 16 (default: 32)\n" + " --mlx-quant-bits N\n" + " Quantize MLX linear weights to 8 bits at load time\n" + " --mlx-quant-group-size N\n" + " MLX quantization group size (default: 64)\n" + " --cuda-gemm-mode MODE\n" + " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" + " --cuda-weight-dtype DTYPE\n" + " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" + " weight memory and uses BF16 tensor cores\n" + " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n" + " --port N Bind port for --dist-worker (default: 8000)\n" + " -b, --batch-size N\n" + " Max texts per engine batch (default: all; --stream: 1)\n" + " -t, --threads N\n" + " CPU threads (default: all cores)\n" + " -e, --embeddings\n" + " Print raw embeddings (with multiple texts)\n" + " -v, --verbose\n" + " Verbose (-vv for debug)\n" + " -h Show this help\n" + "\n" + "Modes:\n" + " 1 text arg Print embedding as space-separated floats\n" + " 2+ text args Print cosine similarity matrix\n" + " no args Batch: read all stdin lines, then similarity matrix\n" + " --stream Streaming: read stdin lines, write JSON per line\n" + " --dist-worker Final layer-shard worker\n" + "\n" + "The HTTP API is served by the separate pplx-embed-server binary.\n" + "\n" + "Examples:\n" + " %s -d ./model \"query: what is AI?\"\n" + " %s -d ./model --mlx --stream < texts.txt\n" + " %s -d ./model --dist-worker --layers 14:28 --port 9000\n" + " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n", + prog, prog, prog, prog, prog); +} + +/* ======================================================================== + * Embed one text: tokenize > forward > return float[dim] + * ======================================================================== */ + +typedef struct { + pplx_model_t *model; + pplx_workspace_t *workspace; + pplx_dist_client_t *dist; +#ifdef USE_MLX + pplx_mlx_ctx_t *mlx_ctx; +#endif +#ifdef USE_CUDA + pplx_cuda_ctx_t *cuda_ctx; +#endif + qwen_tokenizer_t *tok; + qwen_tokenizer_workspace_t *tok_ws; + int dim; +} engine_t; + +typedef struct { + int *ids; + int n_tokens; +} token_buf_t; + +static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, + int batch, float *out_embeddings) +{ + if (e->dist) + return pplx_dist_client_embed_batch(e->dist, inputs, batch, + out_embeddings); +#ifdef USE_MLX + if (e->mlx_ctx) + return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings); +#endif +#ifdef USE_CUDA + if (e->cuda_ctx) + return pplx_cuda_embed_batch(e->cuda_ctx, inputs, batch, out_embeddings); +#endif + return pplx_model_embed_batch(e->model, e->workspace, + inputs, batch, out_embeddings); +} + +static int parse_layers(const char *arg, int *start, int *end) +{ + int used = 0; + if (!arg || sscanf(arg, "%d:%d%n", start, end, &used) != 2 || + arg[used] != '\0' || *start < 0 || *start >= *end) + return -1; + return 0; +} + +static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port) +{ + const char *colon = arg ? strrchr(arg, ':') : NULL; + if (!colon || colon == arg || !colon[1]) return -1; + size_t n = (size_t)(colon - arg); + if (n >= hostlen) return -1; + memcpy(host, arg, n); + host[n] = '\0'; + int used = 0; + if (sscanf(colon + 1, "%d%n", port, &used) != 1 || + colon[1 + used] != '\0' || *port <= 0 || *port > 65535) + return -1; + return 0; +} + +static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) +{ + memset(out, 0, sizeof(*out)); + out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, + text, &out->n_tokens); + if (!out->ids || out->n_tokens == 0) { + fprintf(stderr, "tokenization failed: %s\n", text); + free(out->ids); + out->ids = NULL; + out->n_tokens = 0; + return -1; + } + + if (pplx_verbose >= 1) { + fprintf(stderr, "tokens (%d): ", out->n_tokens); + for (int i = 0; i < out->n_tokens && i < 20; i++) + fprintf(stderr, "%d ", out->ids[i]); + if (out->n_tokens > 20) fprintf(stderr, "..."); + fprintf(stderr, "\n"); + } + return 0; +} + +static void free_tokens(token_buf_t *tokens, int n) +{ + if (!tokens) return; + for (int i = 0; i < n; i++) + free(tokens[i].ids); +} + +/* ======================================================================== + * Output helpers + * ======================================================================== */ + +static void print_embedding_raw(const float *emb, int dim) +{ + for (int i = 0; i < dim; i++) { + if (i > 0) putchar(' '); + printf("%.8f", (double)emb[i]); + } + putchar('\n'); +} + +static size_t engine_workspace_nbytes(const engine_t *e) +{ + return (e && e->workspace) ? pplx_workspace_nbytes(e->workspace) : 0; +} + +static void print_embedding_json(const float *emb, int dim, int n_tokens, + double ms, size_t workspace_bytes) +{ + printf("{\"embedding\":["); + for (int i = 0; i < dim; i++) { + if (i > 0) putchar(','); + printf("%.8f", (double)emb[i]); + } + printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", + dim, n_tokens, ms, workspace_bytes); + fflush(stdout); +} + +/* ======================================================================== + * Stdin mode: read lines from stdin, write JSON to stdout + * ======================================================================== */ + +static int process_stdin_batch(engine_t *e, char **lines, int n_lines) +{ + if (n_lines <= 0) return 0; + + token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t)); + pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_lines * sizeof(pplx_input_t)); + int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int)); + float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float)); + if (!tokens || !inputs || !input_to_line || !embs) { + fprintf(stderr, "OOM\n"); + free(tokens); free(inputs); free(input_to_line); free(embs); + for (int i = 0; i < n_lines; i++) + printf("{\"error\":\"embedding failed\"}\n"); + fflush(stdout); + return 1; + } + + int n_inputs = 0; + for (int i = 0; i < n_lines; i++) { + if (tokenize_text(e, lines[i], &tokens[i]) == 0) { + inputs[n_inputs].ids = tokens[i].ids; + inputs[n_inputs].n_tokens = tokens[i].n_tokens; + input_to_line[n_inputs] = i; + n_inputs++; + } + } + + double ms = 0.0; + int rc = 0; + if (n_inputs > 0) { + double t0 = now_ms(); + rc = engine_embed_batch(e, inputs, n_inputs, embs); + ms = now_ms() - t0; + if (pplx_verbose >= 1) + fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms); + } + + int next_valid = 0; + for (int i = 0; i < n_lines; i++) { + if (!tokens[i].ids) { + printf("{\"error\":\"tokenization failed\"}\n"); + continue; + } + if (rc != 0) { + printf("{\"error\":\"embedding failed\"}\n"); + continue; + } + while (next_valid < n_inputs && input_to_line[next_valid] != i) + next_valid++; + if (next_valid >= n_inputs) { + printf("{\"error\":\"embedding failed\"}\n"); + continue; + } + print_embedding_json(embs + (size_t)next_valid * e->dim, + e->dim, tokens[i].n_tokens, ms, + engine_workspace_nbytes(e)); + next_valid++; + } + fflush(stdout); + + free_tokens(tokens, n_lines); + free(tokens); free(inputs); free(input_to_line); free(embs); + return rc == 0 ? 0 : 1; +} + +static int run_stdin(engine_t *e, int batch_size) +{ + char line[65536]; + if (batch_size <= 0) batch_size = 1; + + if (pplx_verbose >= 1) + fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", + batch_size); + + char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *)); + if (!batch_lines) { + fprintf(stderr, "OOM\n"); + return 1; + } + int n_batch = 0; + int rc = 0; + + while (fgets(line, sizeof(line), stdin)) { + /* Strip newline */ + size_t len = strlen(line); + while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r')) + line[--len] = '\0'; + if (len == 0) continue; + + if (pplx_verbose >= 1) + fprintf(stderr, "stdin: \"%.*s%s\"\n", + (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : ""); + + batch_lines[n_batch] = strdup(line); + if (!batch_lines[n_batch]) { + fprintf(stderr, "OOM\n"); + rc = 1; + break; + } + n_batch++; + + if (n_batch == batch_size) { + if (process_stdin_batch(e, batch_lines, n_batch) != 0) + rc = 1; + for (int i = 0; i < n_batch; i++) { + free(batch_lines[i]); + batch_lines[i] = NULL; + } + n_batch = 0; + } + } + + if (n_batch > 0) { + if (process_stdin_batch(e, batch_lines, n_batch) != 0) + rc = 1; + for (int i = 0; i < n_batch; i++) + free(batch_lines[i]); + } + free(batch_lines); + + if (pplx_verbose >= 1) + fprintf(stderr, "stdin: EOF\n"); + + return rc; +} + +/* ======================================================================== + * Batch mode: embed args or stdin lines, then print similarity or vectors + * ======================================================================== */ + +static int append_text(char ***texts, int *n_texts, int *cap, const char *s) +{ + if (*n_texts == *cap) { + int new_cap = *cap ? *cap * 2 : 8; + char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *)); + if (!new_texts) return -1; + *texts = new_texts; + *cap = new_cap; + } + + (*texts)[*n_texts] = strdup(s); + if (!(*texts)[*n_texts]) return -1; + (*n_texts)++; + return 0; +} + +static int run_batch(engine_t *e, int argc, char **argv, int arg_start, + int print_embs, int batch_size) +{ + int n_texts = 0, cap = 0; + char **texts = NULL; + int rc = 0; + + if (arg_start < argc) { + for (int i = arg_start; i < argc; i++) { + if (append_text(&texts, &n_texts, &cap, argv[i]) != 0) + goto oom_texts; + } + } else { + char line[65536]; + while (fgets(line, sizeof(line), stdin)) { + size_t l = strlen(line); + while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r')) + line[--l] = '\0'; + if (l == 0) continue; + if (append_text(&texts, &n_texts, &cap, line) != 0) + goto oom_texts; + } + } + + if (n_texts == 0) { + fprintf(stderr, "no input texts\n"); + free(texts); + return 1; + } + + int dim = e->dim; + token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t)); + pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_texts * sizeof(pplx_input_t)); + float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float)); + if (!tokens || !inputs || !embs) { + fprintf(stderr, "OOM\n"); + free(tokens); free(inputs); free(embs); + for (int i = 0; i < n_texts; i++) free(texts[i]); + free(texts); + return 1; + } + + for (int i = 0; i < n_texts; i++) { + if (pplx_verbose >= 1) + fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]); + if (tokenize_text(e, texts[i], &tokens[i]) != 0) { + rc = 1; + goto done; + } + inputs[i].ids = tokens[i].ids; + inputs[i].n_tokens = tokens[i].n_tokens; + } + + int max_batch = batch_size > 0 ? batch_size : n_texts; + for (int start = 0; start < n_texts; start += max_batch) { + int cur = n_texts - start; + if (cur > max_batch) cur = max_batch; + + double t0 = now_ms(); + if (engine_embed_batch(e, inputs + start, cur, + embs + (size_t)start * dim) != 0) { + fprintf(stderr, "forward pass failed\n"); + rc = 1; + goto done; + } + if (pplx_verbose >= 1) + fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", + start, start + cur - 1, cur, now_ms() - t0); + } + + if (n_texts == 1) { + print_embedding_raw(embs, dim); + } else { + printf("Cosine similarity matrix (%d texts):\n", n_texts); + printf("%-4s", ""); + for (int j = 0; j < n_texts; j++) printf(" [%d] ", j); + printf("\n"); + for (int i = 0; i < n_texts; i++) { + printf("[%d] ", i); + for (int j = 0; j < n_texts; j++) + printf(" %.4f", (double)pplx_cosine_similarity( + embs + (size_t)i * dim, embs + (size_t)j * dim, dim)); + printf(" \"%s\"\n", texts[i]); + } + if (print_embs) { + printf("\nEmbeddings:\n"); + for (int i = 0; i < n_texts; i++) { + printf("[%d] ", i); + print_embedding_raw(embs + (size_t)i * dim, dim); + } + } + } + +done: + free_tokens(tokens, n_texts); + for (int i = 0; i < n_texts; i++) free(texts[i]); + free(texts); free(tokens); free(inputs); free(embs); + return rc; + +oom_texts: + fprintf(stderr, "OOM\n"); + for (int i = 0; i < n_texts; i++) free(texts[i]); + free(texts); + return 1; +} + +/* ======================================================================== + * Main + * ======================================================================== */ + +int main(int argc, char *argv[]) +{ + const char *model_dir = NULL; + int n_threads = 0; + int print_embs = 0; + int verbose = 0; + int use_mlx = 0; + int use_cuda = 0; + int stdin_mode = 0; + int dist_worker_mode = 0; + const char *dist_remote = NULL; + int layer_start = -1; + int layer_end = -1; + int batch_size = 0; + int mlx_quantize_bits = 0; + int mlx_quantize_group_size = 64; + const char *cuda_fast_gemm = NULL; + const char *cuda_weights = NULL; + int dist_activation_bits = 32; + const char *host = "127.0.0.1"; + int port = 8000; + + int arg_start = 1; + while (arg_start < argc && argv[arg_start][0] == '-') { + const char *f = argv[arg_start]; + if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; } + else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + n_threads = atoi(argv[++arg_start]); + } + else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + batch_size = atoi(argv[++arg_start]); + } + else if (!strcmp(f, "--backend")) { + const char *backend = argv[++arg_start]; + if (!strcmp(backend, "mlx")) { +#ifdef USE_MLX + use_mlx = 1; + use_cuda = 0; +#else + fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); + return 1; +#endif + } else if (!strcmp(backend, "cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); + return 1; +#endif + } else if (!strcmp(backend, "cpu")) { + use_mlx = 0; + use_cuda = 0; + } else { + fprintf(stderr, "invalid --backend: %s\n", backend); + return 1; + } + } + else if (!strcmp(f, "--mlx-quant-bits")) { + mlx_quantize_bits = atoi(argv[++arg_start]); + if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { + fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); + return 1; + } + } + else if (!strcmp(f, "--mlx-quant-group-size")) { + mlx_quantize_group_size = atoi(argv[++arg_start]); + if (mlx_quantize_group_size <= 0) { + fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); + return 1; + } + } + else if (!strcmp(f, "--cuda-gemm-mode")) { + cuda_fast_gemm = argv[++arg_start]; + } + else if (!strcmp(f, "--cuda-weight-dtype")) { + cuda_weights = argv[++arg_start]; + } + else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } + else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } + else if (!strcmp(f, "--layers")) { + if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) { + fprintf(stderr, "--layers expects START:END with START < END\n"); + return 1; + } + } + else if (!strcmp(f, "--dist-activation-bits")) { + dist_activation_bits = atoi(argv[++arg_start]); + if (dist_activation_bits != 32 && dist_activation_bits != 16) { + fprintf(stderr, "--dist-activation-bits must be 32 or 16\n"); + return 1; + } + } + else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { + print_embs = 1; + } + else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } + else if (!strcmp(f, "-vv")) { verbose = 2; } + else if (!strcmp(f, "--stream")) { stdin_mode = 1; } + else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; } + else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; } + else if (!strcmp(f, "--mlx")) { +#ifdef USE_MLX + use_mlx = 1; + use_cuda = 0; +#else + fprintf(stderr, "--mlx not available (build with: make mlx)\n"); + return 1; +#endif + } + else if (!strcmp(f, "--cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "--cuda not available (build with: make cuda)\n"); + return 1; +#endif + } + else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + print_usage(argv[0]); + return 0; + } + else break; + arg_start++; + } + + pplx_verbose = verbose; + qwen_verbose = verbose; + + if (cuda_fast_gemm) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-gemm-mode requires --cuda or --backend cuda\n"); + return 1; + } + if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { + fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); + return 1; + } +#else + fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); + return 1; +#endif + } + + if (cuda_weights) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-weight-dtype requires --cuda or --backend cuda\n"); + return 1; + } + if (!strcmp(cuda_weights, "bf16")) { + pplx_cuda_set_weights_bf16(1); + } else if (!strcmp(cuda_weights, "f32")) { + pplx_cuda_set_weights_bf16(0); + } else { + fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); + return 1; + } +#else + fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); + return 1; +#endif + } + + if (!model_dir || !model_dir[0]) { + fprintf(stderr, "model directory required (-d )\n"); + print_usage(argv[0]); + return 1; + } + + if (dist_worker_mode && dist_remote) { + fprintf(stderr, "--dist-worker and --dist-remote are mutually exclusive\n"); + return 1; + } + if ((dist_worker_mode || dist_remote) && + (layer_start < 0 || layer_end < 0)) { + fprintf(stderr, "distributed mode requires --layers START:END\n"); + return 1; + } + if (dist_activation_bits != 32 && !dist_remote) { + fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n"); + return 1; + } + if (dist_remote && layer_start != 0) { + fprintf(stderr, "--dist-remote coordinator layers must start at zero\n"); + return 1; + } + if ((dist_worker_mode || dist_remote) && use_cuda) { + fprintf(stderr, "distributed mode is not available with --cuda yet\n"); + return 1; + } + + /* Threads (CPU backend) */ + if (!use_mlx && !use_cuda) { + if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + qwen_set_threads(n_threads); + if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + } else { + if (verbose >= 1) + fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); + } + + if (mlx_quantize_bits && !use_mlx) { + fprintf(stderr, "--mlx-quant-bits requires --mlx or --backend mlx\n"); + return 1; + } + + pplx_dist_options_t dist_opts = { + .mlx_quantize_bits = mlx_quantize_bits, + .mlx_quantize_group_size = mlx_quantize_group_size, + .activation_bits = dist_activation_bits, + }; + + if (dist_worker_mode) + return pplx_dist_run_worker_with_options( + model_dir, host, port, layer_start, layer_end, use_mlx, + &dist_opts); + + /* Tokenizer */ + char vocab_path[1024]; + snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); + double t0 = now_ms(); + qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; } + if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0); + + /* Model */ + engine_t e = {0}; + e.tok = tok; + t0 = now_ms(); + +#ifdef USE_MLX + if (dist_remote) { + char remote_host[1024]; + int remote_port = 0; + if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host), + &remote_port) != 0) { + fprintf(stderr, "--dist-remote expects HOST:PORT\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.dist = pplx_dist_client_open_with_options( + model_dir, remote_host, remote_port, layer_end, use_mlx, + &dist_opts); + if (!e.dist) { + fprintf(stderr, "failed to connect distributed worker\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.dim = pplx_dist_client_config(e.dist)->hidden_size; + } else if (use_mlx) { + pplx_mlx_options_t mlx_opts = { + .quantize_bits = mlx_quantize_bits, + .quantize_group_size = mlx_quantize_group_size, + }; + e.mlx_ctx = pplx_mlx_load_with_options(model_dir, &mlx_opts); + if (!e.mlx_ctx) { + fprintf(stderr, "failed to load model\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size; + } else +#endif +#ifdef USE_CUDA + if (use_cuda) { + e.cuda_ctx = pplx_cuda_load(model_dir); + if (!e.cuda_ctx) { + fprintf(stderr, "failed to load CUDA model\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.dim = pplx_cuda_config(e.cuda_ctx)->hidden_size; + } else +#endif + { + if (dist_remote) { + char remote_host[1024]; + int remote_port = 0; + if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host), + &remote_port) != 0) { + fprintf(stderr, "--dist-remote expects HOST:PORT\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.dist = pplx_dist_client_open_with_options( + model_dir, remote_host, remote_port, layer_end, use_mlx, + &dist_opts); + if (!e.dist) { + fprintf(stderr, "failed to connect distributed worker\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.dim = pplx_dist_client_config(e.dist)->hidden_size; + } else { + e.model = pplx_model_load(model_dir); + if (!e.model) { + fprintf(stderr, "failed to load model\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.workspace = pplx_workspace_new(e.model); + if (!e.workspace) { + fprintf(stderr, "failed to allocate workspace\n"); + pplx_model_free(e.model); + qwen_tokenizer_free(tok); + return 1; + } + e.dim = pplx_model_config(e.model)->hidden_size; + } + } + if (verbose >= 1) + fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n", + e.dim, now_ms() - t0, + use_mlx ? " (MLX)" : "", + use_cuda ? " (CUDA)" : ""); + + e.tok_ws = qwen_tokenizer_workspace_new(); + if (!e.tok_ws) { + fprintf(stderr, "failed to allocate tokenizer workspace\n"); + if (e.workspace) pplx_workspace_free(e.workspace); + if (e.model) pplx_model_free(e.model); + if (e.dist) pplx_dist_client_free(e.dist); +#ifdef USE_MLX + if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); +#endif +#ifdef USE_CUDA + if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); +#endif + qwen_tokenizer_free(tok); + return 1; + } + + /* Run */ + int rc; + if (stdin_mode) + rc = run_stdin(&e, batch_size > 0 ? batch_size : 1); + else + rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size); + + /* Cleanup */ + if (e.workspace) pplx_workspace_free(e.workspace); + if (e.model) pplx_model_free(e.model); + if (e.dist) pplx_dist_client_free(e.dist); +#ifdef USE_MLX + if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); +#endif +#ifdef USE_CUDA + if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); +#endif + qwen_tokenizer_workspace_free(e.tok_ws); + qwen_tokenizer_free(tok); + return rc; +} diff --git a/embed_cuda.cu b/embed_cuda.cu new file mode 100644 index 0000000..a00fc84 --- /dev/null +++ b/embed_cuda.cu @@ -0,0 +1,1204 @@ +extern "C" { +#include "pplx_embed_cuda.h" +#include "pplx_embed_internal.h" +} + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +typedef struct { + float *d; // device buffer; holds __nv_bfloat16 elements when bf16 + int rows; + int cols; + int bf16; // 1 if d holds __nv_bfloat16, 0 if F32 +} cuda_matrix_t; + +typedef struct { + cuda_matrix_t qkv, wo; + float *q_norm; + float *k_norm; + float *input_norm; + float *post_attn_norm; + cuda_matrix_t gate_up_proj, down_proj; +} cuda_layer_t; + +struct pplx_cuda_ctx { + pplx_model_t *cpu; + pplx_config_t config; + cudaStream_t stream; + cublasHandle_t blas; + cuda_matrix_t embed_tokens; + cuda_layer_t *layers; + float *norm; + + int seq_cap; + int batch_cap; + int max_seq_cap; + float *x; + float *x_norm; + float *qkv; + float *attn_out; + float *ffn_gate_up; + void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights) + float *pooled_out; + float *rope_cos; + float *rope_sin; + int *token_ids; + int *offsets; + int *offsets_host; + int *positions; + float *kexp; + float *vexp; + float *attn_scores; + int *span_starts; + int *span_lens; + int pooled_rows_cap; + int span_cap; +}; + +#define CUDA_CHECK(expr) do { \ + cudaError_t _e = (expr); \ + if (_e != cudaSuccess) { \ + fprintf(stderr, "cuda: %s failed: %s\n", #expr, \ + cudaGetErrorString(_e)); \ + return -1; \ + } \ +} while (0) + +#define CUBLAS_CHECK(expr) do { \ + cublasStatus_t _s = (expr); \ + if (_s != CUBLAS_STATUS_SUCCESS) { \ + fprintf(stderr, "cuda: %s failed: cublas status %d\n", \ + #expr, (int)_s); \ + return -1; \ + } \ +} while (0) + +static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) +{ + if (!mode || !mode[0] || !strcmp(mode, "f32")) { + *out = CUBLAS_COMPUTE_32F; + return 0; + } + if (!strcmp(mode, "tf32")) { + *out = CUBLAS_COMPUTE_32F_FAST_TF32; + return 0; + } + if (!strcmp(mode, "bf16")) { + *out = CUBLAS_COMPUTE_32F_FAST_16BF; + return 0; + } + if (!strcmp(mode, "16f")) { + *out = CUBLAS_COMPUTE_32F_FAST_16F; + return 0; + } + return -1; +} + +static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F; +static int g_gemm_compute_set = 0; + +int pplx_cuda_set_fast_gemm(const char *mode) +{ + cublasComputeType_t ct; + if (parse_gemm_compute(mode, &ct) != 0) + return -1; + g_gemm_compute = ct; + g_gemm_compute_set = 1; + return 0; +} + +static void cuda_matrix_free(cuda_matrix_t *m) +{ + if (!m) return; + cudaFree(m->d); + memset(m, 0, sizeof(*m)); +} + +static float bf16_to_f32(uint16_t v) +{ + uint32_t u = ((uint32_t)v) << 16; + float f; + memcpy(&f, &u, sizeof(f)); + return f; +} + +// Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are +// preserved. BF16 shares F32's 8-bit exponent, so this never overflows. +static uint16_t f32_to_bf16(float f) +{ + uint32_t x; + memcpy(&x, &f, sizeof(x)); + if ((x & 0x7fffffffu) > 0x7f800000u) // NaN + return (uint16_t)((x >> 16) | 0x0040u); + x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even + return (uint16_t)(x >> 16); +} + +// Store weights as native BF16 on the device when set (halves weight memory and +// feeds BF16 tensor cores); default keeps exact F32. Read at load time. +static int g_weights_bf16 = 0; + +int pplx_cuda_set_weights_bf16(int on) +{ + g_weights_bf16 = on ? 1 : 0; + return 0; +} + +static int copy_weight_host_f32(float *dst, const pplx_weight_ref_t *w, + size_t count) +{ + if (!dst || !w || !w->data) return -1; + if (w->dtype == DTYPE_F32) { + memcpy(dst, w->data, count * sizeof(float)); + return 0; + } + if (w->dtype == DTYPE_BF16) { + const uint16_t *src = (const uint16_t *)w->data; + for (size_t i = 0; i < count; i++) + dst[i] = bf16_to_f32(src[i]); + return 0; + } + return -1; +} + +// Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is +// set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16. +static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, + int rows, int cols) +{ + cudaError_t e; + if (g_weights_bf16) { + uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t)); + if (!tmp16) return -1; + for (size_t i = 0; i < count; i++) + tmp16[i] = f32_to_bf16(src[i]); + e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e)); + free(tmp16); + return -1; + } + e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), + cudaMemcpyHostToDevice); + free(tmp16); + m->bf16 = 1; + } else { + e = cudaMalloc((void **)&m->d, count * sizeof(float)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e)); + return -1; + } + e = cudaMemcpy(m->d, src, count * sizeof(float), cudaMemcpyHostToDevice); + m->bf16 = 0; + } + if (e != cudaSuccess) { + fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e)); + cuda_matrix_free(m); + return -1; + } + m->rows = rows; + m->cols = cols; + return 0; +} + +static int load_matrix(cuda_matrix_t *m, const pplx_weight_ref_t *w, + int rows, int cols) +{ + if (!m || !w || rows <= 0 || cols <= 0) return -1; + size_t count = (size_t)rows * (size_t)cols; + float *tmp = (float *)malloc(count * sizeof(float)); + if (!tmp) return -1; + if (copy_weight_host_f32(tmp, w, count) != 0) { + free(tmp); + return -1; + } + int r = upload_weight(m, tmp, count, rows, cols); + free(tmp); + return r; +} + +static int load_qkv_matrix(cuda_matrix_t *m, const pplx_layer_t *src, + const pplx_config_t *c) +{ + if (!m || !src || !c) return -1; + int rows = c->q_dim + 2 * c->kv_dim; + int cols = c->hidden_size; + size_t q_count = (size_t)c->q_dim * cols; + size_t kv_count = (size_t)c->kv_dim * cols; + size_t total = q_count + 2 * kv_count; + float *tmp = (float *)malloc(total * sizeof(float)); + if (!tmp) return -1; + if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 || + copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 || + copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) { + free(tmp); + return -1; + } + int r = upload_weight(m, tmp, total, rows, cols); + free(tmp); + return r; +} + +static int load_gate_up_matrix(cuda_matrix_t *m, const pplx_layer_t *src, + const pplx_config_t *c) +{ + if (!m || !src || !c) return -1; + int rows = 2 * c->intermediate_size; + int cols = c->hidden_size; + size_t proj_count = (size_t)c->intermediate_size * cols; + size_t total = 2 * proj_count; + float *tmp = (float *)malloc(total * sizeof(float)); + if (!tmp) return -1; + if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 || + copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) { + free(tmp); + return -1; + } + int r = upload_weight(m, tmp, total, rows, cols); + free(tmp); + return r; +} + +static int load_vector(float **out, const float *src, int n) +{ + if (!out || !src || n <= 0) return -1; + cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float)); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e)); + return -1; + } + e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice); + if (e != cudaSuccess) { + fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e)); + cudaFree(*out); + *out = NULL; + return -1; + } + return 0; +} + +__global__ static void embed_lookup_kernel(float *x, const int *ids, + const void *emb, int emb_bf16, + int total, int hidden, int vocab) +{ + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int count = total * hidden; + if (idx >= count) return; + int tok = idx / hidden; + int d = idx - tok * hidden; + int id = ids[tok]; + if (id < 0 || id >= vocab) { + x[idx] = 0.0f; + return; + } + size_t off = (size_t)id * hidden + d; + x[idx] = emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) + : ((const float *)emb)[off]; +} + +// Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs). +__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, + const float *in, size_t n) +{ + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) out[i] = __float2bfloat16(in[i]); +} + +__device__ static float block_sum(float v) +{ + __shared__ float partial[8]; + __shared__ float total; + int lane = threadIdx.x & 31; + int warp = threadIdx.x >> 5; + int warps = blockDim.x >> 5; + + for (int offset = 16; offset > 0; offset >>= 1) + v += __shfl_down_sync(0xffffffff, v, offset); + if (lane == 0) partial[warp] = v; + __syncthreads(); + + if (warp == 0) { + v = lane < warps ? partial[lane] : 0.0f; + for (int offset = 16; offset > 0; offset >>= 1) + v += __shfl_down_sync(0xffffffff, v, offset); + if (lane == 0) total = v; + } + __syncthreads(); + return total; +} + +__device__ static inline void store_act(float *out, size_t i, float v) +{ + out[i] = v; +} + +__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) +{ + out[i] = __float2bfloat16(v); +} + +// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight +// projection GEMM directly (same rounding as a separate cast kernel). +template +__global__ static void rms_norm_kernel(OUT_T *out, const float *x, + const float *weight, int rows, + int dim, float eps) +{ + int row = blockIdx.x; + int tid = threadIdx.x; + float sum = 0.0f; + for (int d = tid; d < dim; d += blockDim.x) { + float v = x[(size_t)row * dim + d]; + sum += v * v; + } + float total = block_sum(sum); + float inv = rsqrtf(total / (float)dim + eps); + for (int d = tid; d < dim; d += blockDim.x) + store_act(out, (size_t)row * dim + d, + x[(size_t)row * dim + d] * inv * weight[d]); +} + +__global__ static void rms_norm_rope_head_kernel(float *x, const float *weight, + const int *positions, + const float *cosv, + const float *sinv, + int rows, int heads, + int head_dim, + int row_stride, + int base_offset, + float eps) +{ + __shared__ float partial[4]; + __shared__ float total; + int row = blockIdx.x; + int head = blockIdx.y; + int tid = threadIdx.x; + int lane = tid & 31; + int warp = tid >> 5; + int half = head_dim / 2; + int pair_d = tid < half ? tid + half : tid - half; + float sign = tid < half ? -1.0f : 1.0f; + size_t base = (size_t)row * row_stride + base_offset + head * head_dim; + float v = tid < head_dim ? x[base + tid] : 0.0f; + float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f; + float sum = v * v; + + for (int offset = 16; offset > 0; offset >>= 1) + sum += __shfl_down_sync(0xffffffff, sum, offset); + if (lane == 0) partial[warp] = sum; + __syncthreads(); + if (warp == 0) { + sum = lane < 4 ? partial[lane] : 0.0f; + for (int offset = 16; offset > 0; offset >>= 1) + sum += __shfl_down_sync(0xffffffff, sum, offset); + if (lane == 0) total = sum; + } + __syncthreads(); + + if (tid < head_dim) { + float inv = rsqrtf(total / (float)head_dim + eps); + int pos = positions[row]; + float c = cosv[(size_t)pos * head_dim + tid]; + float s = sinv[(size_t)pos * head_dim + tid]; + float a = v * inv * weight[tid]; + float b = pair_v * inv * weight[pair_d]; + x[base + tid] = a * c + sign * b * s; + } +} + +// Writes SiLU(gate)*up to out with out_stride. The F32 path writes in place +// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16 +// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM. +template +__global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride, + const float *gate_up, int rows, + int intermediate) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + int n = rows * intermediate; + if (i < n) { + int row = i / intermediate; + int d = i - row * intermediate; + size_t base = (size_t)row * (2 * intermediate); + float g = gate_up[base + d]; + float u = gate_up[base + intermediate + d]; + store_act(out, (size_t)row * out_stride + d, + (g / (1.0f + expf(-g))) * u); + } +} + +// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim] +// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times) +// so the attention GEMMs can stride uniformly over query heads. +__global__ static void attn_expand_kv_kernel( + float *kexp, float *vexp, const float *qkv, int total, int n_heads, + int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset) +{ + int q_dim = n_heads * head_dim; + size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + size_t count = (size_t)total * q_dim; + if (idx >= count) return; + int t = (int)(idx / q_dim); + int rem = (int)(idx - (size_t)t * q_dim); + int h = rem / head_dim; + int e = rem - h * head_dim; + int kv = h / (n_heads / n_kv_heads); + const float *base = qkv + (size_t)t * qkv_dim; + kexp[idx] = base[k_offset + kv * head_dim + e]; + vexp[idx] = base[v_offset + kv * head_dim + e]; +} + +// Row softmax over the key dimension for one sequence's score tensor laid out +// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i). +// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM. +__global__ static void attn_softmax_kernel(float *scores, int L) +{ + float *s = scores + (size_t)blockIdx.x * L; + int tid = threadIdx.x; + __shared__ float red[256]; + + float m = -3.402823466e+38F; + for (int j = tid; j < L; j += blockDim.x) m = fmaxf(m, s[j]); + red[tid] = m; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]); + __syncthreads(); + } + m = red[0]; + __syncthreads(); + + float sum = 0.0f; + for (int j = tid; j < L; j += blockDim.x) { + float e = __expf(s[j] - m); + s[j] = e; + sum += e; + } + red[tid] = sum; + __syncthreads(); + for (int o = blockDim.x / 2; o > 0; o >>= 1) { + if (tid < o) red[tid] += red[tid + o]; + __syncthreads(); + } + float inv = 1.0f / red[0]; + for (int j = tid; j < L; j += blockDim.x) s[j] *= inv; +} + +__global__ static void mean_pool_kernel(float *out, const float *x, + const int *offsets, int batch, + int hidden) +{ + int b = blockIdx.x; + int tid = threadIdx.x; + int start = offsets[b]; + int end = offsets[b + 1]; + int len = end - start; + for (int d = tid; d < hidden; d += blockDim.x) { + float sum = 0.0f; + for (int t = start; t < end; t++) + sum += x[(size_t)t * hidden + d]; + out[(size_t)b * hidden + d] = sum / (float)len; + } +} + +__global__ static void span_pool_kernel(float *out, const float *x, + const int *starts, const int *lens, + int n_spans, int hidden) +{ + int s = blockIdx.x; + int tid = threadIdx.x; + int start = starts[s]; + int len = lens[s]; + for (int d = tid; d < hidden; d += blockDim.x) { + float sum = 0.0f; + for (int t = 0; t < len; t++) + sum += x[(size_t)(start + t) * hidden + d]; + out[(size_t)s * hidden + d] = sum / (float)len; + } +} + +static int launch_check(void) +{ + cudaError_t e = cudaGetLastError(); + if (e != cudaSuccess) { + fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e)); + return -1; + } + return 0; +} + +// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm). +// PPLX_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core +// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands +// internally. +static cublasComputeType_t gemm_compute(void) +{ + static int env_init = 0; + if (!g_gemm_compute_set && !env_init) { + const char *e = getenv("PPLX_CUDA_GEMM_MODE"); + if (e && parse_gemm_compute(e, &g_gemm_compute) == 0) + g_gemm_compute_set = 1; + env_init = 1; + } + return g_gemm_compute; +} + +// y = w @ x (+ beta*y). With BF16 weights both GEMM inputs are BF16 and +// accumulation/output stay F32: when x_is_bf16 the activation operand was +// already produced in BF16 by the upstream kernel, otherwise the F32 operand +// is cast into ctx->act_bf16 first. With F32 weights this is the exact path, +// honoring gemm_compute(). +static int linear_ex(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, + const void *x, int x_is_bf16, float *y, int rows, + int in_dim, int out_dim, int x_stride, float beta) +{ + const float alpha = 1.0f; + if (w->bf16) { + const void *xb = x; + if (!x_is_bf16) { + size_t n = (size_t)rows * (size_t)x_stride; + int threads = 256; + cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, + ctx->stream>>>( + (__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n); + if (launch_check() != 0) return -1; + xb = ctx->act_bf16; + } + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, + out_dim, rows, in_dim, + &alpha, w->d, CUDA_R_16BF, in_dim, + xb, CUDA_R_16BF, x_stride, + &beta, y, CUDA_R_32F, out_dim, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + return 0; + } + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, + out_dim, rows, in_dim, + &alpha, w->d, CUDA_R_32F, in_dim, + x, CUDA_R_32F, x_stride, + &beta, y, CUDA_R_32F, out_dim, + gemm_compute(), CUBLAS_GEMM_DEFAULT)); + return 0; +} + +static int linear(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, + const float *x, float *y, int rows, int in_dim, int out_dim) +{ + return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f); +} + +// x points at activations already stored as BF16 (e.g. ctx->act_bf16). +static int linear_bf16x(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, + const void *x, float *y, int rows, int in_dim, + int out_dim, int x_stride, float beta) +{ + return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta); +} + +static int linear_accum(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, + const float *x, float *y, + int rows, int in_dim, int out_dim) +{ + return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f); +} + +static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa, + cublasOperation_t transb, int m, int n, int k, + const float *alpha, const float *A, int lda, + long long strideA, const float *B, int ldb, + long long strideB, const float *beta, + float *C, int ldc, long long strideC, + int batch_count) +{ + cublasComputeType_t ct = gemm_compute(); + if (ct == CUBLAS_COMPUTE_32F) { + CUBLAS_CHECK(cublasSgemmStridedBatched( + blas, transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb, + strideB, beta, C, ldc, strideC, batch_count)); + } else { + CUBLAS_CHECK(cublasGemmStridedBatchedEx( + blas, transa, transb, m, n, k, + alpha, A, CUDA_R_32F, lda, strideA, + B, CUDA_R_32F, ldb, strideB, + beta, C, CUDA_R_32F, ldc, strideC, + batch_count, ct, CUBLAS_GEMM_DEFAULT)); + } + return 0; +} + +// Attention for one micro-batch. Expands K/V to the per-query-head layout, +// then for each sequence runs batched-over-heads Q@K^T (scaled) -> row softmax +// -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and +// ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets +// is the host copy of the packed sequence boundaries. +static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets, + int batch, int q_offset, int k_offset, + int v_offset, int qkv_dim, float scale) +{ + const pplx_config_t *c = &ctx->config; + int q_dim = c->q_dim; + int hd = c->head_dim; + int H = c->n_heads; + int total = h_offsets[batch]; + const float alpha = scale, beta = 0.0f, one = 1.0f; + + int threads = 256; + size_t exp_count = (size_t)total * q_dim; + attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, + 0, ctx->stream>>>( + ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, + k_offset, v_offset); + if (launch_check() != 0) return -1; + + for (int b = 0; b < batch; b++) { + int start = h_offsets[b]; + int L = h_offsets[b + 1] - start; + const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset; + const float *K = ctx->kexp + (size_t)start * q_dim; + const float *V = ctx->vexp + (size_t)start * q_dim; + float *O = ctx->attn_out + (size_t)start * q_dim; + /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */ + if (gemm_strided_batched( + ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, + &alpha, K, q_dim, hd, Q, qkv_dim, hd, + &beta, ctx->attn_scores, L, (long long)L * L, H) != 0) + return -1; + attn_softmax_kernel<<stream>>>( + ctx->attn_scores, L); + if (launch_check() != 0) return -1; + /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ + if (gemm_strided_batched( + ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, + &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L, + &beta, O, q_dim, hd, H) != 0) + return -1; + } + return 0; +} + +static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_seq) +{ + const pplx_config_t *c = &ctx->config; + if (total > ctx->seq_cap) { + cudaFree(ctx->x); + cudaFree(ctx->x_norm); + cudaFree(ctx->qkv); + cudaFree(ctx->attn_out); + cudaFree(ctx->ffn_gate_up); + cudaFree(ctx->token_ids); + cudaFree(ctx->positions); + cudaFree(ctx->kexp); + cudaFree(ctx->vexp); + cudaFree(ctx->act_bf16); + ctx->x = ctx->x_norm = ctx->qkv = NULL; + ctx->attn_out = ctx->ffn_gate_up = NULL; + ctx->token_ids = ctx->positions = NULL; + ctx->kexp = ctx->vexp = NULL; + ctx->act_bf16 = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up, + (size_t)total * 2 * c->intermediate_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float))); + if (g_weights_bf16) + CUDA_CHECK(cudaMalloc(&ctx->act_bf16, + (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16))); + ctx->seq_cap = total; + } + if (batch + 1 > ctx->batch_cap) { + cudaFree(ctx->offsets); + free(ctx->offsets_host); + CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int))); + ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int)); + if (!ctx->offsets_host) return -1; + ctx->batch_cap = batch + 1; + } + if (max_seq > ctx->max_seq_cap) { + cudaFree(ctx->rope_cos); + cudaFree(ctx->rope_sin); + cudaFree(ctx->attn_scores); + ctx->attn_scores = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, + (size_t)c->n_heads * max_seq * max_seq * sizeof(float))); + float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); + float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); + if (!hcos || !hsin) { + free(hcos); + free(hsin); + return -1; + } + int half = c->head_dim / 2; + for (int pos = 0; pos < max_seq; pos++) { + for (int d = 0; d < half; d++) { + float inv = 1.0f / powf(c->rope_theta, + (float)(2 * d) / (float)c->head_dim); + float angle = (float)pos * inv; + float cv = cosf(angle); + float sv = sinf(angle); + hcos[(size_t)pos * c->head_dim + d] = cv; + hcos[(size_t)pos * c->head_dim + half + d] = cv; + hsin[(size_t)pos * c->head_dim + d] = sv; + hsin[(size_t)pos * c->head_dim + half + d] = sv; + } + } + CUDA_CHECK(cudaMalloc((void **)&ctx->rope_cos, + (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->rope_sin, + (size_t)max_seq * c->head_dim * sizeof(float))); + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, + (size_t)max_seq * c->head_dim * sizeof(float), + cudaMemcpyHostToDevice, ctx->stream)); + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin, + (size_t)max_seq * c->head_dim * sizeof(float), + cudaMemcpyHostToDevice, ctx->stream)); + free(hcos); + free(hsin); + ctx->max_seq_cap = max_seq; + } + return 0; +} + +static int ensure_pooled_rows(pplx_cuda_ctx_t *ctx, int rows) +{ + if (rows <= ctx->pooled_rows_cap) return 0; + const pplx_config_t *c = &ctx->config; + cudaFree(ctx->pooled_out); + ctx->pooled_out = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out, + (size_t)rows * c->hidden_size * sizeof(float))); + ctx->pooled_rows_cap = rows; + return 0; +} + +static int ensure_span_buffers(pplx_cuda_ctx_t *ctx, int n_spans) +{ + if (n_spans <= ctx->span_cap) return 0; + cudaFree(ctx->span_starts); + cudaFree(ctx->span_lens); + ctx->span_starts = NULL; + ctx->span_lens = NULL; + CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, + (size_t)n_spans * sizeof(int))); + CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, + (size_t)n_spans * sizeof(int))); + ctx->span_cap = n_spans; + return 0; +} + +static int load_layer(cuda_layer_t *dst, const pplx_layer_t *src, + const pplx_config_t *c) +{ + return + load_qkv_matrix(&dst->qkv, src, c) || + load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || + load_vector(&dst->q_norm, src->q_norm, c->head_dim) || + load_vector(&dst->k_norm, src->k_norm, c->head_dim) || + load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || + load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || + load_gate_up_matrix(&dst->gate_up_proj, src, c) || + load_matrix(&dst->down_proj, &src->down_proj, + c->hidden_size, c->intermediate_size); +} + +static void free_layer(cuda_layer_t *l) +{ + cuda_matrix_free(&l->qkv); + cuda_matrix_free(&l->wo); + cudaFree(l->q_norm); + cudaFree(l->k_norm); + cudaFree(l->input_norm); + cudaFree(l->post_attn_norm); + cuda_matrix_free(&l->gate_up_proj); + cuda_matrix_free(&l->down_proj); +} + +pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir) +{ + pplx_model_t *cpu = pplx_model_load(model_dir); + if (!cpu) return NULL; + + pplx_cuda_ctx_t *ctx = (pplx_cuda_ctx_t *)calloc(1, sizeof(*ctx)); + if (!ctx) { + pplx_model_free(cpu); + return NULL; + } + ctx->cpu = cpu; + ctx->config = cpu->config; + + if (cudaStreamCreate(&ctx->stream) != cudaSuccess || + cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS || + cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) { + pplx_cuda_free(ctx); + return NULL; + } + + const pplx_config_t *c = &ctx->config; + ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers)); + if (!ctx->layers) { + pplx_cuda_free(ctx); + return NULL; + } + if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, + c->vocab_size, c->hidden_size) != 0 || + load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) { + pplx_cuda_free(ctx); + return NULL; + } + for (int i = 0; i < c->n_layers; i++) { + if (load_layer(&ctx->layers[i], &cpu->weights.layers[i], c) != 0) { + fprintf(stderr, "cuda: failed to load layer %d\n", i); + pplx_cuda_free(ctx); + return NULL; + } + } + cudaStreamSynchronize(ctx->stream); + return ctx; +} + +void pplx_cuda_free(pplx_cuda_ctx_t *ctx) +{ + if (!ctx) return; + cuda_matrix_free(&ctx->embed_tokens); + if (ctx->layers) { + for (int i = 0; i < ctx->config.n_layers; i++) + free_layer(&ctx->layers[i]); + free(ctx->layers); + } + cudaFree(ctx->norm); + cudaFree(ctx->x); + cudaFree(ctx->x_norm); + cudaFree(ctx->qkv); + cudaFree(ctx->attn_out); + cudaFree(ctx->ffn_gate_up); + cudaFree(ctx->pooled_out); + cudaFree(ctx->rope_cos); + cudaFree(ctx->rope_sin); + cudaFree(ctx->token_ids); + cudaFree(ctx->offsets); + free(ctx->offsets_host); + cudaFree(ctx->positions); + cudaFree(ctx->kexp); + cudaFree(ctx->vexp); + cudaFree(ctx->act_bf16); + cudaFree(ctx->attn_scores); + cudaFree(ctx->span_starts); + cudaFree(ctx->span_lens); + if (ctx->blas) cublasDestroy(ctx->blas); + if (ctx->stream) cudaStreamDestroy(ctx->stream); + pplx_model_free(ctx->cpu); + free(ctx); +} + +const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx) +{ + return ctx ? &ctx->config : NULL; +} + +static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, + int batch) +{ + if (!ctx || !inputs || batch <= 0) return -1; + const pplx_config_t *c = &ctx->config; + int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); + if (!h_offsets) return -1; + int total = 0, max_seq = 0; + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) { + free(h_offsets); + return -1; + } + h_offsets[b] = total; + total += inputs[b].n_tokens; + if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + } + h_offsets[batch] = total; + int *h_ids = (int *)malloc((size_t)total * sizeof(int)); + int *h_pos = (int *)malloc((size_t)total * sizeof(int)); + if (!h_offsets || !h_ids || !h_pos) { + free(h_offsets); free(h_ids); free(h_pos); + return -1; + } + + for (int b = 0; b < batch; b++) { + int off = h_offsets[b]; + memcpy(h_ids + off, inputs[b].ids, + (size_t)inputs[b].n_tokens * sizeof(int)); + for (int i = 0; i < inputs[b].n_tokens; i++) + h_pos[off + i] = i; + } + if (ensure_buffers(ctx, total, batch, max_seq) != 0) { + free(h_offsets); free(h_ids); free(h_pos); + return -1; + } + memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int)); + cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, + (size_t)total * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->positions, h_pos, + (size_t)total * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->offsets, h_offsets, + (size_t)(batch + 1) * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + free(h_offsets); free(h_ids); free(h_pos); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce)); + return -1; + } + + int threads = 256; + int blocks_hidden = (total * c->hidden_size + threads - 1) / threads; + embed_lookup_kernel<<stream>>>( + ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, + total, c->hidden_size, c->vocab_size); + if (launch_check() != 0) return -1; + + float scale = 1.0f / sqrtf((float)c->head_dim); + int q_offset = 0; + int k_offset = c->q_dim; + int v_offset = c->q_dim + c->kv_dim; + int qkv_dim = c->q_dim + 2 * c->kv_dim; + // With BF16 weights the norm and SiLU kernels store their results as BF16 + // directly into ctx->act_bf16, which the next projection GEMM consumes + // before any later producer reuses the buffer (single stream). This + // removes three cast launches per layer; only the attention output still + // needs a cast because cuBLAS cannot emit a BF16 C from F32 A/B operands. + __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16; + for (int layer = 0; layer < c->n_layers; layer++) { + cuda_layer_t *l = &ctx->layers[layer]; + int wbf16 = l->qkv.bf16; + if (wbf16) { + rms_norm_kernel<<stream>>>( + act, ctx->x, l->input_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, + total, c->hidden_size, qkv_dim, + c->hidden_size, 0.0f) != 0) + return -1; + } else { + rms_norm_kernel<<stream>>>( + ctx->x_norm, ctx->x, l->input_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, + total, c->hidden_size, qkv_dim) != 0) + return -1; + } + + rms_norm_rope_head_kernel<<n_heads), 128, 0, ctx->stream>>>( + ctx->qkv, l->q_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, + total, c->n_heads, c->head_dim, qkv_dim, q_offset, + c->rms_norm_eps); + rms_norm_rope_head_kernel<<n_kv_heads), 128, 0, ctx->stream>>>( + ctx->qkv, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, + total, c->n_kv_heads, c->head_dim, qkv_dim, k_offset, + c->rms_norm_eps); + if (launch_check() != 0) return -1; + + if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, + k_offset, v_offset, qkv_dim, scale) != 0) + return -1; + + if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, + total, c->q_dim, c->hidden_size) != 0) + return -1; + + int inter_count = total * c->intermediate_size; + if (wbf16) { + rms_norm_kernel<<stream>>>( + act, ctx->x, l->post_attn_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, + total, c->hidden_size, 2 * c->intermediate_size, + c->hidden_size, 0.0f) != 0) + return -1; + silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, + threads, 0, ctx->stream>>>( + act, c->intermediate_size, ctx->ffn_gate_up, total, + c->intermediate_size); + if (launch_check() != 0) return -1; + if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, + total, c->intermediate_size, c->hidden_size, + c->intermediate_size, 1.0f) != 0) + return -1; + } else { + rms_norm_kernel<<stream>>>( + ctx->x_norm, ctx->x, l->post_attn_norm, total, + c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + if (linear(ctx, &l->gate_up_proj, ctx->x_norm, + ctx->ffn_gate_up, total, c->hidden_size, + 2 * c->intermediate_size) != 0) + return -1; + silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, + threads, 0, ctx->stream>>>( + ctx->ffn_gate_up, 2 * c->intermediate_size, ctx->ffn_gate_up, + total, c->intermediate_size); + if (launch_check() != 0) return -1; + if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, + total, c->intermediate_size, c->hidden_size, + 2 * c->intermediate_size, 1.0f) != 0) + return -1; + } + } + + rms_norm_kernel<<stream>>>( + ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps); + if (launch_check() != 0) return -1; + + return 0; +} + +int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + const pplx_config_t *c = &ctx->config; + if (cuda_forward_batch(ctx, inputs, batch) != 0) + return -1; + if (ensure_pooled_rows(ctx, batch) != 0) + return -1; + mean_pool_kernel<<stream>>>( + ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); + if (launch_check() != 0) return -1; + cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, + (size_t)batch * c->hidden_size * sizeof(float), + cudaMemcpyDeviceToHost, ctx->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce)); + return -1; + } + return 0; +} + +int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx, + const pplx_context_input_t *inputs, + int batch, float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + const pplx_config_t *c = &ctx->config; + int total_spans = 0; + int total_tokens = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || + input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans || + total_tokens > INT_MAX - n_tokens) + return -1; + for (int s = 0; s < input->n_spans; s++) { + int start = input->spans[s].start; + int len = input->spans[s].n_tokens; + if (start < 0 || len <= 0 || start > n_tokens || + len > n_tokens - start) + return -1; + } + total_spans += input->n_spans; + total_tokens += n_tokens; + } + + pplx_input_t *packed = (pplx_input_t *)malloc((size_t)batch * sizeof(*packed)); + int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int)); + int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int)); + if (!packed || !h_starts || !h_lens) { + free(packed); + free(h_starts); + free(h_lens); + return -1; + } + + int token_offset = 0; + int span_offset = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + packed[b] = input->input; + for (int s = 0; s < input->n_spans; s++) { + h_starts[span_offset] = token_offset + input->spans[s].start; + h_lens[span_offset] = input->spans[s].n_tokens; + span_offset++; + } + token_offset += input->input.n_tokens; + } + + int rc = -1; + cudaError_t ce; + if (cuda_forward_batch(ctx, packed, batch) != 0) + goto cleanup; + if (ensure_pooled_rows(ctx, total_spans) != 0 || + ensure_span_buffers(ctx, total_spans) != 0) + goto cleanup; + ce = cudaMemcpyAsync(ctx->span_starts, h_starts, + (size_t)total_spans * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce == cudaSuccess) + ce = cudaMemcpyAsync(ctx->span_lens, h_lens, + (size_t)total_spans * sizeof(int), + cudaMemcpyHostToDevice, ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce)); + goto cleanup; + } + span_pool_kernel<<stream>>>( + ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, + total_spans, c->hidden_size); + if (launch_check() != 0) + goto cleanup; + ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, + (size_t)total_spans * c->hidden_size * sizeof(float), + cudaMemcpyDeviceToHost, ctx->stream); + if (ce == cudaSuccess) + ce = cudaStreamSynchronize(ctx->stream); + if (ce != cudaSuccess) { + fprintf(stderr, "cuda span output copy failed: %s\n", + cudaGetErrorString(ce)); + goto cleanup; + } + rc = 0; + +cleanup: + free(h_lens); + free(h_starts); + free(packed); + return rc; +} + +int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding) +{ + pplx_input_t input = { token_ids, n_tokens }; + return pplx_cuda_embed_batch(ctx, &input, 1, out_embedding); +} + +float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens) +{ + if (!ctx) return NULL; + int hidden = ctx->config.hidden_size; + float *out = (float *)malloc((size_t)hidden * sizeof(float)); + if (!out) return NULL; + if (pplx_cuda_embed_into(ctx, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; + } + return out; +} diff --git a/embed_cuda.h b/embed_cuda.h new file mode 100644 index 0000000..badff9c --- /dev/null +++ b/embed_cuda.h @@ -0,0 +1,30 @@ +/* + * pplx_embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference. + */ + +#ifndef PPLX_EMBED_CUDA_H +#define PPLX_EMBED_CUDA_H + +#include "pplx_embed.h" + +typedef struct pplx_cuda_ctx pplx_cuda_ctx_t; + +int pplx_cuda_set_fast_gemm(const char *mode); +int pplx_cuda_set_weights_bf16(int on); + +pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir); +void pplx_cuda_free(pplx_cuda_ctx_t *ctx); + +const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx); + +int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding); +float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids, + int n_tokens); +int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings); +int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx, + const pplx_context_input_t *inputs, + int batch, float *out_embeddings); + +#endif /* PPLX_EMBED_CUDA_H */ diff --git a/embed_distributed.c b/embed_distributed.c new file mode 100644 index 0000000..d4254f7 --- /dev/null +++ b/embed_distributed.c @@ -0,0 +1,888 @@ +/* + * pplx_distributed.c - Stateless two-node layer-sharded inference. + * + * This first transport is deliberately synchronous: one coordinator sends a + * packed hidden-state batch to one final worker and receives pooled + * embeddings. The protocol is bounded and versioned so later route discovery, + * retries, and richer transport can extend it without changing model math. + */ + +#include "pplx_distributed.h" + +#ifdef USE_MLX +#include "pplx_embed_mlx.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PPLX_DIST_MAGIC UINT32_C(0x504c5844) /* "PLXD" */ +#define PPLX_DIST_VERSION 2u + +#define PPLX_DIST_MSG_HELLO 1u +#define PPLX_DIST_MSG_WORK 2u +#define PPLX_DIST_MSG_RESULT 3u + +#define PPLX_DIST_MAX_FRAME_BYTES (512u * 1024u * 1024u) +#define PPLX_DIST_MAX_BATCH 512u +#define PPLX_DIST_MAX_TOKENS 120000u +#define PPLX_DIST_MAX_ERROR_BYTES 1024u +#define PPLX_DIST_SOCKET_TIMEOUT_SEC 60 +#define PPLX_DIST_ACTIVATION_BITS_DEFAULT 32u + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t type; + uint32_t body_bytes; +} dist_frame_header; + +typedef struct { + uint32_t signature_hi; + uint32_t signature_lo; + uint32_t hidden_size; + uint32_t n_layers; + uint32_t vocab_size; + uint32_t n_heads; + uint32_t n_kv_heads; + uint32_t head_dim; + uint32_t intermediate_size; + uint32_t layer_start; + uint32_t layer_end; +} dist_hello; + +typedef struct { + uint32_t request_hi; + uint32_t request_lo; + uint32_t batch; + uint32_t total_tokens; + uint32_t hidden_size; + uint32_t lengths_bytes; + uint32_t states_bytes; + uint32_t states_bits; +} dist_work; + +typedef struct { + uint32_t request_hi; + uint32_t request_lo; + uint32_t status; + uint32_t batch; + uint32_t dimensions; + uint32_t payload_bytes; +} dist_result; + +typedef struct { + pplx_model_t *model; + pplx_workspace_t *workspace; +#ifdef USE_MLX + pplx_mlx_ctx_t *mlx; +#endif + const pplx_config_t *cfg; +} dist_stage; + +struct pplx_dist_client { + dist_stage local; + int fd; + int local_layer_end; + uint64_t next_request_id; + uint32_t activation_bits; +}; + +static int host_is_little_endian(void) +{ + const uint16_t one = 1; + return *(const unsigned char *)&one == 1; +} + +static uint64_t fnv_u32(uint64_t h, uint32_t v) +{ + for (int i = 0; i < 4; i++) { + h ^= (unsigned char)(v >> (i * 8)); + h *= UINT64_C(1099511628211); + } + return h; +} + +static uint64_t config_signature(const pplx_config_t *cfg) +{ + uint64_t h = UINT64_C(1469598103934665603); + uint32_t eps = 0, theta = 0; + memcpy(&eps, &cfg->rms_norm_eps, sizeof(eps)); + memcpy(&theta, &cfg->rope_theta, sizeof(theta)); + h = fnv_u32(h, (uint32_t)cfg->hidden_size); + h = fnv_u32(h, (uint32_t)cfg->n_layers); + h = fnv_u32(h, (uint32_t)cfg->n_heads); + h = fnv_u32(h, (uint32_t)cfg->n_kv_heads); + h = fnv_u32(h, (uint32_t)cfg->head_dim); + h = fnv_u32(h, (uint32_t)cfg->intermediate_size); + h = fnv_u32(h, (uint32_t)cfg->vocab_size); + h = fnv_u32(h, eps); + h = fnv_u32(h, theta); + return h; +} + +static void u64_to_halves(uint64_t v, uint32_t *hi, uint32_t *lo) +{ + *hi = (uint32_t)(v >> 32); + *lo = (uint32_t)v; +} + +static uint64_t u64_from_halves(uint32_t hi, uint32_t lo) +{ + return ((uint64_t)hi << 32) | lo; +} + +static int mul_u32(uint32_t a, uint32_t b, uint32_t *out) +{ + if (a != 0 && b > UINT32_MAX / a) return -1; + *out = a * b; + return 0; +} + +static uint32_t activation_bits_or_default(uint32_t bits) +{ + return bits ? bits : PPLX_DIST_ACTIVATION_BITS_DEFAULT; +} + +static int activation_bits_valid(uint32_t bits) +{ + bits = activation_bits_or_default(bits); + return bits == 32u || bits == 16u; +} + +static int activation_wire_bytes(uint32_t bits, uint64_t values, uint32_t *out) +{ + bits = activation_bits_or_default(bits); + if (!activation_bits_valid(bits)) return -1; + uint64_t bytes = values * (uint64_t)(bits / 8u); + if (bytes > UINT32_MAX) return -1; + if (out) *out = (uint32_t)bytes; + return 0; +} + +static uint16_t f32_to_f16(float f) +{ + uint32_t bits; + memcpy(&bits, &f, sizeof(bits)); + + uint32_t sign = (bits >> 16) & 0x8000u; + int32_t exp = (int32_t)((bits >> 23) & 0xffu) - 127 + 15; + uint32_t mant = bits & 0x7fffffu; + + if (exp <= 0) { + if (exp < -10) return (uint16_t)sign; + mant |= 0x800000u; + uint32_t shift = (uint32_t)(14 - exp); + uint32_t half_mant = mant >> shift; + uint32_t round_bit = (mant >> (shift - 1)) & 1u; + uint32_t sticky = mant & ((1u << (shift - 1)) - 1u); + if (round_bit && (sticky || (half_mant & 1u))) half_mant++; + return (uint16_t)(sign | half_mant); + } + + if (exp >= 31) { + if (((bits >> 23) & 0xffu) == 0xffu && mant != 0) + return (uint16_t)(sign | 0x7e00u); + return (uint16_t)(sign | 0x7c00u); + } + + uint32_t half = sign | ((uint32_t)exp << 10) | (mant >> 13); + uint32_t round = mant & 0x1fffu; + if (round > 0x1000u || (round == 0x1000u && (half & 1u))) half++; + return (uint16_t)half; +} + +static float f16_to_f32(uint16_t h) +{ + uint32_t sign = (uint32_t)(h & 0x8000u) << 16; + int32_t exp = (int32_t)((h >> 10) & 0x1fu); + uint32_t mant = h & 0x03ffu; + uint32_t bits; + + if (exp == 0) { + if (mant == 0) { + bits = sign; + } else { + exp = 1; + while ((mant & 0x0400u) == 0) { + mant <<= 1; + exp--; + } + mant &= 0x03ffu; + bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13); + } + } else if (exp == 31) { + bits = sign | 0x7f800000u | (mant << 13); + } else { + bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13); + } + + float f; + memcpy(&f, &bits, sizeof(f)); + return f; +} + +static int encode_activations(void *dst, const float *src, uint64_t values, + uint32_t bits) +{ + bits = activation_bits_or_default(bits); + if (!dst || !src || !activation_bits_valid(bits)) return -1; + if (bits == 32u) { + memcpy(dst, src, values * sizeof(float)); + return 0; + } + uint16_t *out = dst; + for (uint64_t i = 0; i < values; i++) out[i] = f32_to_f16(src[i]); + return 0; +} + +static int decode_activations(float *dst, const void *wire, uint64_t values, + uint32_t bits) +{ + bits = activation_bits_or_default(bits); + if (!dst || !wire || !activation_bits_valid(bits)) return -1; + if (bits == 32u) { + memcpy(dst, wire, values * sizeof(float)); + return 0; + } + const uint16_t *src = wire; + for (uint64_t i = 0; i < values; i++) dst[i] = f16_to_f32(src[i]); + return 0; +} + +static void frame_to_wire(dist_frame_header *h) +{ + h->magic = htonl(h->magic); + h->version = htonl(h->version); + h->type = htonl(h->type); + h->body_bytes = htonl(h->body_bytes); +} + +static void hello_to_wire(dist_hello *h) +{ + uint32_t *v = (uint32_t *)h; + for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = htonl(v[i]); +} + +static void hello_from_wire(dist_hello *h) +{ + uint32_t *v = (uint32_t *)h; + for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = ntohl(v[i]); +} + +static void work_to_wire(dist_work *w) +{ + uint32_t *v = (uint32_t *)w; + for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = htonl(v[i]); +} + +static void work_from_wire(dist_work *w) +{ + uint32_t *v = (uint32_t *)w; + for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = ntohl(v[i]); +} + +static void result_to_wire(dist_result *r) +{ + uint32_t *v = (uint32_t *)r; + for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = htonl(v[i]); +} + +static void result_from_wire(dist_result *r) +{ + uint32_t *v = (uint32_t *)r; + for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = ntohl(v[i]); +} + +static int write_full(int fd, const void *buf, size_t len) +{ + const unsigned char *p = buf; + while (len) { +#ifdef MSG_NOSIGNAL + ssize_t n = send(fd, p, len, MSG_NOSIGNAL); +#else + ssize_t n = send(fd, p, len, 0); +#endif + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return -1; + p += (size_t)n; + len -= (size_t)n; + } + return 0; +} + +static int read_full(int fd, void *buf, size_t len) +{ + unsigned char *p = buf; + while (len) { + ssize_t n = recv(fd, p, len, 0); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return -1; + p += (size_t)n; + len -= (size_t)n; + } + return 0; +} + +static int write_frame_header(int fd, uint32_t type, uint32_t body_bytes) +{ + dist_frame_header h = {PPLX_DIST_MAGIC, PPLX_DIST_VERSION, type, body_bytes}; + frame_to_wire(&h); + return write_full(fd, &h, sizeof(h)); +} + +static int read_frame_header(int fd, uint32_t *type, uint32_t *body_bytes) +{ + dist_frame_header h; + if (read_full(fd, &h, sizeof(h)) != 0) return -1; + h.magic = ntohl(h.magic); + h.version = ntohl(h.version); + h.type = ntohl(h.type); + h.body_bytes = ntohl(h.body_bytes); + if (h.magic != PPLX_DIST_MAGIC || h.version != PPLX_DIST_VERSION || + h.body_bytes > PPLX_DIST_MAX_FRAME_BYTES) + return -1; + *type = h.type; + *body_bytes = h.body_bytes; + return 0; +} + +static void set_socket_options(int fd) +{ + int yes = 1; + struct timeval tv = {PPLX_DIST_SOCKET_TIMEOUT_SEC, 0}; + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); + setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)); + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); +#ifdef SO_NOSIGPIPE + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &yes, sizeof(yes)); +#endif +} + +static int open_listener(const char *host, int port) +{ + char service[16]; + snprintf(service, sizeof(service), "%d", port); + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + struct addrinfo *list = NULL; + if (getaddrinfo(host, service, &hints, &list) != 0) return -1; + + int listen_fd = -1; + for (struct addrinfo *ai = list; ai; ai = ai->ai_next) { + int fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) continue; + int yes = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + if (bind(fd, ai->ai_addr, ai->ai_addrlen) == 0 && + listen(fd, 16) == 0) { + listen_fd = fd; + break; + } + close(fd); + } + freeaddrinfo(list); + return listen_fd; +} + +static int connect_endpoint(const char *host, int port) +{ + char service[16]; + snprintf(service, sizeof(service), "%d", port); + struct addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo *list = NULL; + if (getaddrinfo(host, service, &hints, &list) != 0) return -1; + + int fd = -1; + for (struct addrinfo *ai = list; ai; ai = ai->ai_next) { + fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (fd < 0) continue; + set_socket_options(fd); + if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) break; + close(fd); + fd = -1; + } + freeaddrinfo(list); + return fd; +} + +static int stage_load(dist_stage *stage, const char *model_dir, + int layer_start, int layer_end, int use_mlx, + const pplx_dist_options_t *opts) +{ + memset(stage, 0, sizeof(*stage)); +#ifndef USE_MLX + (void)opts; +#endif +#ifdef USE_MLX + if (use_mlx) { + pplx_mlx_options_t mlx_opts = {0}; + if (opts) { + mlx_opts.quantize_bits = opts->mlx_quantize_bits; + mlx_opts.quantize_group_size = opts->mlx_quantize_group_size; + } + stage->mlx = pplx_mlx_load_slice_with_options( + model_dir, layer_start, layer_end, &mlx_opts); + if (!stage->mlx) return -1; + stage->cfg = pplx_mlx_config(stage->mlx); + return 0; + } +#else + if (use_mlx) return -1; +#endif + stage->model = pplx_model_load_slice(model_dir, layer_start, layer_end); + if (!stage->model) return -1; + stage->workspace = pplx_workspace_new(stage->model); + if (!stage->workspace) { + pplx_model_free(stage->model); + stage->model = NULL; + return -1; + } + stage->cfg = pplx_model_config(stage->model); + return 0; +} + +static void stage_free(dist_stage *stage) +{ + if (!stage) return; + if (stage->workspace) pplx_workspace_free(stage->workspace); + if (stage->model) pplx_model_free(stage->model); +#ifdef USE_MLX + if (stage->mlx) pplx_mlx_free(stage->mlx); +#endif + memset(stage, 0, sizeof(*stage)); +} + +static int stage_forward(dist_stage *stage, const pplx_input_t *inputs, + int batch, const float *input_states, + int layer_start, int layer_end, int apply_final_norm, + float *out_states) +{ +#ifdef USE_MLX + if (stage->mlx) + return pplx_mlx_forward_slice_batch(stage->mlx, inputs, batch, + input_states, layer_start, + layer_end, apply_final_norm, + out_states); +#endif + return pplx_model_forward_slice_batch(stage->model, stage->workspace, + inputs, batch, input_states, + layer_start, layer_end, + apply_final_norm, out_states); +} + +static int send_hello(int fd, const pplx_config_t *cfg, + int layer_start, int layer_end) +{ + uint64_t sig = config_signature(cfg); + dist_hello h; + u64_to_halves(sig, &h.signature_hi, &h.signature_lo); + h.hidden_size = (uint32_t)cfg->hidden_size; + h.n_layers = (uint32_t)cfg->n_layers; + h.vocab_size = (uint32_t)cfg->vocab_size; + h.n_heads = (uint32_t)cfg->n_heads; + h.n_kv_heads = (uint32_t)cfg->n_kv_heads; + h.head_dim = (uint32_t)cfg->head_dim; + h.intermediate_size = (uint32_t)cfg->intermediate_size; + h.layer_start = (uint32_t)layer_start; + h.layer_end = (uint32_t)layer_end; + hello_to_wire(&h); + if (write_frame_header(fd, PPLX_DIST_MSG_HELLO, sizeof(h)) != 0) + return -1; + return write_full(fd, &h, sizeof(h)); +} + +static int recv_hello(int fd, const pplx_config_t *cfg, int expected_start) +{ + uint32_t type = 0, bytes = 0; + dist_hello h; + if (read_frame_header(fd, &type, &bytes) != 0 || + type != PPLX_DIST_MSG_HELLO || bytes != sizeof(h) || + read_full(fd, &h, sizeof(h)) != 0) + return -1; + hello_from_wire(&h); + if (u64_from_halves(h.signature_hi, h.signature_lo) != + config_signature(cfg) || + h.hidden_size != (uint32_t)cfg->hidden_size || + h.n_layers != (uint32_t)cfg->n_layers || + h.layer_start != (uint32_t)expected_start || + h.layer_end != (uint32_t)cfg->n_layers) + return -1; + return 0; +} + +static int send_error_result(int fd, uint64_t request_id, const char *msg) +{ + size_t len = msg ? strlen(msg) : 0; + if (len > PPLX_DIST_MAX_ERROR_BYTES) len = PPLX_DIST_MAX_ERROR_BYTES; + dist_result result; + memset(&result, 0, sizeof(result)); + u64_to_halves(request_id, &result.request_hi, &result.request_lo); + result.status = 1; + result.payload_bytes = (uint32_t)len; + dist_result wire = result; + result_to_wire(&wire); + if (write_frame_header(fd, PPLX_DIST_MSG_RESULT, + (uint32_t)sizeof(wire) + (uint32_t)len) != 0 || + write_full(fd, &wire, sizeof(wire)) != 0) + return -1; + return len ? write_full(fd, msg, len) : 0; +} + +static int worker_process_work(int fd, dist_stage *stage, + int layer_start, int layer_end, + const void *body, uint32_t body_bytes) +{ + uint64_t request_id = 0; + if (body_bytes < sizeof(dist_work)) + return send_error_result(fd, request_id, "truncated WORK frame"); + + dist_work work; + memcpy(&work, body, sizeof(work)); + work_from_wire(&work); + request_id = u64_from_halves(work.request_hi, work.request_lo); + + uint32_t expected_lengths = 0, expected_state_values = 0; + uint32_t expected_state_f32_bytes = 0, expected_state_wire_bytes = 0; + uint32_t states_bits = activation_bits_or_default(work.states_bits); + if (work.batch == 0 || work.batch > PPLX_DIST_MAX_BATCH || + work.total_tokens == 0 || work.total_tokens > PPLX_DIST_MAX_TOKENS || + work.hidden_size != (uint32_t)stage->cfg->hidden_size || + !activation_bits_valid(states_bits) || + mul_u32(work.batch, sizeof(uint32_t), &expected_lengths) != 0 || + mul_u32(work.total_tokens, work.hidden_size, + &expected_state_values) != 0 || + mul_u32(expected_state_values, sizeof(float), + &expected_state_f32_bytes) != 0 || + activation_wire_bytes(states_bits, expected_state_values, + &expected_state_wire_bytes) != 0 || + work.lengths_bytes != expected_lengths || + work.states_bytes != expected_state_wire_bytes) + return send_error_result(fd, request_id, "invalid WORK frame"); + uint64_t expected_body = (uint64_t)sizeof(work) + expected_lengths + + expected_state_wire_bytes; + if (expected_body > UINT32_MAX || body_bytes != (uint32_t)expected_body) + return send_error_result(fd, request_id, "invalid WORK frame"); + + const unsigned char *p = (const unsigned char *)body + sizeof(work); + int *lengths = malloc((size_t)work.batch * sizeof(*lengths)); + pplx_input_t *inputs = calloc(work.batch, sizeof(*inputs)); + float *states = malloc(expected_state_f32_bytes); + float *out_states = malloc(expected_state_f32_bytes); + uint32_t emb_bytes = 0; + if (mul_u32(work.batch, work.hidden_size, &emb_bytes) != 0 || + mul_u32(emb_bytes, sizeof(float), &emb_bytes) != 0) { + free(lengths); free(inputs); free(states); free(out_states); + return send_error_result(fd, request_id, "embedding result is too large"); + } + float *embeddings = malloc(emb_bytes); + if (!lengths || !inputs || !states || !out_states || !embeddings) { + free(lengths); free(inputs); free(states); free(out_states); + free(embeddings); + return send_error_result(fd, request_id, "out of memory"); + } + + uint32_t total = 0; + for (uint32_t i = 0; i < work.batch; i++) { + uint32_t n = 0; + memcpy(&n, p + (size_t)i * sizeof(n), sizeof(n)); + n = ntohl(n); + if (n == 0 || n > PPLX_DIST_MAX_TOKENS - total) { + free(lengths); free(inputs); free(states); free(out_states); + free(embeddings); + return send_error_result(fd, request_id, "invalid sequence length"); + } + lengths[i] = (int)n; + inputs[i].n_tokens = (int)n; + total += n; + } + if (total != work.total_tokens) { + free(lengths); free(inputs); free(states); free(out_states); + free(embeddings); + return send_error_result(fd, request_id, "sequence lengths do not match token total"); + } + if (decode_activations(states, p + expected_lengths, + expected_state_values, states_bits) != 0) { + free(lengths); free(inputs); free(states); free(out_states); + free(embeddings); + return send_error_result(fd, request_id, "invalid activation payload"); + } + + int rc = stage_forward(stage, inputs, (int)work.batch, states, + layer_start, layer_end, 1, out_states); + if (rc == 0) + rc = pplx_pool_batch(stage->cfg, out_states, lengths, + (int)work.batch, embeddings); + free(lengths); free(inputs); free(states); free(out_states); + if (rc != 0) { + free(embeddings); + return send_error_result(fd, request_id, "worker inference failed"); + } + + dist_result result; + memset(&result, 0, sizeof(result)); + u64_to_halves(request_id, &result.request_hi, &result.request_lo); + result.batch = work.batch; + result.dimensions = work.hidden_size; + result.payload_bytes = emb_bytes; + dist_result wire = result; + result_to_wire(&wire); + rc = write_frame_header(fd, PPLX_DIST_MSG_RESULT, + (uint32_t)sizeof(wire) + emb_bytes); + if (rc == 0) rc = write_full(fd, &wire, sizeof(wire)); + if (rc == 0) rc = write_full(fd, embeddings, emb_bytes); + free(embeddings); + return rc; +} + +static int worker_serve_connection(int fd, dist_stage *stage, + int layer_start, int layer_end) +{ + if (send_hello(fd, stage->cfg, layer_start, layer_end) != 0) return -1; + for (;;) { + uint32_t type = 0, bytes = 0; + if (read_frame_header(fd, &type, &bytes) != 0) return -1; + if (type != PPLX_DIST_MSG_WORK || bytes == 0) return -1; + void *body = malloc(bytes); + if (!body) return -1; + int rc = read_full(fd, body, bytes); + if (rc == 0) + rc = worker_process_work(fd, stage, layer_start, layer_end, + body, bytes); + free(body); + if (rc != 0) return -1; + } +} + +int pplx_dist_run_worker(const char *model_dir, const char *host, int port, + int layer_start, int layer_end, int use_mlx) +{ + return pplx_dist_run_worker_with_options(model_dir, host, port, + layer_start, layer_end, use_mlx, + NULL); +} + +int pplx_dist_run_worker_with_options( + const char *model_dir, const char *host, int port, + int layer_start, int layer_end, int use_mlx, + const pplx_dist_options_t *opts) +{ + if (!model_dir || !host || port <= 0 || port > 65535 || + !host_is_little_endian()) { + fprintf(stderr, "pplx-dist: invalid worker configuration\n"); + return 1; + } + dist_stage stage; + if (stage_load(&stage, model_dir, layer_start, layer_end, use_mlx, + opts) != 0) { + fprintf(stderr, "pplx-dist: failed to load worker shard\n"); + return 1; + } + if (layer_end != stage.cfg->n_layers) { + fprintf(stderr, "pplx-dist: initial worker must own final layer %d\n", + stage.cfg->n_layers); + stage_free(&stage); + return 1; + } + int listen_fd = open_listener(host, port); + if (listen_fd < 0) { + fprintf(stderr, "pplx-dist: listen failed on %s:%d: %s\n", + host, port, strerror(errno)); + stage_free(&stage); + return 1; + } + fprintf(stderr, "pplx-dist: worker listening on %s:%d layers=[%d,%d)\n", + host, port, layer_start, layer_end); + for (;;) { + int fd = accept(listen_fd, NULL, NULL); + if (fd < 0) { + if (errno == EINTR) continue; + fprintf(stderr, "pplx-dist: accept failed: %s\n", strerror(errno)); + break; + } + set_socket_options(fd); + if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator connected\n"); + worker_serve_connection(fd, &stage, layer_start, layer_end); + close(fd); + if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator disconnected\n"); + } + close(listen_fd); + stage_free(&stage); + return 1; +} + +pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, + const char *host, int port, + int local_layer_end, int use_mlx) +{ + return pplx_dist_client_open_with_options(model_dir, host, port, + local_layer_end, use_mlx, NULL); +} + +pplx_dist_client_t *pplx_dist_client_open_with_options( + const char *model_dir, const char *host, int port, + int local_layer_end, int use_mlx, const pplx_dist_options_t *opts) +{ + if (!model_dir || !host || port <= 0 || port > 65535 || + local_layer_end <= 0 || !host_is_little_endian()) + return NULL; + + pplx_dist_client_t *client = calloc(1, sizeof(*client)); + if (!client) return NULL; + client->fd = -1; + if (stage_load(&client->local, model_dir, 0, local_layer_end, use_mlx, + opts) != 0) + goto fail; + client->activation_bits = activation_bits_or_default( + opts ? (uint32_t)opts->activation_bits : 0); + if (!activation_bits_valid(client->activation_bits)) + goto fail; + client->fd = connect_endpoint(host, port); + if (client->fd < 0 || + recv_hello(client->fd, client->local.cfg, local_layer_end) != 0) + goto fail; + client->local_layer_end = local_layer_end; + client->next_request_id = 1; + return client; + +fail: + pplx_dist_client_free(client); + return NULL; +} + +void pplx_dist_client_free(pplx_dist_client_t *client) +{ + if (!client) return; + if (client->fd >= 0) close(client->fd); + stage_free(&client->local); + free(client); +} + +const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client) +{ + return client ? client->local.cfg : NULL; +} + +static int client_transport_error(pplx_dist_client_t *client) +{ + if (client->fd >= 0) { + close(client->fd); + client->fd = -1; + } + return -1; +} + +int pplx_dist_client_embed_batch(pplx_dist_client_t *client, + const pplx_input_t *inputs, int batch, + float *out_embeddings) +{ + if (!client || client->fd < 0 || !inputs || batch <= 0 || + batch > (int)PPLX_DIST_MAX_BATCH || !out_embeddings) + return -1; + + uint32_t total_tokens = 0; + for (int i = 0; i < batch; i++) { + if (!inputs[i].ids || inputs[i].n_tokens <= 0 || + (uint32_t)inputs[i].n_tokens > PPLX_DIST_MAX_TOKENS - total_tokens) + return -1; + total_tokens += (uint32_t)inputs[i].n_tokens; + } + + uint32_t state_values = 0, state_f32_bytes = 0, state_wire_bytes = 0; + uint32_t lengths_bytes = 0; + uint32_t hidden = (uint32_t)client->local.cfg->hidden_size; + uint32_t states_bits = activation_bits_or_default(client->activation_bits); + if (mul_u32(total_tokens, hidden, &state_values) != 0 || + mul_u32(state_values, sizeof(float), &state_f32_bytes) != 0 || + activation_wire_bytes(states_bits, state_values, &state_wire_bytes) != 0 || + mul_u32((uint32_t)batch, sizeof(uint32_t), &lengths_bytes) != 0) + return -1; + uint64_t body_bytes64 = (uint64_t)sizeof(dist_work) + lengths_bytes + + state_wire_bytes; + if (body_bytes64 > PPLX_DIST_MAX_FRAME_BYTES) + return -1; + uint32_t body_bytes = (uint32_t)body_bytes64; + float *states = malloc(state_f32_bytes); + void *wire_states = malloc(state_wire_bytes); + if (!states || !wire_states) { + free(states); free(wire_states); + return -1; + } + int rc = stage_forward(&client->local, inputs, batch, NULL, 0, + client->local_layer_end, 0, states); + if (rc != 0) { + free(states); free(wire_states); + return -1; + } + if (encode_activations(wire_states, states, state_values, states_bits) != 0) { + free(states); free(wire_states); + return -1; + } + + uint64_t request_id = client->next_request_id++; + dist_work work; + u64_to_halves(request_id, &work.request_hi, &work.request_lo); + work.batch = (uint32_t)batch; + work.total_tokens = total_tokens; + work.hidden_size = hidden; + work.lengths_bytes = lengths_bytes; + work.states_bytes = state_wire_bytes; + work.states_bits = states_bits; + dist_work wire = work; + work_to_wire(&wire); + rc = write_frame_header(client->fd, PPLX_DIST_MSG_WORK, body_bytes); + if (rc == 0) rc = write_full(client->fd, &wire, sizeof(wire)); + for (int i = 0; rc == 0 && i < batch; i++) { + uint32_t n = htonl((uint32_t)inputs[i].n_tokens); + rc = write_full(client->fd, &n, sizeof(n)); + } + if (rc == 0) rc = write_full(client->fd, wire_states, state_wire_bytes); + free(states); free(wire_states); + if (rc != 0) return client_transport_error(client); + + uint32_t type = 0, bytes = 0; + dist_result result; + if (read_frame_header(client->fd, &type, &bytes) != 0 || + type != PPLX_DIST_MSG_RESULT || bytes < sizeof(result) || + read_full(client->fd, &result, sizeof(result)) != 0) + return client_transport_error(client); + result_from_wire(&result); + uint32_t payload_bytes = bytes - (uint32_t)sizeof(result); + if (result.payload_bytes != payload_bytes || + u64_from_halves(result.request_hi, result.request_lo) != request_id) + return client_transport_error(client); + if (result.status != 0) { + char error[PPLX_DIST_MAX_ERROR_BYTES + 1]; + if (payload_bytes > PPLX_DIST_MAX_ERROR_BYTES || + read_full(client->fd, error, payload_bytes) != 0) + return client_transport_error(client); + error[payload_bytes] = '\0'; + fprintf(stderr, "pplx-dist: worker error: %s\n", error); + return client_transport_error(client); + } + uint32_t expected = 0; + if (result.batch != (uint32_t)batch || result.dimensions != hidden || + mul_u32((uint32_t)batch, hidden, &expected) != 0 || + mul_u32(expected, sizeof(float), &expected) != 0 || + payload_bytes != expected || + read_full(client->fd, out_embeddings, payload_bytes) != 0) + return client_transport_error(client); + return 0; +} diff --git a/embed_distributed.h b/embed_distributed.h new file mode 100644 index 0000000..cd97147 --- /dev/null +++ b/embed_distributed.h @@ -0,0 +1,47 @@ +/* + * pplx_distributed.h - Stateless two-node layer-sharded inference. + */ + +#ifndef PPLX_DISTRIBUTED_H +#define PPLX_DISTRIBUTED_H + +#include "pplx_embed.h" + +typedef struct pplx_dist_client pplx_dist_client_t; + +typedef struct { + int mlx_quantize_bits; + int mlx_quantize_group_size; + int activation_bits; /* 0 defaults to fp32; supported: 32, 16 */ +} pplx_dist_options_t; + +/* + * Open a coordinator-side connection. The coordinator owns layers + * [0, local_layer_end); the remote worker must own [local_layer_end, N). + */ +pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, + const char *host, int port, + int local_layer_end, int use_mlx); +pplx_dist_client_t *pplx_dist_client_open_with_options( + const char *model_dir, const char *host, int port, + int local_layer_end, int use_mlx, const pplx_dist_options_t *opts); +void pplx_dist_client_free(pplx_dist_client_t *client); + +const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client); + +int pplx_dist_client_embed_batch(pplx_dist_client_t *client, + const pplx_input_t *inputs, int batch, + float *out_embeddings); + +/* + * Run a final-stage worker until the process is terminated. The worker owns + * [layer_start, layer_end), and layer_end must be the model's final layer. + */ +int pplx_dist_run_worker(const char *model_dir, const char *host, int port, + int layer_start, int layer_end, int use_mlx); +int pplx_dist_run_worker_with_options( + const char *model_dir, const char *host, int port, + int layer_start, int layer_end, int use_mlx, + const pplx_dist_options_t *opts); + +#endif /* PPLX_DISTRIBUTED_H */ diff --git a/embed_internal.h b/embed_internal.h new file mode 100644 index 0000000..1e7e4d0 --- /dev/null +++ b/embed_internal.h @@ -0,0 +1,64 @@ +#ifndef PPLX_EMBED_INTERNAL_H +#define PPLX_EMBED_INTERNAL_H + +#include "pplx_embed.h" +#include "qwen_safetensors.h" + +typedef struct { + safetensor_dtype_t dtype; + const void *data; +} pplx_weight_ref_t; + +typedef struct { + pplx_weight_ref_t wq; /* [q_dim, hidden] */ + pplx_weight_ref_t wk; /* [kv_dim, hidden] */ + pplx_weight_ref_t wv; /* [kv_dim, hidden] */ + pplx_weight_ref_t wo; /* [hidden, q_dim] */ + const float *q_norm; /* [head_dim] */ + const float *k_norm; /* [head_dim] */ + const float *input_norm; /* [hidden] */ + const float *post_attn_norm; /* [hidden] */ + pplx_weight_ref_t gate_proj; /* [intermediate, hidden] */ + pplx_weight_ref_t up_proj; /* [intermediate, hidden] */ + pplx_weight_ref_t down_proj; /* [hidden, intermediate] */ +} pplx_layer_t; + +typedef struct { + pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ + pplx_layer_t *layers; /* [n_layers] heap-allocated */ + const float *norm; /* [hidden] */ +} pplx_weights_t; + +struct pplx_model { + pplx_config_t config; + pplx_weights_t weights; + void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ +}; + +struct pplx_workspace { + const pplx_model_t *model; /* immutable model this workspace belongs to */ + int buf_seq_cap; + float *x; /* [seq, hidden] */ + float *x_norm; /* [seq, hidden] */ + float *q; /* [seq, q_dim] */ + float *k; /* [seq, kv_dim] */ + float *v; /* [seq, kv_dim] */ + float *attn_out; /* [seq, q_dim] */ + float *proj_out; /* [seq, hidden] */ + float *ffn_gate; /* [seq, intermediate] */ + float *ffn_up; /* [seq, intermediate] */ + float *attn_scores; /* reusable BLAS attention score tiles */ + size_t attn_scores_bytes; + + int *offsets; /* [batch + 1] packed sequence offsets */ + int offsets_cap; + + /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */ + float *rope_cos; + float *rope_sin; + int rope_cache_cap; +}; + +#endif /* PPLX_EMBED_INTERNAL_H */ diff --git a/embed_mlx.c b/embed_mlx.c new file mode 100644 index 0000000..602571d --- /dev/null +++ b/embed_mlx.c @@ -0,0 +1,2296 @@ +#include "pplx_embed_mlx.h" +#include "pplx_embed.h" +#include "qwen_safetensors.h" + +#include + +#include +#include +#include +#include +#include +#include + +/* ======================================================================== + * Per-layer MLX weight arrays + * ======================================================================== */ + +typedef struct { + mlx_array w; + mlx_array wt; + mlx_array scales; + mlx_array biases; + int quantized; + int bits; + int group_size; +} mlx_linear_t; + +typedef struct { + mlx_linear_t wq, wk, wv, wo; + mlx_array q_norm, k_norm; + mlx_array input_norm, post_attn_norm; + mlx_linear_t gate_proj, up_proj, down_proj; +} mlx_layer_t; + +struct pplx_mlx_ctx { + pplx_config_t config; + mlx_array embed_tokens; + mlx_layer_t *layers; /* heap-allocated [n_layers] */ + mlx_array norm; + mlx_stream stream; + multi_safetensors_t *ms; + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ + int quantize_bits; + int quantize_group_size; + mlx_dtype weight_dtype; +}; + +struct pplx_mlx_late_ctx { + pplx_mlx_ctx_t *base; + multi_safetensors_t *dense_ms; + mlx_linear_t projection; + int token_dim; +}; + +struct pplx_mlx_late_vectors { + pplx_mlx_late_ctx_t *owner; + mlx_array vectors; /* [tokens, dim] */ + int tokens; + int dim; +}; + +/* ======================================================================== + * Helpers + * ======================================================================== */ + +static int arr_ok(mlx_array a) { return a.ctx != NULL; } + +static int mlx_mul_size(size_t a, size_t b, size_t *out) +{ + if (a != 0 && b > SIZE_MAX / a) return -1; + *out = a * b; + return 0; +} + +static mlx_dtype mlx_weight_dtype(const pplx_mlx_ctx_t *ctx) +{ + return ctx->weight_dtype; +} + +static size_t safetensor_dtype_size(safetensor_dtype_t dtype) +{ + switch (dtype) { + case DTYPE_F32: return sizeof(float); + case DTYPE_BF16: return sizeof(uint16_t); + default: return 0; + } +} + +static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, + const char *name, const int *shape, + int ndim) +{ + size_t elem_size = safetensor_dtype_size(t->dtype); + if (elem_size == 0) { + fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); + return 0; + } + if (t->ndim != ndim) { + fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", + name, t->ndim, ndim); + return 0; + } + + size_t numel = 1; + for (int i = 0; i < ndim; i++) { + if (shape[i] <= 0 || t->shape[i] != shape[i]) { + fprintf(stderr, "mlx: bad shape for %s at dim %d: " + "got %lld, expected %d\n", + name, i, (long long)t->shape[i], shape[i]); + return 0; + } + if (mlx_mul_size(numel, (size_t)shape[i], &numel) != 0) { + fprintf(stderr, "mlx: shape too large for %s\n", name); + return 0; + } + } + + size_t bytes; + if (mlx_mul_size(numel, elem_size, &bytes) != 0) { + fprintf(stderr, "mlx: tensor data too large for %s\n", name); + return 0; + } + if (t->data_size != bytes) { + fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", + name, t->data_size, bytes); + return 0; + } + + if (sf->header_size > SIZE_MAX - 8) return 0; + size_t data_start = 8 + sf->header_size; + if (data_start > sf->file_size || + t->data_offset > sf->file_size - data_start || + t->data_size > sf->file_size - data_start - t->data_offset) { + fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name); + return 0; + } + return 1; +} + +static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, + const int *shape, int ndim) +{ + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "mlx: tensor not found: %s\n", name); + return (mlx_array){0}; + } + if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim)) + return (mlx_array){0}; + mlx_dtype dtype; + if (t->dtype == DTYPE_F32) { + dtype = MLX_FLOAT32; + } else if (t->dtype == DTYPE_BF16) { + dtype = MLX_BFLOAT16; + } else { + fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); + return (mlx_array){0}; + } + return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); +} + +static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms, + const char *name, + const int *shape, int ndim) +{ + mlx_linear_t l = {0}; + l.w = load_tensor(ms, name, shape, ndim); + return l; +} + +static int linear_ok(mlx_linear_t l) +{ + return arr_ok(l.w) || arr_ok(l.wt); +} + +static void free_linear(mlx_linear_t *l) +{ + mlx_array_free(l->w); + mlx_array_free(l->wt); + mlx_array_free(l->scales); + mlx_array_free(l->biases); + memset(l, 0, sizeof(*l)); +} + +static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) +{ + if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt)) + return 0; + + mlx_array transposed = mlx_array_new(); + mlx_transpose(&transposed, l->w, s); + if (!arr_ok(transposed)) { + mlx_array_free(transposed); + fprintf(stderr, "mlx: failed to prepare transposed linear weight\n"); + return -1; + } + + l->wt = mlx_array_new(); + mlx_contiguous(&l->wt, transposed, false, s); + mlx_array_free(transposed); + if (!arr_ok(l->wt)) { + mlx_array_free(l->wt); + l->wt = (mlx_array){0}; + fprintf(stderr, "mlx: failed to materialize transposed linear weight\n"); + return -1; + } + mlx_array_eval(l->wt); + mlx_synchronize(s); + mlx_array_free(l->w); + l->w = (mlx_array){0}; + return 0; +} + +static int quantize_linear(mlx_linear_t *l, int bits, int group_size, + mlx_stream s) +{ + if (!l || !arr_ok(l->w) || bits == 0) return 0; + if (bits != 8) { + fprintf(stderr, "mlx: quantized weights support only 8 bits\n"); + return -1; + } + if (group_size <= 0) group_size = 64; + + mlx_optional_int gs = {.value = group_size, .has_value = true}; + mlx_optional_int qb = {.value = bits, .has_value = true}; + mlx_array null_arr = (mlx_array){0}; + mlx_vector_array q = mlx_vector_array_new(); + mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s); + if (!q.ctx || mlx_vector_array_size(q) < 2) { + mlx_vector_array_free(q); + fprintf(stderr, "mlx: quantize failed\n"); + return -1; + } + + mlx_array qw = (mlx_array){0}; + mlx_array scales = (mlx_array){0}; + mlx_array biases = (mlx_array){0}; + mlx_vector_array_get(&qw, q, 0); + mlx_vector_array_get(&scales, q, 1); + if (mlx_vector_array_size(q) >= 3) + mlx_vector_array_get(&biases, q, 2); + mlx_vector_array_free(q); + + if (!arr_ok(qw) || !arr_ok(scales)) { + mlx_array_free(qw); + mlx_array_free(scales); + mlx_array_free(biases); + fprintf(stderr, "mlx: quantize returned invalid arrays\n"); + return -1; + } + mlx_array_eval(qw); + mlx_array_eval(scales); + if (arr_ok(biases)) mlx_array_eval(biases); + mlx_synchronize(s); + + mlx_array_free(l->w); + mlx_array_free(l->wt); + l->w = qw; + l->wt = (mlx_array){0}; + l->scales = scales; + l->biases = biases; + l->quantized = 1; + l->bits = bits; + l->group_size = group_size; + return 0; +} + +/* y = x @ W^T */ +static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) +{ + if (W->quantized) { + if (!arr_ok(W->w)) return (mlx_array){0}; + mlx_array res = mlx_array_new(); + mlx_optional_int gs = {.value = W->group_size, .has_value = true}; + mlx_optional_int bits = {.value = W->bits, .has_value = true}; + mlx_array null_arr = (mlx_array){0}; + mlx_quantized_matmul(&res, x, W->w, W->scales, + arr_ok(W->biases) ? W->biases : null_arr, + true, gs, bits, "affine", s); + return res; + } + mlx_array Wt; + int free_wt = 0; + if (arr_ok(W->wt)) { + Wt = W->wt; + } else if (arr_ok(W->w)) { + Wt = mlx_array_new(); + mlx_transpose(&Wt, W->w, s); + free_wt = 1; + } else { + return (mlx_array){0}; + } + mlx_array res = mlx_array_new(); + if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && + mlx_array_dtype(x) == MLX_FLOAT32) { + /* F32 activations with BF16 weights: upcast the weight operand and run + * the matmul in F32 (so accumulation is F32). Only the weights are + * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers), + * matching the CPU/CUDA accuracy while keeping BF16 weight storage. */ + mlx_array Wf = mlx_array_new(); + mlx_astype(&Wf, Wt, MLX_FLOAT32, s); + mlx_matmul(&res, x, Wf, s); + mlx_array_free(Wf); + } else { + mlx_matmul(&res, x, Wt, s); + } + if (free_wt) mlx_array_free(Wt); + return res; +} + +const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx) +{ + return ctx ? &ctx->config : NULL; +} + +/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the + * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the + * residual stream in F32 while keeping the large matmul weights in BF16. */ +static void upcast_f32_inplace(mlx_array *a, mlx_stream s) +{ + if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return; + mlx_array f = mlx_array_new(); + mlx_astype(&f, *a, MLX_FLOAT32, s); + mlx_array_free(*a); + *a = f; +} + +/* ======================================================================== + * Load + * ======================================================================== */ + +/* We duplicate a minimal config parser here so MLX can load standalone. + * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */ + +static const char *skip_ws_m(const char *p) +{ + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; + return p; +} + +static const char *json_find_m(const char *json, const char *key) +{ + size_t klen = strlen(key); + const char *p = json; + while ((p = strstr(p, key)) != NULL) { + if (p > json && *(p - 1) == '"') { + const char *a = p + klen; + if (*a == '"') { + a = skip_ws_m(a + 1); + if (*a == ':') return skip_ws_m(a + 1); + } + } + p += klen; + } + return NULL; +} + +static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir) +{ + char path[1024]; + int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir); + if (path_len < 0 || (size_t)path_len >= sizeof(path)) { + fprintf(stderr, "mlx: model path too long\n"); + return -1; + } + FILE *f = fopen(path, "rb"); + if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; } + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + fclose(f); free(buf); return -1; + } + fclose(f); + buf[sz] = '\0'; + +#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0) +#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0) + GI("hidden_size", hidden_size, 0); + GI("num_hidden_layers", n_layers, 0); + GI("num_attention_heads", n_heads, 0); + GI("num_key_value_heads", n_kv_heads, 0); + GI("head_dim", head_dim, PPLX_HEAD_DIM); + GI("intermediate_size", intermediate_size, 0); + GI("vocab_size", vocab_size, PPLX_VOCAB_SIZE); + GF("rms_norm_eps", rms_norm_eps, 1e-6f); + GF("rope_theta", rope_theta, 1000000.0f); +#undef GI +#undef GF + + cfg->q_dim = 0; + cfg->kv_dim = 0; + if (cfg->n_heads > 0 && cfg->head_dim > 0 && + cfg->n_heads <= INT_MAX / cfg->head_dim) + cfg->q_dim = cfg->n_heads * cfg->head_dim; + if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && + cfg->n_kv_heads <= INT_MAX / cfg->head_dim) + cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; + free(buf); + + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || + cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || + cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || + cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || + (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || + !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || + !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { + fprintf(stderr, "mlx: invalid config in %s " + "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", + path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, + cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); + return -1; + } + if (cfg->n_layers > PPLX_MAX_LAYERS) { + fprintf(stderr, "mlx: too many layers (%d > %d)\n", + cfg->n_layers, PPLX_MAX_LAYERS); + return -1; + } + return 0; +} + +static int mlx_model_dir_has_file(const char *model_dir, const char *rel) +{ + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); + if (n < 0 || (size_t)n >= sizeof(path)) return 0; + FILE *f = fopen(path, "rb"); + if (!f) return 0; + fclose(f); + return 1; +} + +static int mlx_model_dir_has_late_projection(const char *model_dir) +{ + return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") && + mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors"); +} + +static pplx_mlx_options_t normalize_mlx_options(const pplx_mlx_options_t *opts) +{ + pplx_mlx_options_t out = {0}; + if (opts) out = *opts; + if (out.quantize_bits != 0 && + out.quantize_bits != 8) { + fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); + out.quantize_bits = -1; + } + if (out.quantize_bits && out.quantize_group_size <= 0) + out.quantize_group_size = 64; + return out; +} + +static int quantize_layer(mlx_layer_t *l, int bits, int group_size, + mlx_stream s) +{ + if (bits == 0) return 0; + return quantize_linear(&l->wq, bits, group_size, s) || + quantize_linear(&l->wk, bits, group_size, s) || + quantize_linear(&l->wv, bits, group_size, s) || + quantize_linear(&l->wo, bits, group_size, s) || + quantize_linear(&l->gate_proj, bits, group_size, s) || + quantize_linear(&l->up_proj, bits, group_size, s) || + quantize_linear(&l->down_proj, bits, group_size, s); +} + +static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) +{ + return prepare_linear_transpose(&l->wq, s) || + prepare_linear_transpose(&l->wk, s) || + prepare_linear_transpose(&l->wv, s) || + prepare_linear_transpose(&l->wo, s) || + prepare_linear_transpose(&l->gate_proj, s) || + prepare_linear_transpose(&l->up_proj, s) || + prepare_linear_transpose(&l->down_proj, s); +} + +static pplx_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, + int layer_start, int layer_end, + const pplx_mlx_options_t *opts, + int allow_late) +{ + pplx_mlx_options_t options = normalize_mlx_options(opts); + if (options.quantize_bits < 0) return NULL; + if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "mlx: late-interaction models require token-level " + "MaxSim support and are not valid pooled embedding models " + "yet\n"); + return NULL; + } + + pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); + if (!ctx) return NULL; + ctx->quantize_bits = options.quantize_bits; + ctx->quantize_group_size = options.quantize_group_size; + + if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } + + const pplx_config_t *c = &ctx->config; + if (layer_end == -1) + layer_end = c->n_layers; + if (layer_start < 0 || layer_start >= layer_end || + layer_end > c->n_layers) { + fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", + layer_start, layer_end, c->n_layers); + free(ctx); + return NULL; + } + ctx->layer_start = layer_start; + ctx->layer_end = layer_end; + + ctx->ms = multi_safetensors_open(model_dir); + if (!ctx->ms) { + fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); + free(ctx); return NULL; + } + + int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; + int inter = c->intermediate_size, hd = c->head_dim; + + /* Only the first stage needs token embeddings. */ + if (layer_start == 0) { + int emb_shape[] = {c->vocab_size, h}; + ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", + emb_shape, 2); + if (!arr_ok(ctx->embed_tokens)) goto fail; + } + + /* Layers */ + ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); + if (!ctx->layers) goto fail; + + char name[256]; + for (int i = layer_start; i < layer_end; i++) { + mlx_layer_t *l = &ctx->layers[i]; + +#define LD(fld, fmt, ...) do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ + if (!linear_ok(l->fld)) goto fail; \ +} while (0) + + LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); + LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); + LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); + LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); +#undef LD + +#define LD(fld, fmt, ...) do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ + if (!arr_ok(l->fld)) goto fail; \ +} while (0) + + LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); + LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); + LD(input_norm, "layers.%d.input_layernorm.weight", h); + LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); +#undef LD + +#define LD(fld, fmt, ...) do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ + if (!linear_ok(l->fld)) goto fail; \ +} while (0) + + LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); + LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); + LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); +#undef LD + + if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); + } + + if (layer_end == c->n_layers) { + int ns[] = {h}; + ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); + if (!arr_ok(ctx->norm)) goto fail; + } + + ctx->stream = mlx_default_gpu_stream_new(); + ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w); + /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the + * tiny norm vectors so they match the F32 activations in mlx_forward_layers; + * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */ + if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) { + for (int i = layer_start; i < layer_end; i++) { + mlx_layer_t *l = &ctx->layers[i]; + upcast_f32_inplace(&l->q_norm, ctx->stream); + upcast_f32_inplace(&l->k_norm, ctx->stream); + upcast_f32_inplace(&l->input_norm, ctx->stream); + upcast_f32_inplace(&l->post_attn_norm, ctx->stream); + } + upcast_f32_inplace(&ctx->norm, ctx->stream); + } + if (ctx->quantize_bits) { + for (int i = layer_start; i < layer_end; i++) { + if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, + ctx->quantize_group_size, ctx->stream) != 0) + goto fail; + } + mlx_clear_cache(); + if (pplx_verbose >= 1) + fprintf(stderr, "mlx: quantized linear weights to %d-bit " + "(group_size=%d)\n", + ctx->quantize_bits, ctx->quantize_group_size); + } else { + for (int i = layer_start; i < layer_end; i++) { + if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0) + goto fail; + } + } + + if (pplx_verbose >= 2) { + size_t active = 0, cache = 0, peak = 0; + mlx_get_active_memory(&active); + mlx_get_cache_memory(&cache); + mlx_get_peak_memory(&peak); + fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB " + "peak=%.2f GiB\n", + (double)active / (1024.0 * 1024.0 * 1024.0), + (double)cache / (1024.0 * 1024.0 * 1024.0), + (double)peak / (1024.0 * 1024.0 * 1024.0)); + } + + if (pplx_verbose >= 1) + fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", + layer_start, layer_end, h); + + return ctx; + +fail: + pplx_mlx_free(ctx); + return NULL; +} + +pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) +{ + return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); +} + +pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, + const pplx_mlx_options_t *opts) +{ + return mlx_load_range_ex(model_dir, 0, -1, opts, 0); +} + +pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, + int layer_start, int layer_end) +{ + return mlx_load_range_ex(model_dir, layer_start, layer_end, NULL, 0); +} + +pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( + const char *model_dir, int layer_start, int layer_end, + const pplx_mlx_options_t *opts) +{ + return mlx_load_range_ex(model_dir, layer_start, layer_end, opts, 0); +} + +/* ======================================================================== + * Free + * ======================================================================== */ + +static void free_mlx_layer(mlx_layer_t *l) +{ + free_linear(&l->wq); free_linear(&l->wk); + free_linear(&l->wv); free_linear(&l->wo); + mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); + mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); + free_linear(&l->gate_proj); free_linear(&l->up_proj); + free_linear(&l->down_proj); +} + +void pplx_mlx_free(pplx_mlx_ctx_t *ctx) +{ + if (!ctx) return; + mlx_array_free(ctx->embed_tokens); + if (ctx->layers) { + for (int i = 0; i < ctx->config.n_layers; i++) + free_mlx_layer(&ctx->layers[i]); + free(ctx->layers); + } + mlx_array_free(ctx->norm); + if (ctx->stream.ctx) mlx_stream_free(ctx->stream); + if (ctx->ms) multi_safetensors_close(ctx->ms); + free(ctx); +} + +pplx_mlx_late_ctx_t *pplx_mlx_late_load_with_options( + const char *model_dir, const pplx_mlx_options_t *opts) +{ + if (!mlx_model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "mlx late: missing 1_Dense projection\n"); + return NULL; + } + + pplx_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); + if (!late) return NULL; + + late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1); + if (!late->base) goto fail; + + char dense_dir[1024]; + int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir); + if (n < 0 || (size_t)n >= sizeof(dense_dir)) { + fprintf(stderr, "mlx late: model path too long\n"); + goto fail; + } + + late->dense_ms = multi_safetensors_open(dense_dir); + if (!late->dense_ms) { + fprintf(stderr, "mlx late: failed to open safetensors in %s\n", + dense_dir); + goto fail; + } + + const safetensor_t *t = multi_safetensors_find(late->dense_ms, + "linear.weight", NULL); + if (!t) { + fprintf(stderr, "mlx late: missing linear.weight\n"); + goto fail; + } + if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX || + t->shape[1] != late->base->config.hidden_size) { + fprintf(stderr, "mlx late: bad projection shape\n"); + goto fail; + } + + int shape[] = { (int)t->shape[0], late->base->config.hidden_size }; + late->projection = load_linear_tensor(late->dense_ms, "linear.weight", + shape, 2); + if (!linear_ok(late->projection)) goto fail; + if (prepare_linear_transpose(&late->projection, late->base->stream) != 0) + goto fail; + late->token_dim = shape[0]; + + if (pplx_verbose >= 1) + fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", + late->token_dim); + return late; + +fail: + pplx_mlx_late_free(late); + return NULL; +} + +pplx_mlx_late_ctx_t *pplx_mlx_late_load(const char *model_dir) +{ + return pplx_mlx_late_load_with_options(model_dir, NULL); +} + +void pplx_mlx_late_free(pplx_mlx_late_ctx_t *ctx) +{ + if (!ctx) return; + free_linear(&ctx->projection); + if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms); + pplx_mlx_free(ctx->base); + free(ctx); +} + +const pplx_config_t *pplx_mlx_late_config(const pplx_mlx_late_ctx_t *ctx) +{ + return ctx && ctx->base ? &ctx->base->config : NULL; +} + +int pplx_mlx_late_token_dim(const pplx_mlx_late_ctx_t *ctx) +{ + return ctx ? ctx->token_dim : 0; +} + +/* ======================================================================== + * Forward pass + * ======================================================================== */ + +static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, + int batch, int max_seq, int has_padding, + mlx_array attn_mask, + int layer_start, int layer_end) +{ + const pplx_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + int n_heads = c->n_heads; + int n_kv_heads = c->n_kv_heads; + int head_dim = c->head_dim; + int q_dim = c->q_dim; + float scale = 1.0f / sqrtf((float)head_dim); + mlx_array null_arr = (mlx_array){0}; + + /* For non-quantized BF16 weights, run the residual stream, norms, and + * attention in F32; matmuls downcast their operands to BF16 internally + * (see linear). This keeps weight storage at BF16 while avoiding the + * cross-layer accumulation error of a fully BF16 forward. F32 and quantized + * models are unaffected. */ + mlx_array local_mask = attn_mask; + int free_mask = 0; + int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 && + !ctx->layers[layer_start].wq.quantized; + if (mixed_bf16) { + if (mlx_array_dtype(x) != MLX_FLOAT32) { + mlx_array xf = mlx_array_new(); + mlx_astype(&xf, x, MLX_FLOAT32, S); + mlx_array_free(x); + x = xf; + } + if (has_padding && arr_ok(attn_mask) && + mlx_array_dtype(attn_mask) != MLX_FLOAT32) { + local_mask = mlx_array_new(); + mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S); + free_mask = 1; + } + } + + for (int layer = layer_start; layer < layer_end; layer++) { + mlx_layer_t *l = &ctx->layers[layer]; + + mlx_array xn = mlx_array_new(); + mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); + + mlx_array q_flat = linear(xn, &l->wq, S); + mlx_array k_flat = linear(xn, &l->wk, S); + mlx_array v_flat = linear(xn, &l->wv, S); + mlx_array_free(xn); + + int q_shape[] = {batch, max_seq, n_heads, head_dim}; + int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; + mlx_array q = mlx_array_new(); + mlx_array k = mlx_array_new(); + mlx_array v = mlx_array_new(); + mlx_reshape(&q, q_flat, q_shape, 4, S); + mlx_reshape(&k, k_flat, k_shape, 4, S); + mlx_reshape(&v, v_flat, k_shape, 4, S); + mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); + + mlx_array qn = mlx_array_new(); + mlx_array kn = mlx_array_new(); + mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); + mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); + mlx_array_free(q); mlx_array_free(k); + + /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ + int perm[] = {0, 2, 1, 3}; + mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); + mlx_transpose_axes(&qt, qn, perm, 4, S); + mlx_transpose_axes(&kt, kn, perm, 4, S); + mlx_transpose_axes(&vt, v, perm, 4, S); + mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v); + + mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; + mlx_array qr = mlx_array_new(), kr = mlx_array_new(); + mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_array_free(qt); mlx_array_free(kt); + + mlx_array attn = mlx_array_new(); + mlx_fast_scaled_dot_product_attention( + &attn, qr, kr, vt, scale, "", + has_padding ? local_mask : null_arr, null_arr, S); + mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); + + mlx_array attn_t = mlx_array_new(); + mlx_transpose_axes(&attn_t, attn, perm, 4, S); + mlx_array_free(attn); + + int flat_shape[] = {batch, max_seq, q_dim}; + mlx_array attn_flat = mlx_array_new(); + mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S); + mlx_array_free(attn_t); + + mlx_array proj = linear(attn_flat, &l->wo, S); + mlx_array_free(attn_flat); + mlx_array x2 = mlx_array_new(); + mlx_add(&x2, x, proj, S); + mlx_array_free(x); mlx_array_free(proj); + x = x2; + + mlx_array xn2 = mlx_array_new(); + mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); + + mlx_array gate = linear(xn2, &l->gate_proj, S); + mlx_array up = linear(xn2, &l->up_proj, S); + mlx_array_free(xn2); + + mlx_array gate_sig = mlx_array_new(); + mlx_sigmoid(&gate_sig, gate, S); + mlx_array silu_gate = mlx_array_new(); + mlx_multiply(&silu_gate, gate, gate_sig, S); + mlx_array_free(gate); mlx_array_free(gate_sig); + + mlx_array mid = mlx_array_new(); + mlx_multiply(&mid, silu_gate, up, S); + mlx_array_free(silu_gate); mlx_array_free(up); + + mlx_array ffn = linear(mid, &l->down_proj, S); + mlx_array_free(mid); + + mlx_array x3 = mlx_array_new(); + mlx_add(&x3, x, ffn, S); + mlx_array_free(x); mlx_array_free(ffn); + x = x3; + } + if (free_mask) mlx_array_free(local_mask); + return x; +} + +pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device( + pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, + int normalize) +{ + if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || + ctx->token_dim <= 0 || !linear_ok(ctx->projection)) + return NULL; + + pplx_mlx_ctx_t *base = ctx->base; + if (base->layer_start != 0 || base->layer_end != base->config.n_layers || + !arr_ok(base->embed_tokens) || !arr_ok(base->norm)) + return NULL; + + const pplx_config_t *c = &base->config; + mlx_stream S = base->stream; + for (int i = 0; i < n_tokens; i++) { + if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { + fprintf(stderr, "mlx late: invalid token id %d at position %d\n", + token_ids[i], i); + return NULL; + } + } + + size_t id_bytes; + if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0) + return NULL; + + int *ids_data = (int *)malloc(id_bytes); + if (!ids_data) return NULL; + memcpy(ids_data, token_ids, id_bytes); + + int ids_shape[] = {1, n_tokens}; + mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32); + free(ids_data); + if (!arr_ok(ids)) return NULL; + + pplx_mlx_late_vectors_t *result = NULL; + mlx_array x = mlx_array_new(); + mlx_array x_normed = (mlx_array){0}; + mlx_array projected = (mlx_array){0}; + mlx_array reshaped = (mlx_array){0}; + + mlx_take_axis(&x, base->embed_tokens, ids, 0, S); + mlx_array_free(ids); + if (!arr_ok(x)) goto cleanup; + + x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, + 0, c->n_layers); + if (!arr_ok(x)) goto cleanup; + + x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S); + mlx_array_free(x); + x = (mlx_array){0}; + if (!arr_ok(x_normed)) goto cleanup; + + projected = linear(x_normed, &ctx->projection, S); + mlx_array_free(x_normed); + x_normed = (mlx_array){0}; + if (!arr_ok(projected)) goto cleanup; + + if (normalize) { + mlx_array sq = mlx_array_new(); + mlx_square(&sq, projected, S); + if (!arr_ok(sq)) { + mlx_array_free(sq); + goto cleanup; + } + + mlx_array norm2 = mlx_array_new(); + mlx_sum_axis(&norm2, sq, 2, true, S); + mlx_array_free(sq); + if (!arr_ok(norm2)) { + mlx_array_free(norm2); + goto cleanup; + } + + mlx_array inv_norm = mlx_array_new(); + mlx_rsqrt(&inv_norm, norm2, S); + mlx_array_free(norm2); + if (!arr_ok(inv_norm)) { + mlx_array_free(inv_norm); + goto cleanup; + } + + mlx_array normalized = mlx_array_new(); + mlx_multiply(&normalized, projected, inv_norm, S); + mlx_array_free(inv_norm); + mlx_array_free(projected); + projected = normalized; + if (!arr_ok(projected)) goto cleanup; + } + + int vec_shape[] = {n_tokens, ctx->token_dim}; + reshaped = mlx_array_new(); + mlx_reshape(&reshaped, projected, vec_shape, 2, S); + if (!arr_ok(reshaped)) goto cleanup; + + result = (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*result)); + if (!result) goto cleanup; + result->owner = ctx; + result->vectors = reshaped; + result->tokens = n_tokens; + result->dim = ctx->token_dim; + reshaped = (mlx_array){0}; + +cleanup: + mlx_array_free(reshaped); + mlx_array_free(projected); + mlx_array_free(x_normed); + mlx_array_free(x); + return result; +} + +void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs) +{ + if (!vecs) return; + mlx_array_free(vecs->vectors); + free(vecs); +} + +int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs) +{ + return vecs ? vecs->tokens : 0; +} + +int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs) +{ + return vecs ? vecs->dim : 0; +} + +int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs, + float *out_vectors) +{ + if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || + vecs->tokens <= 0 || vecs->dim <= 0) + return -1; + + size_t values, bytes; + if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 || + mlx_mul_size(values, sizeof(float), &bytes) != 0) + return -1; + + mlx_stream S = vecs->owner->base->stream; + mlx_array out_f32 = mlx_array_new(); + mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S); + if (!arr_ok(out_f32)) { + mlx_array_free(out_f32); + return -1; + } + + int rc = -1; + mlx_array_eval(out_f32); + const float *data = mlx_array_data_float32(out_f32); + if (data) { + memcpy(out_vectors, data, bytes); + rc = 0; + } + mlx_array_free(out_f32); + return rc; +} + +pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat( + pplx_mlx_late_ctx_t *ctx, + const pplx_mlx_late_vectors_t *const *items, int count) +{ + if (!ctx || !ctx->base || !items || count <= 0) return NULL; + int dim = 0; + int total = 0; + for (int i = 0; i < count; i++) { + const pplx_mlx_late_vectors_t *v = items[i]; + if (!v || v->owner != ctx || !arr_ok(v->vectors) || + v->tokens <= 0 || v->dim <= 0) + return NULL; + if (i == 0) dim = v->dim; + if (v->dim != dim || total > INT_MAX - v->tokens) + return NULL; + total += v->tokens; + } + + mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts)); + if (!parts) return NULL; + for (int i = 0; i < count; i++) + parts[i] = items[i]->vectors; + + mlx_stream S = ctx->base->stream; + mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count); + free(parts); + if (!vec.ctx) return NULL; + + mlx_array joined = mlx_array_new(); + mlx_concatenate_axis(&joined, vec, 0, S); + mlx_vector_array_free(vec); + if (!arr_ok(joined)) { + mlx_array_free(joined); + return NULL; + } + + pplx_mlx_late_vectors_t *out = + (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + if (!out) { + mlx_array_free(joined); + return NULL; + } + out->owner = ctx; + out->vectors = joined; + out->tokens = total; + out->dim = dim; + return out; +} + +pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select( + pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs, + const int *token_indices, int count) +{ + if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || + !arr_ok(vecs->vectors) || !token_indices || count <= 0 || + vecs->tokens <= 0 || vecs->dim <= 0) + return NULL; + + for (int i = 0; i < count; i++) { + if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens) + return NULL; + } + + size_t idx_bytes; + if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0) + return NULL; + int *idx_data = (int *)malloc(idx_bytes); + if (!idx_data) return NULL; + memcpy(idx_data, token_indices, idx_bytes); + + int idx_shape[] = {count}; + mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32); + free(idx_data); + if (!arr_ok(idx)) return NULL; + + mlx_stream S = ctx->base->stream; + mlx_array selected = mlx_array_new(); + mlx_take_axis(&selected, vecs->vectors, idx, 0, S); + mlx_array_free(idx); + if (!arr_ok(selected)) { + mlx_array_free(selected); + return NULL; + } + + pplx_mlx_late_vectors_t *out = + (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + if (!out) { + mlx_array_free(selected); + return NULL; + } + out->owner = ctx; + out->vectors = selected; + out->tokens = count; + out->dim = vecs->dim; + return out; +} + +int pplx_mlx_late_maxsim_batch_device( + pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query, + const pplx_mlx_late_vectors_t *docs, const int *doc_offsets, + int docs_count, float *scores) +{ + if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || + !scores || query->owner != ctx || docs->owner != ctx || + query->dim <= 0 || query->dim != docs->dim || + query->tokens <= 0 || docs->tokens <= 0 || + !arr_ok(query->vectors) || !arr_ok(docs->vectors) || + doc_offsets[0] != 0) + return -1; + + int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths)); + if (!lengths) return -1; + int max_doc_tokens = 0; + for (int i = 0; i < docs_count; i++) { + if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] || + doc_offsets[i + 1] > docs->tokens) { + free(lengths); + return -1; + } + int n = doc_offsets[i + 1] - doc_offsets[i]; + lengths[i] = n; + if (n > max_doc_tokens) max_doc_tokens = n; + } + if (max_doc_tokens <= 0) { + free(lengths); + return -1; + } + + mlx_stream S = ctx->base->stream; + int dim = query->dim; + int rc = -1; + mlx_array *doc_rows = + (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); + if (!doc_rows) { + free(lengths); + return -1; + } + + mlx_vector_array doc_vec = (mlx_vector_array){0}; + mlx_array pad_value = mlx_array_new_float32(0.0f); + mlx_array doc_batch = (mlx_array){0}; + mlx_array query_3d = (mlx_array){0}; + mlx_array query_batched = (mlx_array){0}; + mlx_array docs_t = (mlx_array){0}; + mlx_array sims = (mlx_array){0}; + mlx_array lengths_arr = (mlx_array){0}; + mlx_array positions = (mlx_array){0}; + mlx_array lengths_2d = (mlx_array){0}; + mlx_array positions_2d = (mlx_array){0}; + mlx_array valid = (mlx_array){0}; + mlx_array valid_3d = (mlx_array){0}; + mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX); + mlx_array masked_sims = (mlx_array){0}; + mlx_array maxes = (mlx_array){0}; + mlx_array sums = (mlx_array){0}; + mlx_array sums_f32 = (mlx_array){0}; + + for (int i = 0; i < docs_count; i++) { + int n = lengths[i]; + int start[] = {doc_offsets[i], 0}; + int stop[] = {doc_offsets[i + 1], dim}; + int strides[] = {1, 1}; + + mlx_array doc = mlx_array_new(); + mlx_array padded = (mlx_array){0}; + + mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S); + if (!arr_ok(doc)) goto cleanup; + + if (n < max_doc_tokens) { + int axes[] = {0}; + int low[] = {0}; + int high[] = {max_doc_tokens - n}; + padded = mlx_array_new(); + mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, + "constant", S); + mlx_array_free(doc); + doc = padded; + padded = (mlx_array){0}; + if (!arr_ok(doc)) goto cleanup; + } + + doc_rows[i] = mlx_array_new(); + mlx_expand_dims(&doc_rows[i], doc, 0, S); + mlx_array_free(doc); + if (!arr_ok(doc_rows[i])) goto cleanup; + } + + doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count); + if (!doc_vec.ctx) goto cleanup; + + doc_batch = mlx_array_new(); + mlx_concatenate_axis(&doc_batch, doc_vec, 0, S); + if (!arr_ok(doc_batch)) goto cleanup; + + query_3d = mlx_array_new(); + mlx_expand_dims(&query_3d, query->vectors, 0, S); + if (!arr_ok(query_3d)) goto cleanup; + + int q_shape[] = {docs_count, query->tokens, dim}; + query_batched = mlx_array_new(); + mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S); + if (!arr_ok(query_batched)) goto cleanup; + + int perm[] = {0, 2, 1}; + docs_t = mlx_array_new(); + mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S); + if (!arr_ok(docs_t)) goto cleanup; + + sims = mlx_array_new(); + mlx_matmul(&sims, query_batched, docs_t, S); + if (!arr_ok(sims)) goto cleanup; + + int len_shape[] = {docs_count}; + lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32); + if (!arr_ok(lengths_arr)) goto cleanup; + + positions = mlx_array_new(); + mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S); + if (!arr_ok(positions)) goto cleanup; + + lengths_2d = mlx_array_new(); + mlx_expand_dims(&lengths_2d, lengths_arr, 1, S); + if (!arr_ok(lengths_2d)) goto cleanup; + + positions_2d = mlx_array_new(); + mlx_expand_dims(&positions_2d, positions, 0, S); + if (!arr_ok(positions_2d)) goto cleanup; + + valid = mlx_array_new(); + mlx_less(&valid, positions_2d, lengths_2d, S); + if (!arr_ok(valid)) goto cleanup; + + valid_3d = mlx_array_new(); + mlx_expand_dims(&valid_3d, valid, 1, S); + if (!arr_ok(valid_3d)) goto cleanup; + + masked_sims = mlx_array_new(); + mlx_where(&masked_sims, valid_3d, sims, invalid_value, S); + if (!arr_ok(masked_sims)) goto cleanup; + + maxes = mlx_array_new(); + mlx_max_axis(&maxes, masked_sims, 2, false, S); + if (!arr_ok(maxes)) goto cleanup; + + sums = mlx_array_new(); + mlx_sum_axis(&sums, maxes, 1, false, S); + if (!arr_ok(sums)) goto cleanup; + + sums_f32 = mlx_array_new(); + mlx_astype(&sums_f32, sums, MLX_FLOAT32, S); + if (!arr_ok(sums_f32)) goto cleanup; + + mlx_array_eval(sums_f32); + const float *data = mlx_array_data_float32(sums_f32); + if (!data) goto cleanup; + memcpy(scores, data, (size_t)docs_count * sizeof(float)); + rc = 0; + +cleanup: + mlx_array_free(sums_f32); + mlx_array_free(sums); + mlx_array_free(maxes); + mlx_array_free(masked_sims); + mlx_array_free(invalid_value); + mlx_array_free(valid_3d); + mlx_array_free(valid); + mlx_array_free(positions_2d); + mlx_array_free(lengths_2d); + mlx_array_free(positions); + mlx_array_free(lengths_arr); + mlx_array_free(sims); + mlx_array_free(docs_t); + mlx_array_free(query_batched); + mlx_array_free(query_3d); + mlx_array_free(doc_batch); + if (doc_vec.ctx) mlx_vector_array_free(doc_vec); + mlx_array_free(pad_value); + for (int i = 0; i < docs_count; i++) + mlx_array_free(doc_rows[i]); + free(doc_rows); + free(lengths); + return rc; +} + +int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx, + const int *token_ids, int n_tokens, + int normalize, float *out_vectors) +{ + pplx_mlx_late_vectors_t *vecs = + pplx_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, + normalize); + if (!vecs) return -1; + int rc = pplx_mlx_late_vectors_copy(vecs, out_vectors); + pplx_mlx_late_vectors_free(vecs); + return rc; +} + +static int pplx_mlx_embed_batch_dense(pplx_mlx_ctx_t *ctx, + const pplx_input_t *inputs, int batch, + float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings || + ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) + return -1; + + const pplx_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + + int hidden = c->hidden_size; + int max_seq = 0; + int has_padding = 0; + + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1; + if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + } + if (max_seq <= 0) return -1; + + for (int b = 0; b < batch; b++) { + if (inputs[b].n_tokens != max_seq) has_padding = 1; + } + + size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 || + mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 || + mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 || + mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || + mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || + mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) + return -1; + int *padded_ids = (int *)malloc(ids_bytes); + float *pool_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; + float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; + float *len_data = has_padding ? (float *)malloc(len_bytes) : NULL; + if (!padded_ids || (has_padding && (!pool_mask_data || !attn_mask_data || !len_data))) { + free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + return -1; + } + + for (int b = 0; b < batch; b++) { + if (has_padding) len_data[b] = (float)inputs[b].n_tokens; + for (int t = 0; t < max_seq; t++) { + size_t idx = (size_t)b * max_seq + t; + if (t < inputs[b].n_tokens) { + int id = inputs[b].ids[t]; + if (id < 0 || id >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", + id, b, t); + free(padded_ids); free(pool_mask_data); + free(attn_mask_data); free(len_data); + return -1; + } + padded_ids[idx] = id; + if (has_padding) { + pool_mask_data[idx] = 1.0f; + attn_mask_data[idx] = 0.0f; + } + } else { + padded_ids[idx] = 0; + pool_mask_data[idx] = 0.0f; + attn_mask_data[idx] = -1.0e9f; + } + } + } + + int ids_shape[] = {batch, max_seq}; + int pool_mask_shape[] = {batch, max_seq, 1}; + int attn_mask_shape[] = {batch, 1, 1, max_seq}; + int len_shape[] = {batch, 1}; + + mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + mlx_array pool_mask = has_padding + ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array attn_mask = has_padding + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array lengths = has_padding + ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) + : (mlx_array){0}; + free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + + if (!arr_ok(ids) || (has_padding && (!arr_ok(pool_mask) || + !arr_ok(attn_mask) || !arr_ok(lengths)))) { + mlx_array_free(ids); mlx_array_free(pool_mask); + if (has_padding) mlx_array_free(attn_mask); + mlx_array_free(lengths); + return -1; + } + if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) { + mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths); + return -1; + } + } + + /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */ + mlx_array x = mlx_array_new(); + mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); + mlx_array_free(ids); + + /* 2. Transformer layers. */ + x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, + 0, c->n_layers); + + /* 3. Final RMSNorm. */ + mlx_array x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); + mlx_array_free(x); + + mlx_array emb = mlx_array_new(); + if (has_padding) { + /* 4. Masked mean pool over T. */ + mlx_array masked = mlx_array_new(); + mlx_multiply(&masked, x_normed, pool_mask, S); + mlx_array_free(x_normed); + mlx_array_free(pool_mask); + + mlx_array emb_sum = mlx_array_new(); + mlx_sum_axis(&emb_sum, masked, 1, false, S); + mlx_array_free(masked); + + mlx_divide(&emb, emb_sum, lengths, S); + mlx_array_free(emb_sum); mlx_array_free(lengths); + mlx_array_free(attn_mask); + } else { + /* Equal-length batches do not need masks or a separate divide. */ + mlx_mean_axis(&emb, x_normed, 1, false, S); + mlx_array_free(x_normed); + } + + /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are + * intentionally unnormalized; callers should use cosine similarity. */ + mlx_array emb_f32 = mlx_array_new(); + mlx_astype(&emb_f32, emb, MLX_FLOAT32, S); + mlx_array_free(emb); + + mlx_array_eval(emb_f32); + const float *data = mlx_array_data_float32(emb_f32); + int rc = -1; + if (data) { + memcpy(out_embeddings, data, out_bytes); + rc = 0; + } + mlx_array_free(emb_f32); + return rc; +} + +typedef struct { + int index; + const pplx_input_t *input; +} mlx_batch_item_t; + +static int mlx_batch_item_cmp(const void *a, const void *b) +{ + const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a; + const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b; + int da = ia->input->n_tokens; + int db = ib->input->n_tokens; + return (da > db) - (da < db); +} + +static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) +{ + if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5) + return 1; + return padded * 5 > tokens * 11; +} + +static int mlx_batch_should_split(const pplx_input_t *inputs, int batch, + int *out_total_tokens, int *out_max_seq) +{ + int max_seq = 0; + int total_tokens = 0; + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0; + if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0; + total_tokens += inputs[b].n_tokens; + } + if (out_total_tokens) *out_total_tokens = total_tokens; + if (out_max_seq) *out_max_seq = max_seq; + if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0; + + size_t padded; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) + return 0; + + return mlx_padding_exceeds_limit(padded, (size_t)total_tokens); +} + +static int mlx_should_add_to_group(const pplx_mlx_ctx_t *ctx, + int group_count, int group_max, + int next_tokens) +{ + if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0) + return 1; + + int next_count = group_count + 1; + int next_max = next_tokens > group_max ? next_tokens : group_max; + size_t merged, split_left, split_total; + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &merged) != 0 || + mlx_mul_size((size_t)group_count, (size_t)group_max, &split_left) != 0) + return 0; + /* Treat one extra MLX launch as roughly 256 padded token rows for 4B. + * This preserves small mixed groups while splitting larger padded tails. */ + if (split_left > SIZE_MAX - (size_t)next_tokens || + split_left + (size_t)next_tokens > SIZE_MAX - 256) + return 0; + split_total = split_left + (size_t)next_tokens + 256; + return merged <= split_total; +} + +int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; + + if (!mlx_batch_should_split(inputs, batch, NULL, NULL)) + return pplx_mlx_embed_batch_dense(ctx, inputs, batch, out_embeddings); + + const int hidden = ctx->config.hidden_size; + mlx_batch_item_t *items = (mlx_batch_item_t *)malloc( + (size_t)batch * sizeof(*items)); + pplx_input_t *group_inputs = (pplx_input_t *)malloc( + (size_t)batch * sizeof(*group_inputs)); + size_t group_values, group_bytes; + if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 || + mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { + free(items); free(group_inputs); + return -1; + } + float *group_out = (float *)malloc(group_bytes); + if (!items || !group_inputs || !group_out) { + free(items); free(group_inputs); free(group_out); + return -1; + } + for (int i = 0; i < batch; i++) { + items[i].index = i; + items[i].input = &inputs[i]; + } + qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp); + + int rc = 0; + for (int start = 0; start < batch && rc == 0; ) { + int group_max = 0; + int group_tokens = 0; + int end = start; + while (end < batch) { + int next_tokens = items[end].input->n_tokens; + int next_count = end - start + 1; + int next_max = next_tokens > group_max ? next_tokens : group_max; + int next_total = group_tokens + next_tokens; + if (end > start) { + size_t next_padded; + if (mlx_mul_size((size_t)next_count, (size_t)next_max, + &next_padded) != 0 || + mlx_padding_exceeds_limit(next_padded, + (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, + group_max, next_tokens)) + break; + } + group_max = next_max; + group_tokens = next_total; + end++; + } + + int group_count = end - start; + for (int i = 0; i < group_count; i++) + group_inputs[i] = *items[start + i].input; + rc = pplx_mlx_embed_batch_dense(ctx, group_inputs, group_count, + group_out); + if (rc != 0) break; + for (int i = 0; i < group_count; i++) { + int dst = items[start + i].index; + memcpy(out_embeddings + (size_t)dst * hidden, + group_out + (size_t)i * hidden, + (size_t)hidden * sizeof(float)); + } + start = end; + } + + free(items); free(group_inputs); free(group_out); + return rc; +} + +int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding) +{ + pplx_input_t input = { token_ids, n_tokens }; + return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding); +} + +int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, + const pplx_input_t *inputs, int batch, + const float *input_states, + int layer_start, int layer_end, + int apply_final_norm, + float *out_states) +{ + if (!ctx || !inputs || batch <= 0 || !out_states || + layer_start < 0 || layer_start > layer_end || + layer_end > ctx->config.n_layers || + layer_start < ctx->layer_start || layer_end > ctx->layer_end || + (layer_start == 0 && input_states) || + (layer_start != 0 && !input_states) || + (layer_start == 0 && !arr_ok(ctx->embed_tokens)) || + (apply_final_norm && + (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm)))) + return -1; + + const pplx_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + int hidden = c->hidden_size; + int max_seq = 0; + int total_seq = 0; + int has_padding = 0; + for (int b = 0; b < batch; b++) { + if ((layer_start == 0 && !inputs[b].ids) || inputs[b].n_tokens <= 0 || + total_seq > INT_MAX - inputs[b].n_tokens) + return -1; + total_seq += inputs[b].n_tokens; + if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + } + for (int b = 0; b < batch; b++) { + if (inputs[b].n_tokens != max_seq) has_padding = 1; + } + + size_t rows, ids_bytes, mask_bytes, state_bytes, row_bytes; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || + mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || + mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || + mlx_mul_size((size_t)hidden, sizeof(float), &row_bytes) != 0) + return -1; + if (rows == 0 || row_bytes == 0 || (has_padding && mask_bytes == 0)) + return -1; + if (layer_start != 0 && + mlx_mul_size(rows, (size_t)hidden, &state_bytes) != 0) + return -1; + if (layer_start == 0) + state_bytes = 0; + + int *padded_ids = layer_start == 0 + ? (int *)malloc(ids_bytes) : NULL; + float *padded_states = layer_start != 0 + ? (float *)calloc(1, state_bytes) : NULL; + float *attn_mask_data = has_padding + ? (float *)malloc(mask_bytes) : NULL; + if ((layer_start == 0 && !padded_ids) || + (layer_start != 0 && !padded_states) || + (has_padding && !attn_mask_data)) { + free(padded_ids); free(padded_states); free(attn_mask_data); + return -1; + } + + int packed_offset = 0; + for (int b = 0; b < batch; b++) { + for (int t = 0; t < max_seq; t++) { + size_t dense_row = (size_t)b * max_seq + t; + if (t < inputs[b].n_tokens) { + if (layer_start == 0) { + int id = inputs[b].ids[t]; + if (id < 0 || id >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", + id, b, t); + free(padded_ids); free(padded_states); + free(attn_mask_data); + return -1; + } + padded_ids[dense_row] = id; + } else { + memcpy(padded_states + dense_row * hidden, + input_states + (size_t)(packed_offset + t) * hidden, + row_bytes); + } + if (has_padding) attn_mask_data[dense_row] = 0.0f; + } else { + if (layer_start == 0) padded_ids[dense_row] = 0; + if (has_padding) attn_mask_data[dense_row] = -1.0e9f; + } + } + packed_offset += inputs[b].n_tokens; + } + + int attn_mask_shape[] = {batch, 1, 1, max_seq}; + mlx_array attn_mask = has_padding + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + free(attn_mask_data); + if (has_padding && !arr_ok(attn_mask)) { + free(padded_ids); free(padded_states); + return -1; + } + if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) { + free(padded_ids); free(padded_states); + return -1; + } + } + + mlx_array x; + if (layer_start == 0) { + int ids_shape[] = {batch, max_seq}; + mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + free(padded_ids); + if (!arr_ok(ids)) { + if (has_padding) mlx_array_free(attn_mask); + return -1; + } + x = mlx_array_new(); + mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); + mlx_array_free(ids); + } else { + int states_shape[] = {batch, max_seq, hidden}; + mlx_array states = mlx_array_new_data(padded_states, states_shape, 3, + MLX_FLOAT32); + free(padded_states); + if (!arr_ok(states)) { + if (has_padding) mlx_array_free(attn_mask); + return -1; + } + if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + x = mlx_array_new(); + mlx_astype(&x, states, MLX_BFLOAT16, S); + mlx_array_free(states); + } else { + x = states; + } + } + + x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, + layer_start, layer_end); + if (has_padding) mlx_array_free(attn_mask); + + if (apply_final_norm) { + mlx_array x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); + mlx_array_free(x); + x = x_normed; + } + + mlx_array x_f32 = mlx_array_new(); + mlx_astype(&x_f32, x, MLX_FLOAT32, S); + mlx_array_free(x); + mlx_array_eval(x_f32); + + const float *data = mlx_array_data_float32(x_f32); + if (!data) { + mlx_array_free(x_f32); + return -1; + } + packed_offset = 0; + for (int b = 0; b < batch; b++) { + memcpy(out_states + (size_t)packed_offset * hidden, + data + (size_t)b * max_seq * hidden, + (size_t)inputs[b].n_tokens * row_bytes); + packed_offset += inputs[b].n_tokens; + } + mlx_array_free(x_f32); + return 0; +} + +typedef struct { + int index; + int span_offset; + const pplx_context_input_t *input; +} mlx_context_batch_item_t; + +static int mlx_context_batch_item_cmp(const void *a, const void *b) +{ + const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a; + const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b; + int da = ia->input->input.n_tokens; + int db = ib->input->input.n_tokens; + return (da > db) - (da < db); +} + +static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) +{ + if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4) + return 1; + return padded * 4 > tokens * 5; +} + +static int mlx_context_batch_should_split(const pplx_context_input_t *inputs, + int batch, int *out_total_spans) +{ + int max_seq = 0; + int total_tokens = 0; + int total_spans = 0; + for (int b = 0; b < batch; b++) { + int n_tokens = inputs[b].input.n_tokens; + if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || + inputs[b].n_spans <= 0) + return 0; + if (n_tokens > max_seq) max_seq = n_tokens; + if (total_tokens > INT_MAX - n_tokens || + total_spans > INT_MAX - inputs[b].n_spans) + return 0; + total_tokens += n_tokens; + total_spans += inputs[b].n_spans; + } + if (out_total_spans) *out_total_spans = total_spans; + if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0; + + size_t padded; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) + return 0; + return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens); +} + +static int pplx_mlx_embed_spans_batch_dense(pplx_mlx_ctx_t *ctx, + const pplx_context_input_t *inputs, + int batch, float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings || + ctx->layer_start != 0 || + ctx->layer_end != ctx->config.n_layers || + !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) + return -1; + + const pplx_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + int hidden = c->hidden_size; + int max_seq = 0; + int total_spans = 0; + int has_padding = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || + input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans) + return -1; + total_spans += input->n_spans; + if (n_tokens > max_seq) max_seq = n_tokens; + for (int t = 0; t < n_tokens; t++) { + if (input->input.ids[t] < 0 || + input->input.ids[t] >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", + input->input.ids[t], b, t); + return -1; + } + } + for (int s = 0; s < input->n_spans; s++) { + if (input->spans[s].start < 0 || + input->spans[s].start > n_tokens || + input->spans[s].n_tokens <= 0 || + input->spans[s].n_tokens > n_tokens - input->spans[s].start) + return -1; + } + } + for (int b = 0; b < batch; b++) + if (inputs[b].input.n_tokens != max_seq) has_padding = 1; + + size_t rows, ids_bytes, mask_bytes, index_bytes; + size_t span_float_bytes, out_values, out_bytes; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || + mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || + mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || + mlx_mul_size((size_t)total_spans, sizeof(int), &index_bytes) != 0 || + mlx_mul_size((size_t)total_spans, sizeof(float), &span_float_bytes) != 0 || + mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 || + mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) + return -1; + if (rows > INT_MAX) return -1; + int *padded_ids = (int *)malloc(ids_bytes); + float *attn_mask_data = has_padding + ? (float *)malloc(mask_bytes) : NULL; + if (!padded_ids || (has_padding && !attn_mask_data)) { + free(padded_ids); + free(attn_mask_data); + return -1; + } + for (int b = 0; b < batch; b++) { + const pplx_input_t *input = &inputs[b].input; + for (int t = 0; t < max_seq; t++) { + size_t row = (size_t)b * max_seq + t; + if (t < input->n_tokens) { + padded_ids[row] = input->ids[t]; + if (has_padding) attn_mask_data[row] = 0.0f; + } else { + padded_ids[row] = 0; + if (has_padding) attn_mask_data[row] = -1.0e9f; + } + } + } + + mlx_array ids = (mlx_array){0}; + mlx_array attn_mask = (mlx_array){0}; + mlx_array x = (mlx_array){0}; + mlx_array x_normed = (mlx_array){0}; + mlx_array prefix = (mlx_array){0}; + mlx_array flat_prefix = (mlx_array){0}; + mlx_array end_idx = (mlx_array){0}; + mlx_array start_idx = (mlx_array){0}; + mlx_array lengths = (mlx_array){0}; + mlx_array start_mask = (mlx_array){0}; + mlx_array end_values = (mlx_array){0}; + mlx_array start_values = (mlx_array){0}; + mlx_array masked_start = (mlx_array){0}; + mlx_array span_sums = (mlx_array){0}; + mlx_array embeddings = (mlx_array){0}; + mlx_array embeddings_f32 = (mlx_array){0}; + mlx_array *pooled = NULL; + mlx_vector_array pooled_vec = (mlx_vector_array){0}; + int *end_idx_data = NULL; + int *start_idx_data = NULL; + float *lengths_data = NULL; + float *start_mask_data = NULL; + int rc = -1; + + int ids_shape[] = {batch, max_seq}; + int attn_mask_shape[] = {batch, 1, 1, max_seq}; + ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + attn_mask = has_padding + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + free(padded_ids); + free(attn_mask_data); + if (!arr_ok(ids)) goto cleanup; + if (has_padding && !arr_ok(attn_mask)) goto cleanup; + if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) goto cleanup; + } + + x = mlx_array_new(); + mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); + mlx_array_free(ids); + ids = (mlx_array){0}; + if (!arr_ok(x)) goto cleanup; + + x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, + 0, c->n_layers); + if (has_padding) { + mlx_array_free(attn_mask); + attn_mask = (mlx_array){0}; + } + if (!arr_ok(x)) goto cleanup; + + x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); + mlx_array_free(x); + x = (mlx_array){0}; + if (!arr_ok(x_normed)) goto cleanup; + + if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled)); + if (!pooled) goto cleanup; + + int pooled_count = 0; + for (int b = 0; b < batch; b++) { + for (int s = 0; s < inputs[b].n_spans; s++) { + const pplx_span_t *span = &inputs[b].spans[s]; + int start[] = {b, span->start, 0}; + int stop[] = {b + 1, span->start + span->n_tokens, hidden}; + int strides[] = {1, 1, 1}; + mlx_array slice = mlx_array_new(); + mlx_slice(&slice, x_normed, start, 3, stop, 3, strides, 3, S); + if (!arr_ok(slice)) { + mlx_array_free(slice); + goto cleanup; + } + + pooled[pooled_count] = mlx_array_new(); + mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S); + mlx_array_free(slice); + if (!arr_ok(pooled[pooled_count])) goto cleanup; + pooled_count++; + } + } + + pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans); + if (!pooled_vec.ctx) goto cleanup; + embeddings = mlx_array_new(); + mlx_concatenate_axis(&embeddings, pooled_vec, 0, S); + if (!arr_ok(embeddings)) goto cleanup; + } else { + end_idx_data = (int *)malloc(index_bytes); + start_idx_data = (int *)malloc(index_bytes); + lengths_data = (float *)malloc(span_float_bytes); + start_mask_data = (float *)malloc(span_float_bytes); + if (!end_idx_data || !start_idx_data || !lengths_data || + !start_mask_data) + goto cleanup; + + int span_count = 0; + for (int b = 0; b < batch; b++) { + for (int s = 0; s < inputs[b].n_spans; s++) { + const pplx_span_t *span = &inputs[b].spans[s]; + int base = b * max_seq; + int start = span->start; + int stop = span->start + span->n_tokens; + end_idx_data[span_count] = base + stop - 1; + start_idx_data[span_count] = start > 0 ? base + start - 1 : base; + lengths_data[span_count] = (float)span->n_tokens; + start_mask_data[span_count] = start > 0 ? 1.0f : 0.0f; + span_count++; + } + } + + int index_shape[] = {total_spans}; + int scale_shape[] = {total_spans, 1}; + end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32); + start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, + MLX_INT32); + lengths = mlx_array_new_data(lengths_data, scale_shape, 2, + MLX_FLOAT32); + start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, + MLX_FLOAT32); + free(end_idx_data); end_idx_data = NULL; + free(start_idx_data); start_idx_data = NULL; + free(lengths_data); lengths_data = NULL; + free(start_mask_data); start_mask_data = NULL; + if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || + !arr_ok(start_mask)) + goto cleanup; + + prefix = mlx_array_new(); + mlx_cumsum(&prefix, x_normed, 1, false, true, S); + mlx_array_free(x_normed); + x_normed = (mlx_array){0}; + if (!arr_ok(prefix)) goto cleanup; + + int flat_shape[] = {(int)rows, hidden}; + flat_prefix = mlx_array_new(); + mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S); + mlx_array_free(prefix); + prefix = (mlx_array){0}; + if (!arr_ok(flat_prefix)) goto cleanup; + + end_values = mlx_array_new(); + mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S); + if (!arr_ok(end_values)) goto cleanup; + + start_values = mlx_array_new(); + mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S); + if (!arr_ok(start_values)) goto cleanup; + + masked_start = mlx_array_new(); + mlx_multiply(&masked_start, start_values, start_mask, S); + if (!arr_ok(masked_start)) goto cleanup; + + span_sums = mlx_array_new(); + mlx_subtract(&span_sums, end_values, masked_start, S); + if (!arr_ok(span_sums)) goto cleanup; + + embeddings = mlx_array_new(); + mlx_divide(&embeddings, span_sums, lengths, S); + if (!arr_ok(embeddings)) goto cleanup; + } + + embeddings_f32 = mlx_array_new(); + mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S); + if (!arr_ok(embeddings_f32)) goto cleanup; + + mlx_array_eval(embeddings_f32); + const float *data = mlx_array_data_float32(embeddings_f32); + if (data) { + memcpy(out_embeddings, data, out_bytes); + rc = 0; + } + +cleanup: + free(start_mask_data); + free(lengths_data); + free(start_idx_data); + free(end_idx_data); + mlx_array_free(embeddings_f32); + mlx_array_free(embeddings); + if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec); + if (pooled) { + for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]); + free(pooled); + } + mlx_array_free(span_sums); + mlx_array_free(masked_start); + mlx_array_free(start_values); + mlx_array_free(end_values); + mlx_array_free(start_mask); + mlx_array_free(lengths); + mlx_array_free(start_idx); + mlx_array_free(end_idx); + mlx_array_free(flat_prefix); + mlx_array_free(prefix); + mlx_array_free(x_normed); + mlx_array_free(x); + mlx_array_free(attn_mask); + mlx_array_free(ids); + return rc; +} + +int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings) +{ + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; + + int total_spans = 0; + if (!mlx_context_batch_should_split(inputs, batch, &total_spans)) + return pplx_mlx_embed_spans_batch_dense(ctx, inputs, batch, + out_embeddings); + + const int hidden = ctx->config.hidden_size; + mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc( + (size_t)batch * sizeof(*items)); + pplx_context_input_t *group_inputs = (pplx_context_input_t *)malloc( + (size_t)batch * sizeof(*group_inputs)); + size_t group_values, group_bytes; + if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 || + mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { + free(items); free(group_inputs); + return -1; + } + float *group_out = (float *)malloc(group_bytes); + if (!items || !group_inputs || !group_out) { + free(items); free(group_inputs); free(group_out); + return -1; + } + + int span_offset = 0; + for (int i = 0; i < batch; i++) { + items[i].index = i; + items[i].span_offset = span_offset; + items[i].input = &inputs[i]; + span_offset += inputs[i].n_spans; + } + qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp); + + int rc = 0; + for (int start = 0; start < batch && rc == 0; ) { + int group_max = 0; + int group_tokens = 0; + int end = start; + while (end < batch) { + int next_tokens = items[end].input->input.n_tokens; + int next_count = end - start + 1; + int next_max = next_tokens > group_max ? next_tokens : group_max; + int next_total = group_tokens + next_tokens; + if (end > start) { + size_t next_padded; + if (mlx_mul_size((size_t)next_count, (size_t)next_max, + &next_padded) != 0 || + mlx_context_padding_exceeds_limit(next_padded, + (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, + group_max, next_tokens)) + break; + } + group_max = next_max; + group_tokens = next_total; + end++; + } + + int group_count = end - start; + for (int i = 0; i < group_count; i++) + group_inputs[i] = *items[start + i].input; + rc = pplx_mlx_embed_spans_batch_dense(ctx, group_inputs, + group_count, group_out); + if (rc != 0) break; + int src_span = 0; + for (int i = 0; i < group_count; i++) { + int n_spans = group_inputs[i].n_spans; + int dst_span = items[start + i].span_offset; + memcpy(out_embeddings + (size_t)dst_span * hidden, + group_out + (size_t)src_span * hidden, + (size_t)n_spans * hidden * sizeof(float)); + src_span += n_spans; + } + start = end; + } + + free(items); free(group_inputs); free(group_out); + return rc; +} + +int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, const pplx_span_t *spans, + int n_spans, float *out_embeddings) +{ + pplx_context_input_t input = { + .input = { token_ids, n_tokens }, + .spans = spans, + .n_spans = n_spans, + }; + return pplx_mlx_embed_spans_batch(ctx, &input, 1, out_embeddings); +} + +float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) +{ + if (!ctx || !token_ids || n_tokens <= 0) return NULL; + + int hidden = ctx->config.hidden_size; + size_t out_bytes; + if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0) + return NULL; + float *out = (float *)malloc(out_bytes); + if (!out) return NULL; + + if (pplx_mlx_embed_into(ctx, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; + } + return out; +} diff --git a/embed_mlx.h b/embed_mlx.h new file mode 100644 index 0000000..f2ad136 --- /dev/null +++ b/embed_mlx.h @@ -0,0 +1,149 @@ +/* + * pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference + * + * Uses Apple's mlx-c to run the transformer on Metal. + */ + +#ifndef PPLX_EMBED_MLX_H +#define PPLX_EMBED_MLX_H + +#include +#include +#include "pplx_embed.h" + +/* Opaque MLX model context */ +typedef struct pplx_mlx_ctx pplx_mlx_ctx_t; +typedef struct pplx_mlx_late_ctx pplx_mlx_late_ctx_t; +typedef struct pplx_mlx_late_vectors pplx_mlx_late_vectors_t; + +typedef struct { + int quantize_bits; /* 0 disables; currently 8 */ + int quantize_group_size; /* default 64 when quantize_bits is set */ +} pplx_mlx_options_t; + +/* + * Load model into MLX arrays from safetensors + config.json. + * Returns NULL on error. + */ +pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); +pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, + const pplx_mlx_options_t *opts); + +/* + * Load one contiguous transformer layer range for distributed execution. + * Uses the same half-open range and endpoint ownership rules as + * pplx_model_load_slice(). + */ +pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, + int layer_start, int layer_end); +pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( + const char *model_dir, int layer_start, int layer_end, + const pplx_mlx_options_t *opts); + +void pplx_mlx_free(pplx_mlx_ctx_t *ctx); + +/* + * Late-interaction MLX path. This is separate from pooled embeddings because + * late snapshots produce one projected token vector per input token for MaxSim + * scoring, not one pooled document vector. + */ +pplx_mlx_late_ctx_t *pplx_mlx_late_load(const char *model_dir); +pplx_mlx_late_ctx_t *pplx_mlx_late_load_with_options( + const char *model_dir, const pplx_mlx_options_t *opts); +void pplx_mlx_late_free(pplx_mlx_late_ctx_t *ctx); + +const pplx_config_t *pplx_mlx_late_config(const pplx_mlx_late_ctx_t *ctx); +int pplx_mlx_late_token_dim(const pplx_mlx_late_ctx_t *ctx); + +/* + * Encode token_ids into out_vectors[n_tokens, token_dim]. + * + * If normalize is non-zero, every token vector is L2-normalized on device + * before copying back to CPU memory. + */ +int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx, + const int *token_ids, int n_tokens, + int normalize, float *out_vectors); + +/* + * Device-resident late-interaction token vectors. + * + * The model context owns the MLX stream; the returned vector handle must not + * outlive ctx. Copying is optional: MaxSim can consume these handles directly + * and copy back only final candidate scores. + */ +pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device( + pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, + int normalize); +void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs); +int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs); +int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs); +int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs, + float *out_vectors); +pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat( + pplx_mlx_late_ctx_t *ctx, + const pplx_mlx_late_vectors_t *const *items, int count); +pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select( + pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs, + const int *token_indices, int count); +int pplx_mlx_late_maxsim_batch_device( + pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query, + const pplx_mlx_late_vectors_t *docs, const int *doc_offsets, + int docs_count, float *scores); + +/* + * Compute embedding for token_ids[0..n_tokens-1]. + * Returns malloc'd float[hidden_size] (caller frees). NULL on error. + */ +float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens); + +/* + * Compute one embedding into caller-provided out[hidden_size]. + * Returns 0 on success, -1 on error. + */ +int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, float *out_embedding); + +/* + * Compute a true padded dense batch on MLX. + * + * out_embeddings is caller-provided [batch, hidden_size]. + * Padded tokens are masked out of attention keys and mean pooling. + * Returns 0 on success, -1 on error. + */ +int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, + int batch, float *out_embeddings); + +/* + * MLX equivalent of pplx_model_forward_slice_batch(). + * + * Hidden states cross the API boundary as packed float32 rows even though MLX + * executes a padded dense batch internally. + */ +int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, + const pplx_input_t *inputs, int batch, + const float *input_states, + int layer_start, int layer_end, + int apply_final_norm, + float *out_states); + +/* + * Run one contextual sequence and pool selected token spans. + * Returns 0 on success, -1 on error. + */ +int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, const pplx_span_t *spans, + int n_spans, float *out_embeddings); + +/* + * Run a padded dense contextual document batch and pool every selected span. + * out_embeddings contains spans in document order. + */ +int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings); + +/* Get the config (so main.c can read hidden_size etc.) */ +const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx); + +#endif /* PPLX_EMBED_MLX_H */ diff --git a/embed_server.c b/embed_server.c new file mode 100644 index 0000000..d831062 --- /dev/null +++ b/embed_server.c @@ -0,0 +1,2431 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "pplx_server.h" +#include "pplx_embed.h" +#include "qwen_safetensors.h" +#include "qwen_tokenizer.h" + +#ifdef USE_MLX +#include "pplx_embed_mlx.h" +#endif +#ifdef USE_CUDA +#include "pplx_embed_cuda.h" +#endif + +#include "deps/ae/ae.h" +#include "deps/ae/anet.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#endif + +#include + +#define PPLX_HTTP_IO_BUF 8192 +#define PPLX_HTTP_MAX_HEADER (64u * 1024u) +#define PPLX_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define PPLX_HTTP_MAX_PATH 255u +#define PPLX_HTTP_BACKLOG 128 +#define PPLX_HTTP_SETSIZE 10240 +#define PPLX_HTTP_CLIENT_TIMEOUT_MS 30000 +#define PPLX_HTTP_SWEEP_MS 1000 + +#define PPLX_API_MAX_STANDARD_INPUTS 512 +#define PPLX_API_MAX_CONTEXT_DOCS 512 +#define PPLX_API_MAX_CONTEXT_CHUNKS 16000 +#define PPLX_API_MAX_ITEM_TOKENS 32768 +#define PPLX_API_MAX_TOTAL_TOKENS 120000 +#define PPLX_SERVER_DEFAULT_BATCH_SIZE 8 +#define PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 +#define PPLX_SERVER_BATCH_WAIT_US 1000 +#define PPLX_SERVER_MICROBATCH_MAX_JOBS 128 +#define PPLX_MLX_MEMORY_BUDGET_PERCENT 90 +#define PPLX_MLX_RESIDENT_MULTIPLIER 2 + +typedef struct { + char *ptr; + size_t len; + size_t cap; +} sbuf; + +static void die_oom(void) { + fprintf(stderr, "pplx-serve: out of memory\n"); + exit(1); +} + +static void *xmalloc(size_t n) { + void *p = malloc(n ? n : 1); + if (!p) die_oom(); + return p; +} + +static void *xcalloc(size_t n, size_t sz) { + void *p = calloc(n ? n : 1, sz ? sz : 1); + if (!p) die_oom(); + return p; +} + +static void *xrealloc(void *p, size_t n) { + p = realloc(p, n ? n : 1); + if (!p) die_oom(); + return p; +} + +static char *xstrdup(const char *s) { + size_t n = strlen(s); + char *p = xmalloc(n + 1); + memcpy(p, s, n + 1); + return p; +} + +static char *xstrndup(const char *s, size_t n) { + char *p = xmalloc(n + 1); + memcpy(p, s, n); + p[n] = '\0'; + return p; +} + +static uint64_t mstime(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) + return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; + struct timeval tv; + gettimeofday(&tv, NULL); + return (uint64_t)tv.tv_sec * 1000u + (uint64_t)tv.tv_usec / 1000u; +} + +static uint64_t nstime(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) + return (uint64_t)ts.tv_sec * 1000000000u + (uint64_t)ts.tv_nsec; + struct timeval tv; + gettimeofday(&tv, NULL); + return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u; +} + +static double ns_to_ms(uint64_t ns) { + return (double)ns / 1000000.0; +} + +static void server_log(const char *fmt, ...) { + char ts[32]; + time_t now = time(NULL); + struct tm tmv; + localtime_r(&now, &tmv); + strftime(ts, sizeof(ts), "%m%d %H:%M:%S", &tmv); + fprintf(stderr, "%s ", ts); + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); +} + +static void sbuf_reserve(sbuf *b, size_t add) { + if (add > SIZE_MAX - b->len - 1) die_oom(); + size_t need = b->len + add + 1; + if (need <= b->cap) return; + size_t cap = b->cap ? b->cap * 2 : 256; + while (cap < need) { + if (cap > SIZE_MAX / 2) die_oom(); + cap *= 2; + } + b->ptr = xrealloc(b->ptr, cap); + b->cap = cap; +} + +static void sbuf_append(sbuf *b, const void *p, size_t n) { + sbuf_reserve(b, n); + memcpy(b->ptr + b->len, p, n); + b->len += n; + b->ptr[b->len] = '\0'; +} + +static void sbuf_putc(sbuf *b, char c) { sbuf_append(b, &c, 1); } +static void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); } + +static void sbuf_clear(sbuf *b) { + b->len = 0; + if (b->ptr) b->ptr[0] = '\0'; +} + +static void sbuf_printf(sbuf *b, const char *fmt, ...) { + va_list ap, ap2; + va_start(ap, fmt); + va_copy(ap2, ap); + int n = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + if (n < 0) die_oom(); + sbuf_reserve(b, (size_t)n); + int n2 = vsnprintf(b->ptr + b->len, b->cap - b->len, fmt, ap2); + va_end(ap2); + if (n2 < 0 || n2 != n) die_oom(); + b->len += (size_t)n; +} + +static void sbuf_free(sbuf *b) { + free(b->ptr); + memset(b, 0, sizeof(*b)); +} + +typedef enum { + MODEL_STD_06, + MODEL_STD_4, + MODEL_CTX_06, + MODEL_CTX_4, + MODEL_UNKNOWN +} model_slot; + +typedef struct { + const char *id; + int is_contextual; + int dim; + double price_per_mtok; +} model_info; + +static const model_info k_models[] = { + {"pplx-embed-v1-0.6b", 0, 1024, 0.004}, + {"pplx-embed-v1-4b", 0, 2560, 0.030}, + {"pplx-embed-context-v1-0.6b", 1, 1024, 0.008}, + {"pplx-embed-context-v1-4b", 1, 2560, 0.050}, +}; + +static model_slot model_slot_for_id(const char *id) { + if (!id) return MODEL_UNKNOWN; + for (int i = 0; i < 4; i++) { + if (!strcmp(id, k_models[i].id)) return (model_slot)i; + } + return MODEL_UNKNOWN; +} + +typedef struct { + const model_info *info; + char *path; + qwen_tokenizer_t *tok; + qwen_tokenizer_workspace_t *tok_ws; + int context_separator_id; + pplx_model_t *cpu_model; + pplx_workspace_t *cpu_ws; +#ifdef USE_MLX + pplx_mlx_ctx_t *mlx_ctx; +#endif +#ifdef USE_CUDA + pplx_cuda_ctx_t *cuda_ctx; +#endif +} loaded_model; + +typedef struct client client; +typedef struct job job; + +typedef struct { + aeEventLoop *loop; + int listen_fd; + int completion_pipe[2]; + int signal_pipe[2]; + const char *host; + int port; + int batch_size; + int max_batch_tokens; + int batch_wait_us; + int enable_cors; + char *api_key; + loaded_model models[4]; + int use_mlx; + int use_cuda; + int mlx_quantize_bits; + int mlx_quantize_group_size; + + pthread_t worker; + pthread_mutex_t mu; + pthread_cond_t cv; + int stopping; + int worker_ready; + int worker_init_rc; + job *job_head; + job *job_tail; + job *done_head; + job *done_tail; + + client *clients; + int n_clients; +} http_server; + +typedef struct { + char method[8]; + char path[PPLX_HTTP_MAX_PATH + 1]; + char version[16]; + char *body; + size_t body_len; + char *auth; +} http_request; + +struct client { + http_server *srv; + int fd; + int refcount; + int cancelled; + int linked; + uint64_t last_active_ms; + sbuf in; + sbuf out; + size_t sent; + int header_done; + size_t header_len; + size_t content_length; + int close_after_write; + http_request req; + client *prev; + client *next; +}; + +struct job { + http_server *srv; + client *c; + char method[8]; + char path[PPLX_HTTP_MAX_PATH + 1]; + char *body; + size_t body_len; + char *auth; + int status; + char *content_type; + char *extra_headers; + char *response; + size_t response_len; + uint64_t created_ns; + uint64_t started_ns; + uint64_t parse_ns; + uint64_t tokenize_ns; + uint64_t infer_ns; + uint64_t encode_ns; + job *next; +}; + +static int queue_write(client *c); +static void dispatch_request(client *c); +static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask); + +static volatile sig_atomic_t g_stop_requested = 0; +static volatile sig_atomic_t g_signal_wfd = -1; + +static void stop_signal_handler(int sig) { + (void)sig; + if (g_stop_requested) _exit(130); + int saved = errno; + g_stop_requested = 1; + int fd = (int)g_signal_wfd; + if (fd >= 0) { + const unsigned char byte = 1; + (void)write(fd, &byte, 1); + } + errno = saved; +} + +static int set_nonblock(int fd) { + int flags = fcntl(fd, F_GETFL, 0); + if (flags < 0) return -1; + return fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +static void http_request_free(http_request *r) { + free(r->body); + free(r->auth); + memset(r, 0, sizeof(*r)); +} + +static void client_free(client *c) { + if (!c) return; + http_request_free(&c->req); + sbuf_free(&c->in); + sbuf_free(&c->out); + free(c); +} + +static void client_incref(client *c) { + pthread_mutex_lock(&c->srv->mu); + c->refcount++; + pthread_mutex_unlock(&c->srv->mu); +} + +static void client_decref(client *c) { + int free_now = 0; + pthread_mutex_lock(&c->srv->mu); + if (--c->refcount == 0) free_now = 1; + pthread_mutex_unlock(&c->srv->mu); + if (free_now) client_free(c); +} + +static void client_decref_n(client *c, int n) { + int free_now = 0; + if (n <= 0) return; + pthread_mutex_lock(&c->srv->mu); + c->refcount -= n; + if (c->refcount <= 0) free_now = 1; + pthread_mutex_unlock(&c->srv->mu); + if (free_now) client_free(c); +} + +static void client_unlink(client *c) { + http_server *s = c->srv; + if (!c->linked) return; + if (c->prev) c->prev->next = c->next; + else s->clients = c->next; + if (c->next) c->next->prev = c->prev; + c->prev = c->next = NULL; + c->linked = 0; + if (s->n_clients > 0) s->n_clients--; +} + +static int close_client_unlink(client *c) { + if (!c || c->cancelled) return 0; + c->cancelled = 1; + if (c->fd >= 0) { + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + aeDeleteFileEvent(c->srv->loop, c->fd, AE_WRITABLE); + close(c->fd); + c->fd = -1; + } + client_unlink(c); + return 1; +} + +static void close_client(client *c) { + if (!close_client_unlink(c)) return; + client_decref(c); +} + +static void append_http_response_ex(client *c, int status, const char *ctype, + const char *extra_headers, + const char *body, size_t body_len) { + const char *reason = "OK"; + if (status == 204) reason = "No Content"; + else if (status == 400) reason = "Bad Request"; + else if (status == 401) reason = "Unauthorized"; + else if (status == 404) reason = "Not Found"; + else if (status == 405) reason = "Method Not Allowed"; + else if (status == 422) reason = "Unprocessable Entity"; + else if (status == 503) reason = "Service Unavailable"; + else if (status >= 500) reason = "Internal Server Error"; + + sbuf_printf(&c->out, + "HTTP/1.1 %d %s\r\n" + "Content-Length: %zu\r\n" + "Connection: %s\r\n", + status, reason, body_len, + c->close_after_write ? "close" : "keep-alive"); + if (ctype) + sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype); + if (c->srv->enable_cors) { + sbuf_puts(&c->out, "Access-Control-Allow-Origin: *\r\n"); + sbuf_puts(&c->out, "Access-Control-Allow-Headers: authorization, content-type\r\n"); + sbuf_puts(&c->out, "Access-Control-Allow-Methods: POST, OPTIONS\r\n"); + } + if (extra_headers) + sbuf_puts(&c->out, extra_headers); + sbuf_puts(&c->out, "\r\n"); + if (body_len) sbuf_append(&c->out, body, body_len); +} + +static void append_http_response(client *c, int status, const char *ctype, + const char *body, size_t body_len) { + append_http_response_ex(c, status, ctype, NULL, body, body_len); +} + +static void append_json_error(sbuf *b, const char *message, const char *type) { + sbuf_puts(b, "{\"error\":{\"message\":\""); + for (const char *p = message ? message : ""; *p; p++) { + if (*p == '"' || *p == '\\') { + sbuf_putc(b, '\\'); + sbuf_putc(b, *p); + } else { + sbuf_putc(b, *p); + } + } + sbuf_puts(b, "\",\"type\":\""); + sbuf_puts(b, type ? type : "server_error"); + sbuf_puts(b, "\"}}"); +} + +static char *json_error_body(const char *message, const char *type, size_t *len) { + sbuf b = {0}; + append_json_error(&b, message, type); + *len = b.len; + return b.ptr; +} + +static void job_set_error(job *j, int status, const char *message, const char *type) { + free(j->response); + j->status = status; + j->content_type = "application/json"; + j->response = json_error_body(message, type, &j->response_len); +} + +static bool auth_ok(http_server *s, const char *auth) { + if (!s->api_key || !s->api_key[0]) return true; + if (!auth) return false; + const char prefix[] = "Bearer "; + return !strncmp(auth, prefix, sizeof(prefix) - 1) && + !strcmp(auth + sizeof(prefix) - 1, s->api_key); +} + +static ssize_t find_header_end(const char *p, size_t n) { + for (size_t i = 3; i < n; i++) { + if (p[i - 3] == '\r' && p[i - 2] == '\n' && + p[i - 1] == '\r' && p[i] == '\n') + return (ssize_t)(i + 1); + } + for (size_t i = 1; i < n; i++) { + if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1); + } + return -1; +} + +static char *header_value_dup(const char *h, size_t n, const char *name) { + size_t name_len = strlen(name); + const char *p = h, *end = h + n; + while (p < end) { + const char *line = p; + while (p < end && *p != '\n') p++; + size_t len = (size_t)(p - line); + if (len && line[len - 1] == '\r') len--; + if (len > name_len && !strncasecmp(line, name, name_len) && + line[name_len] == ':') { + const char *v = line + name_len + 1; + while (v < line + len && isspace((unsigned char)*v)) v++; + const char *e = line + len; + while (e > v && isspace((unsigned char)e[-1])) e--; + return xstrndup(v, (size_t)(e - v)); + } + if (p < end) p++; + } + return NULL; +} + +static int header_has_token(const char *value, const char *token) { + size_t token_len = strlen(token); + const char *p = value; + while (p && *p) { + while (*p == ',' || isspace((unsigned char)*p)) p++; + const char *start = p; + while (*p && *p != ',') p++; + const char *end = p; + while (end > start && isspace((unsigned char)end[-1])) end--; + if ((size_t)(end - start) == token_len && + !strncasecmp(start, token, token_len)) + return 1; + } + return 0; +} + +static int parse_request_headers(client *c) { + char line[512]; + size_t i = 0; + while (i < c->header_len && c->in.ptr[i] != '\n' && i + 1 < sizeof(line)) { + line[i] = c->in.ptr[i]; + i++; + } + line[i] = '\0'; + int fields = sscanf(line, "%7s %255s %15s", + c->req.method, c->req.path, c->req.version); + if (fields < 2) + return -1; + if (fields < 3) + snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0"); + char *q = strchr(c->req.path, '?'); + if (q) *q = '\0'; + + c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0; + char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection"); + if (conn) { + if (header_has_token(conn, "close")) + c->close_after_write = 1; + else if (header_has_token(conn, "keep-alive")) + c->close_after_write = 0; + free(conn); + } + + char *cl = header_value_dup(c->in.ptr, c->header_len, "Content-Length"); + if (cl) { + char *end = NULL; + errno = 0; + unsigned long long v = strtoull(cl, &end, 10); + if (errno || !end || *end || v > PPLX_HTTP_MAX_BODY) { + free(cl); + return -1; + } + c->content_length = (size_t)v; + free(cl); + } + c->req.auth = header_value_dup(c->in.ptr, c->header_len, "Authorization"); + return 0; +} + +static int complete_request(client *c) { + if (!c->header_done) { + ssize_t hend = find_header_end(c->in.ptr, c->in.len); + if (hend < 0) { + if (c->in.len > PPLX_HTTP_MAX_HEADER) return -1; + return 0; + } + c->header_done = 1; + c->header_len = (size_t)hend; + if (parse_request_headers(c) != 0) return -1; + } + if (c->content_length > PPLX_HTTP_MAX_BODY) return -1; + if (c->in.len < c->header_len + c->content_length) return 0; + c->req.body_len = c->content_length; + c->req.body = xmalloc(c->content_length + 1); + memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length); + c->req.body[c->content_length] = '\0'; + return 1; +} + +static void send_bad_http_request(client *c) { + size_t len; + char *body = json_error_body("bad HTTP request", "invalid_request_error", + &len); + c->close_after_write = 1; + append_http_response(c, 400, "application/json", body, len); + free(body); + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + if (queue_write(c) != AE_OK) close_client(c); +} + +static void reset_client_for_next_request(client *c) { + size_t consumed = c->header_len + c->content_length; + if (consumed > c->in.len) consumed = c->in.len; + size_t remain = c->in.len - consumed; + if (remain) + memmove(c->in.ptr, c->in.ptr + consumed, remain); + c->in.len = remain; + if (c->in.ptr) c->in.ptr[remain] = '\0'; + + http_request_free(&c->req); + sbuf_clear(&c->out); + c->sent = 0; + c->header_done = 0; + c->header_len = 0; + c->content_length = 0; + c->close_after_write = 0; +} + +static void arm_next_request(client *c) { + reset_client_for_next_request(c); + if (c->cancelled || c->fd < 0) return; + + if (c->in.len) { + int done = complete_request(c); + if (done < 0) { + send_bad_http_request(c); + return; + } + if (done > 0) { + dispatch_request(c); + return; + } + } + + if (aeCreateFileEvent(c->srv->loop, c->fd, AE_READABLE, read_cb, c) != AE_OK) + close_client(c); +} + +static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + client *c = clientData; + while (c->sent < c->out.len) { + ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent); + if (n > 0) { + c->sent += (size_t)n; + c->last_active_ms = mstime(); + continue; + } + if (n < 0 && errno == EINTR) continue; + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return; + close_client(c); + return; + } + aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE); + if (c->close_after_write) close_client(c); + else arm_next_request(c); +} + +static int queue_write(client *c) { + if (!c || c->cancelled || c->fd < 0) return -1; + return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c); +} + +static void enqueue_done(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->done_tail) s->done_tail->next = j; + else s->done_head = j; + s->done_tail = j; + pthread_mutex_unlock(&s->mu); + const unsigned char byte = 1; + (void)write(s->completion_pipe[1], &byte, 1); +} + +static job *dequeue_job(http_server *s) { + pthread_mutex_lock(&s->mu); + while (!s->job_head && !s->stopping) + pthread_cond_wait(&s->cv, &s->mu); + job *j = s->job_head; + if (j) { + s->job_head = j->next; + if (!s->job_head) s->job_tail = NULL; + j->next = NULL; + } + pthread_mutex_unlock(&s->mu); + return j; +} + +static int drain_ready_jobs(http_server *s, job **jobs, int max_jobs) { + int n = 0; + pthread_mutex_lock(&s->mu); + while (n < max_jobs && s->job_head) { + job *j = s->job_head; + s->job_head = j->next; + if (!s->job_head) s->job_tail = NULL; + j->next = NULL; + jobs[n++] = j; + } + pthread_mutex_unlock(&s->mu); + return n; +} + +static void enqueue_job(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->job_tail) s->job_tail->next = j; + else s->job_head = j; + s->job_tail = j; + pthread_cond_signal(&s->cv); + pthread_mutex_unlock(&s->mu); +} + +static job *pop_done(http_server *s) { + pthread_mutex_lock(&s->mu); + job *j = s->done_head; + if (j) { + s->done_head = j->next; + if (!s->done_head) s->done_tail = NULL; + j->next = NULL; + } + pthread_mutex_unlock(&s->mu); + return j; +} + +static void job_free(job *j) { + if (!j) return; + free(j->body); + free(j->auth); + free(j->extra_headers); + free(j->response); + free(j); +} + +static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + http_server *s = clientData; + unsigned char tmp[128]; + while (read(fd, tmp, sizeof(tmp)) > 0) {} + + job *j; + while ((j = pop_done(s)) != NULL) { + client *c = j->c; + int refs_to_drop = 1; /* job reference */ + if (!c->cancelled) { + append_http_response_ex(c, j->status, j->content_type, + j->extra_headers, + j->response ? j->response : "", + j->response ? j->response_len : 0); + if (queue_write(c) != AE_OK) + refs_to_drop += close_client_unlink(c); + } + client_decref_n(c, refs_to_drop); + job_free(j); + } +} + +static bool route_is_embedding(const char *method, const char *path) { + return !strcmp(method, "POST") && + (!strcmp(path, "/v1/embeddings") || + !strcmp(path, "/v1/contextualizedembeddings")); +} + +static void dispatch_request(client *c) { + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + + if (!strcmp(c->req.method, "OPTIONS")) { + append_http_response(c, 204, NULL, "", 0); + if (queue_write(c) != AE_OK) close_client(c); + return; + } + + if (!route_is_embedding(c->req.method, c->req.path)) { + size_t len; + char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); + append_http_response(c, 404, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) close_client(c); + return; + } + + if (!auth_ok(c->srv, c->req.auth)) { + size_t len; + char *body = json_error_body("missing or invalid bearer token", + "authentication_error", &len); + append_http_response(c, 401, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) close_client(c); + return; + } + + job *j = xcalloc(1, sizeof(*j)); + j->srv = c->srv; + j->c = c; + snprintf(j->method, sizeof(j->method), "%s", c->req.method); + snprintf(j->path, sizeof(j->path), "%s", c->req.path); + j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); + j->body_len = c->req.body_len; + j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; + j->created_ns = nstime(); + client_incref(c); + enqueue_job(j); +} + +static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + client *c = clientData; + char tmp[PPLX_HTTP_IO_BUF]; + for (;;) { + ssize_t n = read(fd, tmp, sizeof(tmp)); + if (n > 0) { + sbuf_append(&c->in, tmp, (size_t)n); + c->last_active_ms = mstime(); + int done = complete_request(c); + if (done < 0) { + send_bad_http_request(c); + return; + } + if (done > 0) { + dispatch_request(c); + return; + } + continue; + } + if (n == 0) { + close_client(c); + return; + } + if (errno == EINTR) continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) return; + close_client(c); + return; + } +} + +static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; (void)mask; + http_server *s = clientData; + char cip[64]; + int cport = 0; + for (;;) { + int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno == EAGAIN || errno == EWOULDBLOCK) return; + if (errno == EINTR) continue; + return; + } + set_nonblock(cfd); + client *c = xcalloc(1, sizeof(*c)); + c->srv = s; + c->fd = cfd; + c->refcount = 1; + c->last_active_ms = mstime(); + c->linked = 1; + c->next = s->clients; + if (s->clients) s->clients->prev = c; + s->clients = c; + s->n_clients++; + if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) + close_client(c); + } +} + +static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { + (void)loop; (void)id; + http_server *s = clientData; + uint64_t now = mstime(); + client *c = s->clients; + while (c) { + client *next = c->next; + if (!c->cancelled && + now - c->last_active_ms > PPLX_HTTP_CLIENT_TIMEOUT_MS) + close_client(c); + c = next; + } + return PPLX_HTTP_SWEEP_MS; +} + +static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)fd; (void)mask; + http_server *s = clientData; + unsigned char tmp[128]; + while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {} + aeStop(loop); +} + +static bool ve_add(cJSON *detail, const char *loc_json, + const char *msg, const char *type) { + cJSON *obj = cJSON_CreateObject(); + cJSON *loc = cJSON_Parse(loc_json); + if (!obj || !loc) { + cJSON_Delete(obj); + cJSON_Delete(loc); + return false; + } + cJSON_AddItemToObject(obj, "loc", loc); + cJSON_AddStringToObject(obj, "msg", msg ? msg : ""); + cJSON_AddStringToObject(obj, "type", type ? type : ""); + cJSON_AddItemToArray(detail, obj); + return true; +} + +static void job_set_422(job *j, cJSON *detail) { + cJSON *root = cJSON_CreateObject(); + cJSON_AddItemToObject(root, "detail", cJSON_Duplicate(detail, 1)); + char *s = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + if (!s) { + job_set_error(j, 500, "failed to render validation error", "server_error"); + return; + } + j->status = 422; + j->content_type = "application/json"; + j->response = xstrdup(s); + j->response_len = strlen(j->response); + cJSON_free(s); +} + +static void job_set_timing_header(job *j) { + if (!j || j->extra_headers || !j->created_ns) return; + uint64_t done_ns = nstime(); + uint64_t queue_ns = j->started_ns > j->created_ns + ? j->started_ns - j->created_ns : 0; + uint64_t worker_ns = j->started_ns && done_ns > j->started_ns + ? done_ns - j->started_ns : 0; + + char buf[384]; + int n = snprintf(buf, sizeof(buf), + "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " + "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " + "worker;dur=%.3f\r\n", + ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), + ns_to_ms(j->tokenize_ns), ns_to_ms(j->infer_ns), + ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); + if (n > 0 && (size_t)n < sizeof(buf)) + j->extra_headers = xstrdup(buf); +} + +static cJSON *parse_json_body(job *j, cJSON *detail) { + if (!j->body || j->body_len == 0) { + ve_add(detail, "[\"body\"]", "body required", "missing"); + return NULL; + } + const char *end = NULL; + cJSON *root = cJSON_ParseWithLengthOpts(j->body, j->body_len, &end, false); + if (!root) { + ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); + return NULL; + } + const char *body_end = j->body + j->body_len; + while (end && end < body_end && isspace((unsigned char)*end)) end++; + if (!end || end != body_end) { + ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); + cJSON_Delete(root); + return NULL; + } + if (!cJSON_IsObject(root)) { + ve_add(detail, "[\"body\"]", "body must be a JSON object", "type_error.object"); + cJSON_Delete(root); + return NULL; + } + return root; +} + +static bool cjson_is_integer(cJSON *item) { + if (!cJSON_IsNumber(item)) return false; + double d = item->valuedouble; + int i = item->valueint; + return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i; +} + +static const char *encoding_from_root(cJSON *root, cJSON *detail) { + cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); + if (!encoding) return "base64_int8"; + if (!cJSON_IsString(encoding) || !encoding->valuestring) { + ve_add(detail, "[\"body\",\"encoding_format\"]", + "encoding_format must be a string", "type_error.string"); + return "base64_int8"; + } + if (strcmp(encoding->valuestring, "base64_int8") && + strcmp(encoding->valuestring, "base64_binary") && + strcmp(encoding->valuestring, "float")) { + ve_add(detail, "[\"body\",\"encoding_format\"]", + "value is not a valid enum member; permitted: 'base64_int8', 'base64_binary', 'float'", + "enum"); + return "base64_int8"; + } + return encoding->valuestring; +} + +static int dimensions_from_root(cJSON *root, cJSON *detail, int max_dim, + const char *encoding) { + int dims = max_dim; + cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); + if (dimensions) { + if (!cjson_is_integer(dimensions)) { + ve_add(detail, "[\"body\",\"dimensions\"]", "dimensions must be an integer", + "type_error.integer"); + } else { + dims = dimensions->valueint; + if (dims < 128 || dims > max_dim) { + char msg[96]; + snprintf(msg, sizeof(msg), "dimensions must be between 128 and %d", max_dim); + ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); + } + } + } + if (!strcmp(encoding, "base64_binary") && dims % 8 != 0) + ve_add(detail, "[\"body\",\"dimensions\"]", + "dimensions must be divisible by 8 for base64_binary", + "value_error.divisible"); + return dims; +} + +static char *base64_encode(const unsigned char *src, size_t n) { + static const char tbl[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + size_t out_len = ((n + 2) / 3) * 4; + char *out = xmalloc(out_len + 1); + size_t o = 0; + for (size_t i = 0; i < n; i += 3) { + unsigned v = (unsigned)src[i] << 16; + int remain = (int)(n - i); + if (remain > 1) v |= (unsigned)src[i + 1] << 8; + if (remain > 2) v |= (unsigned)src[i + 2]; + out[o++] = tbl[(v >> 18) & 63]; + out[o++] = tbl[(v >> 12) & 63]; + out[o++] = remain > 1 ? tbl[(v >> 6) & 63] : '='; + out[o++] = remain > 2 ? tbl[v & 63] : '='; + } + out[o] = '\0'; + return out; +} + +static signed char quantize_int8_tanh(float x) { + int v = (int)lrintf(tanhf(x) * 127.0f); + if (v > 127) v = 127; + if (v < -128) v = -128; + return (signed char)v; +} + +static char *encode_embedding(const float *emb, int dims, const char *encoding) { + if (!strcmp(encoding, "base64_binary")) { + size_t bytes = (size_t)dims / 8; + unsigned char *bits = xcalloc(bytes, 1); + for (int i = 0; i < dims; i++) { + if (emb[i] >= 0.0f) + bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7)); + } + char *out = base64_encode(bits, bytes); + free(bits); + return out; + } + + signed char *q = xmalloc((size_t)dims); + for (int i = 0; i < dims; i++) + q[i] = quantize_int8_tanh(emb[i]); + char *out = base64_encode((const unsigned char *)q, (size_t)dims); + free(q); + return out; +} + +typedef struct { + int *ids; + int n_tokens; +} token_buf; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + int n_inputs; + token_buf *tokens; + pplx_input_t *inputs; + int total_tokens; + int ready; +} embedding_request; + +typedef struct { + int *ids; + int n_tokens; + pplx_span_t *spans; + int n_spans; +} contextual_doc; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + contextual_doc *docs; + int n_docs; + int total_chunks; + int total_tokens; + int ready; +} contextual_request; + +static void free_token_bufs(token_buf *t, int n) { + if (!t) return; + for (int i = 0; i < n; i++) free(t[i].ids); +} + +static void embedding_request_free(embedding_request *r) { + if (!r) return; + free_token_bufs(r->tokens, r->n_inputs); + free(r->tokens); + free(r->inputs); + if (r->root) cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +static void contextual_request_free(contextual_request *r) { + if (!r) return; + for (int i = 0; i < r->n_docs; i++) { + free(r->docs[i].ids); + free(r->docs[i].spans); + } + free(r->docs); + if (r->root) cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +static int tokenize_one(loaded_model *m, job *j, const char *text, + token_buf *out) { + memset(out, 0, sizeof(*out)); + uint64_t t0 = nstime(); + out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, + text, &out->n_tokens); + if (j) j->tokenize_ns += nstime() - t0; + if (!out->ids || out->n_tokens <= 0) { + free(out->ids); + memset(out, 0, sizeof(*out)); + return -1; + } + return 0; +} + +static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs, + int batch, float *out) { +#ifdef USE_MLX + if (m->mlx_ctx) + return pplx_mlx_embed_batch(m->mlx_ctx, inputs, batch, out); +#endif +#ifdef USE_CUDA + if (m->cuda_ctx) + return pplx_cuda_embed_batch(m->cuda_ctx, inputs, batch, out); +#endif + return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); +} + +static int model_embed_spans_batch(loaded_model *m, + const pplx_context_input_t *inputs, + int batch, float *out) { +#ifdef USE_MLX + if (m->mlx_ctx) + return pplx_mlx_embed_spans_batch(m->mlx_ctx, inputs, batch, out); +#endif +#ifdef USE_CUDA + if (m->cuda_ctx) + return pplx_cuda_embed_spans_batch(m->cuda_ctx, inputs, batch, out); +#endif + return pplx_model_embed_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, + out); +} + +static int model_uses_dense_batches(const loaded_model *m) { +#ifdef USE_MLX + return m->mlx_ctx != NULL; +#else + (void)m; + return 0; +#endif +} + +/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX + * pads each dense row to the longest input in the chunk. */ +static int inference_batch_accepts_input(const loaded_model *m, int batch, + int packed_tokens, int next_tokens, + int max_batch_tokens) { + if (batch == 0) return 1; + if (model_uses_dense_batches(m)) + return next_tokens <= max_batch_tokens / (batch + 1); + return next_tokens <= max_batch_tokens - packed_tokens; +} + +static void append_embedding_object(sbuf *b, int index, const char *embedding) { + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index); + sbuf_puts(b, embedding); + sbuf_puts(b, "\"}"); +} + +static void append_float_embedding_object(sbuf *b, int index, + const float *emb, int dims) { + sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); + for (int i = 0; i < dims; i++) { + if (i) sbuf_putc(b, ','); + float v = (float)quantize_int8_tanh(emb[i]) / 128.0f; + sbuf_printf(b, "%.9g", (double)v); + } + sbuf_puts(b, "]}"); +} + +static void set_response_from_buf(job *j, sbuf *b) { + j->status = 200; + j->content_type = "application/json"; + j->response = b->ptr; + j->response_len = b->len; + memset(b, 0, sizeof(*b)); +} + +static void render_embedding_response(embedding_request *r, const float *embs) { + char **encoded = NULL; + if (strcmp(r->encoding, "float")) { + encoded = xcalloc((size_t)r->n_inputs, sizeof(*encoded)); + for (int i = 0; i < r->n_inputs; i++) + encoded[i] = encode_embedding(embs + (size_t)i * r->model->info->dim, + r->dims, r->encoding); + } + + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + for (int i = 0; i < r->n_inputs; i++) { + if (i) sbuf_putc(&b, ','); + if (encoded) + append_embedding_object(&b, i, encoded[i]); + else + append_float_embedding_object(&b, i, + embs + (size_t)i * r->model->info->dim, r->dims); + } + double cost = ((double)r->total_tokens / 1000000.0) * + r->model->info->price_per_mtok; + sbuf_printf(&b, + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," + "\"total_tokens\":%d,\"cost\":{\"input_cost\":%.10g," + "\"total_cost\":%.10g,\"currency\":\"USD\"}}}", + r->model->info->id, r->total_tokens, r->total_tokens, cost, cost); + set_response_from_buf(r->j, &b); + + for (int i = 0; encoded && i < r->n_inputs; i++) free(encoded[i]); + free(encoded); +} + +static int embedding_request_compatible(const embedding_request *a, + const embedding_request *b) { + return a->ready && b->ready && + a->model == b->model && + a->dims == b->dims && + !strcmp(a->encoding, b->encoding); +} + +typedef struct { + pplx_input_t input; + int output_index; +} embedding_batch_item; + +static int embedding_batch_item_cmp(const void *a, const void *b) { + const embedding_batch_item *ia = a; + const embedding_batch_item *ib = b; + if (ia->input.n_tokens < ib->input.n_tokens) return -1; + if (ia->input.n_tokens > ib->input.n_tokens) return 1; + return ia->output_index - ib->output_index; +} + +static int embedding_request_fits_group(const embedding_request *r, + int group_inputs, int group_tokens, + int max_batch, int max_batch_tokens) { + if (group_inputs == 0) return 1; + return r->n_inputs <= max_batch - group_inputs && + r->total_tokens <= max_batch_tokens - group_tokens; +} + +static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { + if (n_reqs <= 0) return; + loaded_model *m = reqs[0]->model; + int dim = m->info->dim; + int total_inputs = 0; + for (int i = 0; i < n_reqs; i++) + total_inputs += reqs[i]->n_inputs; + + embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items)); + float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float)); + + int pos = 0; + for (int i = 0; i < n_reqs; i++) { + for (int k = 0; k < reqs[i]->n_inputs; k++) { + items[pos].input = reqs[i]->inputs[k]; + items[pos].output_index = pos; + pos++; + } + } + qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); + + int max_batch = reqs[0]->j->srv->batch_size > 0 + ? reqs[0]->j->srv->batch_size : total_inputs; + if (max_batch > total_inputs) max_batch = total_inputs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + pplx_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); + + int failed = 0; + uint64_t infer_ns = 0; + for (int start = 0; start < total_inputs;) { + int cur = 0; + int tokens = 0; + while (start + cur < total_inputs && cur < max_batch) { + int n_tokens = items[start + cur].input.n_tokens; + if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, + max_batch_tokens)) + break; + inputs[cur] = items[start + cur].input; + tokens += n_tokens; + cur++; + } + uint64_t t0 = nstime(); + int embed_rc = model_embed_batch(m, inputs, cur, batch_embs); + infer_ns += nstime() - t0; + if (embed_rc != 0) { + failed = 1; + break; + } + for (int i = 0; i < cur; i++) + memcpy(embs + (size_t)items[start + i].output_index * dim, + batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float)); + start += cur; + } + + pos = 0; + for (int i = 0; i < n_reqs; i++) { + reqs[i]->j->infer_ns += infer_ns; + if (failed) { + job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); + } else { + uint64_t t0 = nstime(); + render_embedding_response(reqs[i], embs + (size_t)pos * dim); + reqs[i]->j->encode_ns += nstime() - t0; + } + pos += reqs[i]->n_inputs; + } + + free(batch_embs); + free(inputs); + free(embs); + free(items); +} + +static void render_contextual_response(contextual_request *r, + const float *embs) { + sbuf b = {0}; + sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); + int span_offset = 0; + for (int di = 0; di < r->n_docs; di++) { + if (di) sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); + for (int ci = 0; ci < r->docs[di].n_spans; ci++) { + if (ci) sbuf_putc(&b, ','); + const float *emb = embs + (size_t)span_offset * r->model->info->dim; + if (!strcmp(r->encoding, "float")) { + append_float_embedding_object(&b, ci, + emb, r->dims); + } else { + char *encoded = encode_embedding(emb, r->dims, r->encoding); + append_embedding_object(&b, ci, encoded); + free(encoded); + } + span_offset++; + } + sbuf_puts(&b, "]}"); + } + sbuf_printf(&b, + "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", + r->model->info->id, r->total_tokens, r->total_tokens); + set_response_from_buf(r->j, &b); +} + +static int contextual_request_compatible(const contextual_request *a, + const contextual_request *b) { + return a->ready && b->ready && + a->model == b->model && + a->dims == b->dims && + !strcmp(a->encoding, b->encoding); +} + +static int contextual_request_fits_group(const contextual_request *r, + int group_docs, int group_tokens, + int max_batch, int max_batch_tokens) { + if (group_docs == 0) return 1; + return r->n_docs <= max_batch - group_docs && + r->total_tokens <= max_batch_tokens - group_tokens; +} + +typedef struct { + pplx_context_input_t input; + int output_span_index; + int order; +} contextual_batch_item; + +static int contextual_batch_item_cmp(const void *a, const void *b) { + const contextual_batch_item *ia = a; + const contextual_batch_item *ib = b; + if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1; + if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1; + return ia->order - ib->order; +} + +static void execute_contextual_request_list(contextual_request **reqs, + int n_reqs) { + if (n_reqs <= 0) return; + loaded_model *m = reqs[0]->model; + int dim = m->info->dim; + int total_docs = 0; + int total_chunks = 0; + for (int i = 0; i < n_reqs; i++) { + if (total_docs > INT_MAX - reqs[i]->n_docs || + total_chunks > INT_MAX - reqs[i]->total_chunks) { + for (int k = 0; k < n_reqs; k++) + job_set_error(reqs[k]->j, 500, "contextual batch is too large", + "server_error"); + return; + } + total_docs += reqs[i]->n_docs; + total_chunks += reqs[i]->total_chunks; + } + if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { + for (int i = 0; i < n_reqs; i++) + job_set_error(reqs[i]->j, 500, "contextual batch is too large", + "server_error"); + return; + } + + contextual_batch_item *items = + xmalloc((size_t)total_docs * sizeof(*items)); + float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); + int pos = 0; + int span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + for (int d = 0; d < reqs[i]->n_docs; d++) { + contextual_doc *doc = &reqs[i]->docs[d]; + items[pos].input.input.ids = doc->ids; + items[pos].input.input.n_tokens = doc->n_tokens; + items[pos].input.spans = doc->spans; + items[pos].input.n_spans = doc->n_spans; + items[pos].output_span_index = span_offset; + items[pos].order = pos; + span_offset += doc->n_spans; + pos++; + } + } + qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); + + int max_batch = reqs[0]->j->srv->batch_size > 0 + ? reqs[0]->j->srv->batch_size : total_docs; + if (max_batch > total_docs) max_batch = total_docs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + pplx_context_input_t *inputs = + xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = NULL; + size_t batch_emb_cap = 0; + + int failed = 0; + uint64_t infer_ns = 0; + for (int start = 0; start < total_docs;) { + int cur = 0; + int tokens = 0; + int chunks = 0; + while (start + cur < total_docs && cur < max_batch) { + const pplx_context_input_t *input = &items[start + cur].input; + if (!inference_batch_accepts_input(m, cur, tokens, + input->input.n_tokens, + max_batch_tokens)) + break; + inputs[cur] = *input; + tokens += input->input.n_tokens; + chunks += input->n_spans; + cur++; + } + size_t needed = (size_t)chunks * dim; + if (needed > batch_emb_cap) { + batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs)); + batch_emb_cap = needed; + } + uint64_t t0 = nstime(); + int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs); + infer_ns += nstime() - t0; + if (embed_rc != 0) { + failed = 1; + break; + } + int batch_span = 0; + for (int i = 0; i < cur; i++) { + int n_spans = inputs[i].n_spans; + memcpy(embs + (size_t)items[start + i].output_span_index * dim, + batch_embs + (size_t)batch_span * dim, + (size_t)n_spans * dim * sizeof(*embs)); + batch_span += n_spans; + } + start += cur; + } + + span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + reqs[i]->j->infer_ns += infer_ns; + if (failed) { + job_set_error(reqs[i]->j, 500, "contextual embedding failed", + "server_error"); + } else { + uint64_t t0 = nstime(); + render_contextual_response(reqs[i], + embs + (size_t)span_offset * dim); + reqs[i]->j->encode_ns += nstime() - t0; + } + span_offset += reqs[i]->total_chunks; + } + + free(batch_embs); + free(inputs); + free(embs); + free(items); +} + +static int validate_common(cJSON *root, cJSON *detail, int expect_context, + loaded_model **out_model, int *out_dims, + const char **out_encoding, http_server *s) { + cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); + const char *model_id = NULL; + if (!model_item) { + ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); + } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); + } else { + model_id = model_item->valuestring; + model_slot slot = model_slot_for_id(model_id); + if (slot == MODEL_UNKNOWN || k_models[slot].is_contextual != expect_context) { + ve_add(detail, "[\"body\",\"model\"]", + "value is not a valid enum member for this endpoint", "enum"); + } else { + loaded_model *m = &s->models[slot]; + if (!m->info) { + *out_model = NULL; + } else { + *out_model = m; + } + } + } + const char *enc = encoding_from_root(root, detail); + int max_dim = model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN + ? k_models[model_slot_for_id(model_id)].dim : 2560; + int dims = dimensions_from_root(root, detail, max_dim, enc); + *out_encoding = enc; + *out_dims = dims; + return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && + !s->models[model_slot_for_id(model_id)].info ? 503 : 0; +} + +static void prepare_embedding_request(job *j, cJSON *root, http_server *s, + embedding_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + int unloaded = validate_common(root, detail, 0, &m, &dims, &encoding, s); + out->model = m; + out->dims = dims; + out->encoding = encoding; + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_inputs = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (cJSON_IsString(input)) { + if (!input->valuestring || input->valuestring[0] == '\0') + ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", + "value_error.empty"); + n_inputs = 1; + } else if (cJSON_IsArray(input)) { + n_inputs = cJSON_GetArraySize(input); + if (n_inputs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", + "value_error.list.min_items"); + else if (n_inputs > PPLX_API_MAX_STANDARD_INPUTS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", + "value_error.list.max_items"); + cJSON *item; + int i = 0; + cJSON_ArrayForEach(item, input) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i); + if (!cJSON_IsString(item)) { + ve_add(detail, loc, "input item must be a string", "type_error.string"); + } else if (!item->valuestring || item->valuestring[0] == '\0') { + ve_add(detail, loc, "input item must not be empty", "value_error.empty"); + } + i++; + } + } else { + ve_add(detail, "[\"body\",\"input\"]", + "input must be a string or an array of strings", "type_error"); + } + + if (cJSON_GetArraySize(detail) == 0 && unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", + "model_not_loaded"); + cJSON_Delete(detail); + return; + } + + if (cJSON_GetArraySize(detail) == 0 && m && input) { + out->n_inputs = n_inputs; + out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens)); + out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs)); + + int idx = 0; + if (input && cJSON_IsString(input) && input->valuestring) { + if (tokenize_one(m, j, input->valuestring, &out->tokens[0]) != 0) { + ve_add(detail, "[\"body\",\"input\"]", "tokenization failed", + "value_error.tokenization"); + } else { + out->inputs[0].ids = out->tokens[0].ids; + out->inputs[0].n_tokens = out->tokens[0].n_tokens; + out->total_tokens = out->tokens[0].n_tokens; + if (out->tokens[0].n_tokens > PPLX_API_MAX_ITEM_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", + "input exceeds 32768 token limit", + "value_error.context_length"); + } + } else { + cJSON *item; + cJSON_ArrayForEach(item, input) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); + if (!cJSON_IsString(item) || !item->valuestring) { + ve_add(detail, loc, "input item must be a string", + "type_error.string"); + idx++; + continue; + } + if (tokenize_one(m, j, item->valuestring, + &out->tokens[idx]) != 0) { + ve_add(detail, loc, "tokenization failed", + "value_error.tokenization"); + } else { + out->inputs[idx].ids = out->tokens[idx].ids; + out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; + out->total_tokens += out->tokens[idx].n_tokens; + if (out->tokens[idx].n_tokens > PPLX_API_MAX_ITEM_TOKENS) + ve_add(detail, loc, "input exceeds 32768 token limit", + "value_error.context_length"); + } + idx++; + } + } + + if (out->total_tokens > PPLX_API_MAX_TOTAL_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + embedding_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +static void prepare_contextual_request(job *j, cJSON *root, http_server *s, + contextual_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + int unloaded = validate_common(root, detail, 1, &m, &dims, &encoding, s); + out->model = m; + out->dims = dims; + out->encoding = encoding; + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_docs = 0; + int total_chunks = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (!cJSON_IsArray(input)) { + ve_add(detail, "[\"body\",\"input\"]", + "input must be an array of document chunk arrays", "type_error.array"); + } else { + n_docs = cJSON_GetArraySize(input); + if (n_docs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", + "value_error.list.min_items"); + else if (n_docs > PPLX_API_MAX_CONTEXT_DOCS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", + "value_error.list.max_items"); + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + if (!cJSON_IsArray(doc_arr)) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must be an array of strings", + "type_error.array"); + di++; + continue; + } + int doc_chunks = cJSON_GetArraySize(doc_arr); + if (doc_chunks > PPLX_API_MAX_CONTEXT_CHUNKS - total_chunks) + total_chunks = PPLX_API_MAX_CONTEXT_CHUNKS + 1; + else + total_chunks += doc_chunks; + if (doc_chunks < 1) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must contain at least 1 chunk", + "value_error.list.min_items"); + } + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (!cJSON_IsString(chunk)) { + ve_add(detail, loc, "chunk must be a string", "type_error.string"); + } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { + ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); + } + ci++; + } + di++; + } + if (total_chunks > PPLX_API_MAX_CONTEXT_CHUNKS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", + "value_error.list.max_items"); + } + + if (cJSON_GetArraySize(detail) == 0 && unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", + "model_not_loaded"); + cJSON_Delete(detail); + return; + } + + if (cJSON_GetArraySize(detail) == 0 && m) { + out->n_docs = n_docs; + out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); + int64_t request_tokens = 0; + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + contextual_doc *doc = &out->docs[di]; + int n_chunks = cJSON_GetArraySize(doc_arr); + token_buf *chunk_tokens = + xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); + doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); + doc->n_spans = n_chunks; + int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; + int doc_valid = 1; + + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (tokenize_one(m, j, chunk->valuestring, + &chunk_tokens[ci]) != 0) { + ve_add(detail, loc, "tokenization failed", + "value_error.tokenization"); + doc_valid = 0; + } else { + doc_tokens += chunk_tokens[ci].n_tokens; + } + ci++; + } + + if (doc_tokens > PPLX_API_MAX_ITEM_TOKENS) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document exceeds 32768 token limit", + "value_error.context_length"); + doc_valid = 0; + } + request_tokens += doc_tokens; + + if (doc_valid) { + doc->n_tokens = (int)doc_tokens; + doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); + int pos = 0; + for (ci = 0; ci < n_chunks; ci++) { + doc->spans[ci].start = pos; + doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens; + memcpy(doc->ids + pos, chunk_tokens[ci].ids, + (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); + pos += chunk_tokens[ci].n_tokens; + if (ci + 1 < n_chunks) { + doc->ids[pos++] = m->context_separator_id; + } + } + } + free_token_bufs(chunk_tokens, n_chunks); + free(chunk_tokens); + di++; + } + if (request_tokens > PPLX_API_MAX_TOTAL_TOKENS) { + ve_add(detail, "[\"body\",\"input\"]", + "request exceeds 120000 token limit", + "value_error.context_length"); + } else { + out->total_tokens = (int)request_tokens; + out->total_chunks = total_chunks; + } + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + contextual_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +static int parse_job_root(job *j, cJSON **out_root) { + cJSON *detail = cJSON_CreateArray(); + cJSON *root = parse_json_body(j, detail); + if (!root) { + job_set_422(j, detail); + cJSON_Delete(detail); + *out_root = NULL; + return -1; + } + cJSON_Delete(detail); + *out_root = root; + return 0; +} + +static void process_unknown_job(job *j) { + cJSON *root = NULL; + if (!j->started_ns) j->started_ns = nstime(); + uint64_t t0 = nstime(); + int rc = parse_job_root(j, &root); + j->parse_ns += nstime() - t0; + if (rc != 0) + return; + job_set_error(j, 404, "unknown endpoint", "invalid_request_error"); + cJSON_Delete(root); +} + +static void process_job_group(http_server *s, job **jobs, int n_jobs) { + embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); + contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); + int *kind = xcalloc((size_t)n_jobs, sizeof(*kind)); + int *done = xcalloc((size_t)n_jobs, sizeof(*done)); + embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); + contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group)); + + for (int i = 0; i < n_jobs; i++) { + cJSON *root = NULL; + if (!jobs[i]->started_ns) jobs[i]->started_ns = nstime(); + if (!strcmp(jobs[i]->path, "/v1/embeddings")) { + kind[i] = 1; + } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { + kind[i] = 2; + } else { + continue; + } + uint64_t t0 = nstime(); + int parse_rc = parse_job_root(jobs[i], &root); + jobs[i]->parse_ns += nstime() - t0; + if (parse_rc != 0) + continue; + if (kind[i] == 1) + prepare_embedding_request(jobs[i], root, s, &std_reqs[i]); + else + prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]); + } + + for (int i = 0; i < n_jobs; i++) { + if (done[i]) + continue; + + if (!kind[i]) { + process_unknown_job(jobs[i]); + job_set_timing_header(jobs[i]); + enqueue_done(jobs[i]); + done[i] = 1; + continue; + } + + if ((kind[i] == 1 && !std_reqs[i].ready) || + (kind[i] == 2 && !ctx_reqs[i].ready)) { + job_set_timing_header(jobs[i]); + enqueue_done(jobs[i]); + done[i] = 1; + continue; + } + + if (kind[i] == 1) { + int group_n = 1; + int group_inputs = std_reqs[i].n_inputs; + int group_tokens = std_reqs[i].total_tokens; + std_group[0] = &std_reqs[i]; + for (int k = i + 1; k < n_jobs; k++) { + if (!done[k] && kind[k] == 1 && std_reqs[k].ready && + embedding_request_compatible(&std_reqs[i], &std_reqs[k]) && + embedding_request_fits_group(&std_reqs[k], group_inputs, + group_tokens, s->batch_size, s->max_batch_tokens)) { + std_group[group_n++] = &std_reqs[k]; + group_inputs += std_reqs[k].n_inputs; + group_tokens += std_reqs[k].total_tokens; + } + } + execute_embedding_request_list(std_group, group_n); + for (int k = 0; k < group_n; k++) { + int idx = (int)(std_group[k] - std_reqs); + job_set_timing_header(jobs[idx]); + enqueue_done(jobs[idx]); + done[idx] = 1; + } + } else { + int group_n = 1; + int group_docs = ctx_reqs[i].n_docs; + int group_tokens = ctx_reqs[i].total_tokens; + ctx_group[0] = &ctx_reqs[i]; + for (int k = i + 1; k < n_jobs; k++) { + if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && + contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) && + contextual_request_fits_group(&ctx_reqs[k], group_docs, + group_tokens, s->batch_size, s->max_batch_tokens)) { + ctx_group[group_n++] = &ctx_reqs[k]; + group_docs += ctx_reqs[k].n_docs; + group_tokens += ctx_reqs[k].total_tokens; + } + } + execute_contextual_request_list(ctx_group, group_n); + for (int k = 0; k < group_n; k++) { + int idx = (int)(ctx_group[k] - ctx_reqs); + job_set_timing_header(jobs[idx]); + enqueue_done(jobs[idx]); + done[idx] = 1; + } + } + } + + for (int i = 0; i < n_jobs; i++) { + embedding_request_free(&std_reqs[i]); + contextual_request_free(&ctx_reqs[i]); + } + free(ctx_group); + free(std_group); + free(done); + free(kind); + free(ctx_reqs); + free(std_reqs); +} + +static void *worker_main(void *arg) { + http_server *s = arg; +#ifdef USE_MLX + if (s->use_mlx) { + int rc = 0; + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (!m->info) continue; + pplx_mlx_options_t mlx_opts = { + .quantize_bits = s->mlx_quantize_bits, + .quantize_group_size = s->mlx_quantize_group_size, + }; + m->mlx_ctx = pplx_mlx_load_with_options(m->path, &mlx_opts); + if (!m->mlx_ctx) { + server_log("pplx-serve: failed to load MLX model on worker: %s", + m->path); + rc = 1; + break; + } + if (pplx_mlx_config(m->mlx_ctx)->hidden_size != m->info->dim) { + server_log("pplx-serve: model %s has unexpected dimension %d", + m->info->id, + pplx_mlx_config(m->mlx_ctx)->hidden_size); + rc = 1; + break; + } + } + pthread_mutex_lock(&s->mu); + s->worker_init_rc = rc; + s->worker_ready = 1; + if (rc) s->stopping = 1; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + if (rc) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->mlx_ctx) { + pplx_mlx_free(m->mlx_ctx); + m->mlx_ctx = NULL; + } + } + return NULL; + } + } else +#endif +#ifdef USE_CUDA + if (s->use_cuda) { + int rc = 0; + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (!m->info) continue; + m->cuda_ctx = pplx_cuda_load(m->path); + if (!m->cuda_ctx) { + server_log("pplx-serve: failed to load CUDA model on worker: %s", + m->path); + rc = 1; + break; + } + if (pplx_cuda_config(m->cuda_ctx)->hidden_size != m->info->dim) { + server_log("pplx-serve: model %s has unexpected dimension %d", + m->info->id, + pplx_cuda_config(m->cuda_ctx)->hidden_size); + rc = 1; + break; + } + } + pthread_mutex_lock(&s->mu); + s->worker_init_rc = rc; + s->worker_ready = 1; + if (rc) s->stopping = 1; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + if (rc) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->cuda_ctx) { + pplx_cuda_free(m->cuda_ctx); + m->cuda_ctx = NULL; + } + } + return NULL; + } + } else +#endif + { + pthread_mutex_lock(&s->mu); + s->worker_ready = 1; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + } + + for (;;) { + job *j = dequeue_job(s); + if (!j) break; + job *jobs[PPLX_SERVER_MICROBATCH_MAX_JOBS]; + int n_jobs = 1; + jobs[0] = j; + + if ((!strcmp(j->path, "/v1/embeddings") || + !strcmp(j->path, "/v1/contextualizedembeddings")) && + s->batch_size > 1) { + if (s->batch_wait_us > 0) + usleep((useconds_t)s->batch_wait_us); + n_jobs += drain_ready_jobs(s, jobs + 1, + PPLX_SERVER_MICROBATCH_MAX_JOBS - 1); + } + + process_job_group(s, jobs, n_jobs); + } +#ifdef USE_MLX + if (s->use_mlx) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->mlx_ctx) { + pplx_mlx_free(m->mlx_ctx); + m->mlx_ctx = NULL; + } + } + } +#endif +#ifdef USE_CUDA + if (s->use_cuda) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + if (m->cuda_ctx) { + pplx_cuda_free(m->cuda_ctx); + m->cuda_ctx = NULL; + } + } + } +#endif + return NULL; +} + +static int listen_on(const char *host, int port) { + char portbuf[32]; + snprintf(portbuf, sizeof(portbuf), "%d", port); + char err[ANET_ERR_LEN]; + int fd = anetTcpServer(err, port, (char *)host, PPLX_HTTP_BACKLOG); + (void)portbuf; + if (fd == ANET_ERR) { + server_log("pplx-serve: listen failed on %s:%d: %s", host, port, err); + return -1; + } + set_nonblock(fd); + return fd; +} + +static int load_one_model(http_server *s, model_slot slot, const char *path) { + loaded_model *m = &s->models[slot]; + m->info = &k_models[slot]; + m->path = xstrdup(path); + char vocab_path[1024]; + snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); + m->tok = qwen_tokenizer_load(vocab_path); + if (!m->tok) { + server_log("pplx-serve: failed to load tokenizer: %s", vocab_path); + return -1; + } + m->tok_ws = qwen_tokenizer_workspace_new(); + if (!m->tok_ws) { + server_log("pplx-serve: failed to allocate tokenizer workspace: %s", + path); + return -1; + } + m->context_separator_id = PPLX_CONTEXT_SEPARATOR_TOKEN_ID; +#ifdef USE_MLX + if (s->use_mlx) { + /* MLX streams are thread-local. The inference worker loads the MLX + * model so all MLX arrays/streams are created, used, and freed by the + * same thread. */ + return 0; + } +#endif +#ifdef USE_CUDA + if (s->use_cuda) { + return 0; + } +#endif + m->cpu_model = pplx_model_load(path); + if (!m->cpu_model) { + server_log("pplx-serve: failed to load model: %s", path); + return -1; + } + m->cpu_ws = pplx_workspace_new(m->cpu_model); + if (!m->cpu_ws) { + server_log("pplx-serve: failed to allocate workspace: %s", path); + return -1; + } + if (pplx_model_config(m->cpu_model)->hidden_size != m->info->dim) { + server_log("pplx-serve: model %s has unexpected dimension %d", + m->info->id, pplx_model_config(m->cpu_model)->hidden_size); + return -1; + } + return 0; +} + +static void free_models(http_server *s) { + for (int i = 0; i < 4; i++) { + loaded_model *m = &s->models[i]; + free(m->path); + if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws); + if (m->tok) qwen_tokenizer_free(m->tok); + if (m->cpu_ws) pplx_workspace_free(m->cpu_ws); + if (m->cpu_model) pplx_model_free(m->cpu_model); +#ifdef USE_MLX + if (m->mlx_ctx) pplx_mlx_free(m->mlx_ctx); +#endif +#ifdef USE_CUDA + if (m->cuda_ctx) pplx_cuda_free(m->cuda_ctx); +#endif + } +} + +static uint64_t physical_memory_nbytes(void) { +#ifdef __APPLE__ + uint64_t bytes = 0; + size_t len = sizeof(bytes); + if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && + len == sizeof(bytes) && bytes > 0) + return bytes; +#endif + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + if (pages <= 0 || page_size <= 0 || + (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) + return 0; + return (uint64_t)pages * (uint64_t)page_size; +} + +static int validate_model_specs(const pplx_server_config_t *cfg) { + int seen[4] = {0}; + for (int i = 0; i < cfg->n_models; i++) { + const pplx_server_model_spec_t *spec = &cfg->models[i]; + model_slot slot = model_slot_for_id(spec->id); + if (slot == MODEL_UNKNOWN) { + fprintf(stderr, "pplx-serve: unknown model id: %s\n", + spec->id ? spec->id : ""); + return -1; + } + if (!spec->path || !spec->path[0]) { + fprintf(stderr, "pplx-serve: model %s has an empty path\n", + spec->id); + return -1; + } + if (seen[slot]) { + fprintf(stderr, "pplx-serve: duplicate model id: %s\n", spec->id); + return -1; + } + seen[slot] = 1; + } + return 0; +} + +static int mlx_memory_preflight(const pplx_server_config_t *cfg) { + if (!cfg->use_mlx) return 0; + + uint64_t payload = 0; + for (int i = 0; i < cfg->n_models; i++) { + multi_safetensors_t *ms = + multi_safetensors_open(cfg->models[i].path); + size_t model_payload = 0; + if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { + multi_safetensors_close(ms); + fprintf(stderr, "pplx-serve: failed to inspect MLX model: %s\n", + cfg->models[i].path); + return -1; + } + multi_safetensors_close(ms); + if ((uint64_t)model_payload > UINT64_MAX - payload) { + fprintf(stderr, "pplx-serve: MLX model payload size overflow\n"); + return -1; + } + payload += (uint64_t)model_payload; + } + + if (payload > UINT64_MAX / PPLX_MLX_RESIDENT_MULTIPLIER) { + fprintf(stderr, "pplx-serve: MLX resident-memory estimate overflow\n"); + return -1; + } + uint64_t estimated = payload * PPLX_MLX_RESIDENT_MULTIPLIER; + uint64_t physical = physical_memory_nbytes(); + if (physical == 0) { + server_log("pplx-serve: warning: could not determine physical memory; " + "skipping MLX memory preflight"); + return 0; + } + double utilization = cfg->memory_utilization > 0.0 + ? cfg->memory_utilization + : PPLX_MLX_MEMORY_BUDGET_PERCENT / 100.0; + uint64_t budget = (uint64_t)((double)physical * utilization); + const double gib = 1024.0 * 1024.0 * 1024.0; + server_log("pplx-serve: MLX memory preflight: %.1f GiB tensors, " + "%.1f GiB estimated resident, %.1f GiB budget " + "(%.2f of physical memory)", + (double)payload / gib, (double)estimated / gib, + (double)budget / gib, utilization); + if (cfg->mlx_quantize_bits) { + server_log("pplx-serve: MLX %d-bit quantization enabled; preflight " + "uses source payload as a conservative peak estimate", + cfg->mlx_quantize_bits); + } + if (estimated <= budget) return 0; + server_log("pplx-serve: refusing MLX model set above the host-memory " + "budget"); + server_log("pplx-serve: use BF16 artifacts, load fewer models, or raise " + "--memory-utilization (values above 1.0 overcommit)"); + return -1; +} + +int pplx_run_server(const pplx_server_config_t *cfg) { + if (!cfg || !cfg->models || cfg->n_models <= 0) { + fprintf(stderr, "pplx-serve: at least one --model MODEL_ID=PATH is required\n"); + return 1; + } + if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0) + return 1; + + http_server s; + memset(&s, 0, sizeof(s)); + s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; + s.port = cfg->port > 0 ? cfg->port : 8000; + s.batch_size = cfg->batch_size > 0 + ? cfg->batch_size : PPLX_SERVER_DEFAULT_BATCH_SIZE; + s.max_batch_tokens = cfg->max_batch_tokens > 0 + ? cfg->max_batch_tokens : PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS; + s.batch_wait_us = cfg->batch_wait_us >= 0 + ? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US; + s.enable_cors = cfg->enable_cors; + s.use_mlx = cfg->use_mlx; + s.use_cuda = cfg->use_cuda; + s.mlx_quantize_bits = cfg->mlx_quantize_bits; + s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 + ? cfg->mlx_quantize_group_size : 64; + if (cfg->api_key && cfg->api_key[0]) + s.api_key = xstrdup(cfg->api_key); + else { + const char *env_key = getenv("PPLX_API_KEY"); + if (env_key && env_key[0]) s.api_key = xstrdup(env_key); + } + + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + for (int i = 0; i < cfg->n_models; i++) { + model_slot slot = model_slot_for_id(cfg->models[i].id); + if (slot == MODEL_UNKNOWN) { + fprintf(stderr, "pplx-serve: unknown model id: %s\n", cfg->models[i].id); + free_models(&s); + return 1; + } + if (load_one_model(&s, slot, cfg->models[i].path) != 0) { + free_models(&s); + return 1; + } + } + + if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { + perror("pipe"); + free_models(&s); + return 1; + } + set_nonblock(s.completion_pipe[0]); + set_nonblock(s.completion_pipe[1]); + set_nonblock(s.signal_pipe[0]); + set_nonblock(s.signal_pipe[1]); + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = stop_signal_handler; + sigemptyset(&sa.sa_mask); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + signal(SIGPIPE, SIG_IGN); + g_signal_wfd = s.signal_pipe[1]; + + s.loop = aeCreateEventLoop(PPLX_HTTP_SETSIZE); + if (!s.loop) { + fprintf(stderr, "pplx-serve: failed to create event loop\n"); + free_models(&s); + return 1; + } + + if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { + fprintf(stderr, "pplx-serve: failed to start worker\n"); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + pthread_mutex_lock(&s.mu); + while (!s.worker_ready) + pthread_cond_wait(&s.cv, &s.mu); + int worker_init_rc = s.worker_init_rc; + pthread_mutex_unlock(&s.mu); + if (worker_init_rc) { + pthread_join(s.worker, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 1; + } + + s.listen_fd = listen_on(s.host, s.port); + if (s.listen_fd < 0) { + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + + aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); + aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); + aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); + aeCreateTimeEvent(s.loop, PPLX_HTTP_SWEEP_MS, timeout_cb, &s, NULL); + + server_log("pplx-serve: listening on http://%s:%d", s.host, s.port); + aeMain(s.loop); + + server_log("pplx-serve: shutdown requested"); + if (s.listen_fd >= 0) close(s.listen_fd); + client *c = s.clients; + while (c) { + client *next = c->next; + close_client(c); + c = next; + } + + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + + aeDeleteEventLoop(s.loop); + close(s.completion_pipe[0]); + close(s.completion_pipe[1]); + close(s.signal_pipe[0]); + close(s.signal_pipe[1]); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 0; +} diff --git a/embed_server.h b/embed_server.h new file mode 100644 index 0000000..c38f0ca --- /dev/null +++ b/embed_server.h @@ -0,0 +1,32 @@ +#ifndef PPLX_SERVER_H +#define PPLX_SERVER_H + +#include + +typedef struct { + const char *id; + const char *path; +} pplx_server_model_spec_t; + +typedef struct { + const pplx_server_model_spec_t *models; + int n_models; + const char *host; + int port; + int batch_size; + int max_batch_tokens; + int batch_wait_us; + int use_mlx; + int use_cuda; + int mlx_quantize_bits; + int mlx_quantize_group_size; + double memory_utilization; /* fraction of physical memory the MLX + preflight may plan for; 0 = default 0.90, + values above 1.0 allow overcommit */ + int enable_cors; + const char *api_key; +} pplx_server_config_t; + +int pplx_run_server(const pplx_server_config_t *cfg); + +#endif diff --git a/main.c b/main.c deleted file mode 100644 index 140494b..0000000 --- a/main.c +++ /dev/null @@ -1,825 +0,0 @@ -/* main.c - pplx_embed command-line tool */ - -#include "pplx_embed.h" -#include "pplx_distributed.h" -#include "qwen_kernels.h" -#include "qwen_tokenizer.h" - -#ifdef USE_MLX -#include "pplx_embed_mlx.h" -#endif -#ifdef USE_CUDA -#include "pplx_embed_cuda.h" -#endif - -#include -#include -#include -#include -#include - -static double now_ms(void) -{ - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6; -} - -static void print_usage(const char *prog) -{ - fprintf(stderr, - "Usage: %s -d [options] [text...]\n" - "\n" - "Options:\n" - " -d Model directory (required)\n" - " --mlx Use Apple MLX GPU backend\n" - " --cuda Use CUDA/cuBLAS GPU backend\n" - " --stream Read lines from stdin, write one JSON embedding per line\n" - " --dist-worker\n" - " Serve one final distributed layer shard\n" - " --dist-remote HOST:PORT\n" - " Use one remote final shard for CLI or --stream inference\n" - " --layers A:B Distributed half-open layer range, for example 0:14\n" - " --dist-activation-bits N\n" - " Distributed activation transport width: 32 or 16 (default: 32)\n" - " --mlx-quant-bits N\n" - " Quantize MLX linear weights to 8 bits at load time\n" - " --mlx-quant-group-size N\n" - " MLX quantization group size (default: 64)\n" - " --cuda-gemm-mode MODE\n" - " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" - " --cuda-weight-dtype DTYPE\n" - " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" - " weight memory and uses BF16 tensor cores\n" - " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n" - " --port N Bind port for --dist-worker (default: 8000)\n" - " -b, --batch-size N\n" - " Max texts per engine batch (default: all; --stream: 1)\n" - " -t, --threads N\n" - " CPU threads (default: all cores)\n" - " -e, --embeddings\n" - " Print raw embeddings (with multiple texts)\n" - " -v, --verbose\n" - " Verbose (-vv for debug)\n" - " -h Show this help\n" - "\n" - "Modes:\n" - " 1 text arg Print embedding as space-separated floats\n" - " 2+ text args Print cosine similarity matrix\n" - " no args Batch: read all stdin lines, then similarity matrix\n" - " --stream Streaming: read stdin lines, write JSON per line\n" - " --dist-worker Final layer-shard worker\n" - "\n" - "The HTTP API is served by the separate pplx-embed-server binary.\n" - "\n" - "Examples:\n" - " %s -d ./model \"query: what is AI?\"\n" - " %s -d ./model --mlx --stream < texts.txt\n" - " %s -d ./model --dist-worker --layers 14:28 --port 9000\n" - " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n", - prog, prog, prog, prog, prog); -} - -/* ======================================================================== - * Embed one text: tokenize > forward > return float[dim] - * ======================================================================== */ - -typedef struct { - pplx_model_t *model; - pplx_workspace_t *workspace; - pplx_dist_client_t *dist; -#ifdef USE_MLX - pplx_mlx_ctx_t *mlx_ctx; -#endif -#ifdef USE_CUDA - pplx_cuda_ctx_t *cuda_ctx; -#endif - qwen_tokenizer_t *tok; - qwen_tokenizer_workspace_t *tok_ws; - int dim; -} engine_t; - -typedef struct { - int *ids; - int n_tokens; -} token_buf_t; - -static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, - int batch, float *out_embeddings) -{ - if (e->dist) - return pplx_dist_client_embed_batch(e->dist, inputs, batch, - out_embeddings); -#ifdef USE_MLX - if (e->mlx_ctx) - return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings); -#endif -#ifdef USE_CUDA - if (e->cuda_ctx) - return pplx_cuda_embed_batch(e->cuda_ctx, inputs, batch, out_embeddings); -#endif - return pplx_model_embed_batch(e->model, e->workspace, - inputs, batch, out_embeddings); -} - -static int parse_layers(const char *arg, int *start, int *end) -{ - int used = 0; - if (!arg || sscanf(arg, "%d:%d%n", start, end, &used) != 2 || - arg[used] != '\0' || *start < 0 || *start >= *end) - return -1; - return 0; -} - -static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port) -{ - const char *colon = arg ? strrchr(arg, ':') : NULL; - if (!colon || colon == arg || !colon[1]) return -1; - size_t n = (size_t)(colon - arg); - if (n >= hostlen) return -1; - memcpy(host, arg, n); - host[n] = '\0'; - int used = 0; - if (sscanf(colon + 1, "%d%n", port, &used) != 1 || - colon[1 + used] != '\0' || *port <= 0 || *port > 65535) - return -1; - return 0; -} - -static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) -{ - memset(out, 0, sizeof(*out)); - out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, - text, &out->n_tokens); - if (!out->ids || out->n_tokens == 0) { - fprintf(stderr, "tokenization failed: %s\n", text); - free(out->ids); - out->ids = NULL; - out->n_tokens = 0; - return -1; - } - - if (pplx_verbose >= 1) { - fprintf(stderr, "tokens (%d): ", out->n_tokens); - for (int i = 0; i < out->n_tokens && i < 20; i++) - fprintf(stderr, "%d ", out->ids[i]); - if (out->n_tokens > 20) fprintf(stderr, "..."); - fprintf(stderr, "\n"); - } - return 0; -} - -static void free_tokens(token_buf_t *tokens, int n) -{ - if (!tokens) return; - for (int i = 0; i < n; i++) - free(tokens[i].ids); -} - -/* ======================================================================== - * Output helpers - * ======================================================================== */ - -static void print_embedding_raw(const float *emb, int dim) -{ - for (int i = 0; i < dim; i++) { - if (i > 0) putchar(' '); - printf("%.8f", (double)emb[i]); - } - putchar('\n'); -} - -static size_t engine_workspace_nbytes(const engine_t *e) -{ - return (e && e->workspace) ? pplx_workspace_nbytes(e->workspace) : 0; -} - -static void print_embedding_json(const float *emb, int dim, int n_tokens, - double ms, size_t workspace_bytes) -{ - printf("{\"embedding\":["); - for (int i = 0; i < dim; i++) { - if (i > 0) putchar(','); - printf("%.8f", (double)emb[i]); - } - printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", - dim, n_tokens, ms, workspace_bytes); - fflush(stdout); -} - -/* ======================================================================== - * Stdin mode: read lines from stdin, write JSON to stdout - * ======================================================================== */ - -static int process_stdin_batch(engine_t *e, char **lines, int n_lines) -{ - if (n_lines <= 0) return 0; - - token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t)); - pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_lines * sizeof(pplx_input_t)); - int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int)); - float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float)); - if (!tokens || !inputs || !input_to_line || !embs) { - fprintf(stderr, "OOM\n"); - free(tokens); free(inputs); free(input_to_line); free(embs); - for (int i = 0; i < n_lines; i++) - printf("{\"error\":\"embedding failed\"}\n"); - fflush(stdout); - return 1; - } - - int n_inputs = 0; - for (int i = 0; i < n_lines; i++) { - if (tokenize_text(e, lines[i], &tokens[i]) == 0) { - inputs[n_inputs].ids = tokens[i].ids; - inputs[n_inputs].n_tokens = tokens[i].n_tokens; - input_to_line[n_inputs] = i; - n_inputs++; - } - } - - double ms = 0.0; - int rc = 0; - if (n_inputs > 0) { - double t0 = now_ms(); - rc = engine_embed_batch(e, inputs, n_inputs, embs); - ms = now_ms() - t0; - if (pplx_verbose >= 1) - fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms); - } - - int next_valid = 0; - for (int i = 0; i < n_lines; i++) { - if (!tokens[i].ids) { - printf("{\"error\":\"tokenization failed\"}\n"); - continue; - } - if (rc != 0) { - printf("{\"error\":\"embedding failed\"}\n"); - continue; - } - while (next_valid < n_inputs && input_to_line[next_valid] != i) - next_valid++; - if (next_valid >= n_inputs) { - printf("{\"error\":\"embedding failed\"}\n"); - continue; - } - print_embedding_json(embs + (size_t)next_valid * e->dim, - e->dim, tokens[i].n_tokens, ms, - engine_workspace_nbytes(e)); - next_valid++; - } - fflush(stdout); - - free_tokens(tokens, n_lines); - free(tokens); free(inputs); free(input_to_line); free(embs); - return rc == 0 ? 0 : 1; -} - -static int run_stdin(engine_t *e, int batch_size) -{ - char line[65536]; - if (batch_size <= 0) batch_size = 1; - - if (pplx_verbose >= 1) - fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", - batch_size); - - char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *)); - if (!batch_lines) { - fprintf(stderr, "OOM\n"); - return 1; - } - int n_batch = 0; - int rc = 0; - - while (fgets(line, sizeof(line), stdin)) { - /* Strip newline */ - size_t len = strlen(line); - while (len > 0 && (line[len-1] == '\n' || line[len-1] == '\r')) - line[--len] = '\0'; - if (len == 0) continue; - - if (pplx_verbose >= 1) - fprintf(stderr, "stdin: \"%.*s%s\"\n", - (int)(len > 60 ? 60 : len), line, len > 60 ? "..." : ""); - - batch_lines[n_batch] = strdup(line); - if (!batch_lines[n_batch]) { - fprintf(stderr, "OOM\n"); - rc = 1; - break; - } - n_batch++; - - if (n_batch == batch_size) { - if (process_stdin_batch(e, batch_lines, n_batch) != 0) - rc = 1; - for (int i = 0; i < n_batch; i++) { - free(batch_lines[i]); - batch_lines[i] = NULL; - } - n_batch = 0; - } - } - - if (n_batch > 0) { - if (process_stdin_batch(e, batch_lines, n_batch) != 0) - rc = 1; - for (int i = 0; i < n_batch; i++) - free(batch_lines[i]); - } - free(batch_lines); - - if (pplx_verbose >= 1) - fprintf(stderr, "stdin: EOF\n"); - - return rc; -} - -/* ======================================================================== - * Batch mode: embed args or stdin lines, then print similarity or vectors - * ======================================================================== */ - -static int append_text(char ***texts, int *n_texts, int *cap, const char *s) -{ - if (*n_texts == *cap) { - int new_cap = *cap ? *cap * 2 : 8; - char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *)); - if (!new_texts) return -1; - *texts = new_texts; - *cap = new_cap; - } - - (*texts)[*n_texts] = strdup(s); - if (!(*texts)[*n_texts]) return -1; - (*n_texts)++; - return 0; -} - -static int run_batch(engine_t *e, int argc, char **argv, int arg_start, - int print_embs, int batch_size) -{ - int n_texts = 0, cap = 0; - char **texts = NULL; - int rc = 0; - - if (arg_start < argc) { - for (int i = arg_start; i < argc; i++) { - if (append_text(&texts, &n_texts, &cap, argv[i]) != 0) - goto oom_texts; - } - } else { - char line[65536]; - while (fgets(line, sizeof(line), stdin)) { - size_t l = strlen(line); - while (l > 0 && (line[l-1] == '\n' || line[l-1] == '\r')) - line[--l] = '\0'; - if (l == 0) continue; - if (append_text(&texts, &n_texts, &cap, line) != 0) - goto oom_texts; - } - } - - if (n_texts == 0) { - fprintf(stderr, "no input texts\n"); - free(texts); - return 1; - } - - int dim = e->dim; - token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t)); - pplx_input_t *inputs = (pplx_input_t *)malloc((size_t)n_texts * sizeof(pplx_input_t)); - float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float)); - if (!tokens || !inputs || !embs) { - fprintf(stderr, "OOM\n"); - free(tokens); free(inputs); free(embs); - for (int i = 0; i < n_texts; i++) free(texts[i]); - free(texts); - return 1; - } - - for (int i = 0; i < n_texts; i++) { - if (pplx_verbose >= 1) - fprintf(stderr, "[%d/%d] \"%s\"\n", i+1, n_texts, texts[i]); - if (tokenize_text(e, texts[i], &tokens[i]) != 0) { - rc = 1; - goto done; - } - inputs[i].ids = tokens[i].ids; - inputs[i].n_tokens = tokens[i].n_tokens; - } - - int max_batch = batch_size > 0 ? batch_size : n_texts; - for (int start = 0; start < n_texts; start += max_batch) { - int cur = n_texts - start; - if (cur > max_batch) cur = max_batch; - - double t0 = now_ms(); - if (engine_embed_batch(e, inputs + start, cur, - embs + (size_t)start * dim) != 0) { - fprintf(stderr, "forward pass failed\n"); - rc = 1; - goto done; - } - if (pplx_verbose >= 1) - fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", - start, start + cur - 1, cur, now_ms() - t0); - } - - if (n_texts == 1) { - print_embedding_raw(embs, dim); - } else { - printf("Cosine similarity matrix (%d texts):\n", n_texts); - printf("%-4s", ""); - for (int j = 0; j < n_texts; j++) printf(" [%d] ", j); - printf("\n"); - for (int i = 0; i < n_texts; i++) { - printf("[%d] ", i); - for (int j = 0; j < n_texts; j++) - printf(" %.4f", (double)pplx_cosine_similarity( - embs + (size_t)i * dim, embs + (size_t)j * dim, dim)); - printf(" \"%s\"\n", texts[i]); - } - if (print_embs) { - printf("\nEmbeddings:\n"); - for (int i = 0; i < n_texts; i++) { - printf("[%d] ", i); - print_embedding_raw(embs + (size_t)i * dim, dim); - } - } - } - -done: - free_tokens(tokens, n_texts); - for (int i = 0; i < n_texts; i++) free(texts[i]); - free(texts); free(tokens); free(inputs); free(embs); - return rc; - -oom_texts: - fprintf(stderr, "OOM\n"); - for (int i = 0; i < n_texts; i++) free(texts[i]); - free(texts); - return 1; -} - -/* ======================================================================== - * Main - * ======================================================================== */ - -int main(int argc, char *argv[]) -{ - const char *model_dir = NULL; - int n_threads = 0; - int print_embs = 0; - int verbose = 0; - int use_mlx = 0; - int use_cuda = 0; - int stdin_mode = 0; - int dist_worker_mode = 0; - const char *dist_remote = NULL; - int layer_start = -1; - int layer_end = -1; - int batch_size = 0; - int mlx_quantize_bits = 0; - int mlx_quantize_group_size = 64; - const char *cuda_fast_gemm = NULL; - const char *cuda_weights = NULL; - int dist_activation_bits = 32; - const char *host = "127.0.0.1"; - int port = 8000; - - int arg_start = 1; - while (arg_start < argc && argv[arg_start][0] == '-') { - const char *f = argv[arg_start]; - if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; } - else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { - n_threads = atoi(argv[++arg_start]); - } - else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { - batch_size = atoi(argv[++arg_start]); - } - else if (!strcmp(f, "--backend")) { - const char *backend = argv[++arg_start]; - if (!strcmp(backend, "mlx")) { -#ifdef USE_MLX - use_mlx = 1; - use_cuda = 0; -#else - fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); - return 1; -#endif - } else if (!strcmp(backend, "cuda")) { -#ifdef USE_CUDA - use_cuda = 1; - use_mlx = 0; -#else - fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); - return 1; -#endif - } else if (!strcmp(backend, "cpu")) { - use_mlx = 0; - use_cuda = 0; - } else { - fprintf(stderr, "invalid --backend: %s\n", backend); - return 1; - } - } - else if (!strcmp(f, "--mlx-quant-bits")) { - mlx_quantize_bits = atoi(argv[++arg_start]); - if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); - return 1; - } - } - else if (!strcmp(f, "--mlx-quant-group-size")) { - mlx_quantize_group_size = atoi(argv[++arg_start]); - if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); - return 1; - } - } - else if (!strcmp(f, "--cuda-gemm-mode")) { - cuda_fast_gemm = argv[++arg_start]; - } - else if (!strcmp(f, "--cuda-weight-dtype")) { - cuda_weights = argv[++arg_start]; - } - else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } - else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } - else if (!strcmp(f, "--layers")) { - if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) { - fprintf(stderr, "--layers expects START:END with START < END\n"); - return 1; - } - } - else if (!strcmp(f, "--dist-activation-bits")) { - dist_activation_bits = atoi(argv[++arg_start]); - if (dist_activation_bits != 32 && dist_activation_bits != 16) { - fprintf(stderr, "--dist-activation-bits must be 32 or 16\n"); - return 1; - } - } - else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { - print_embs = 1; - } - else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } - else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "--stream")) { stdin_mode = 1; } - else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; } - else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; } - else if (!strcmp(f, "--mlx")) { -#ifdef USE_MLX - use_mlx = 1; - use_cuda = 0; -#else - fprintf(stderr, "--mlx not available (build with: make mlx)\n"); - return 1; -#endif - } - else if (!strcmp(f, "--cuda")) { -#ifdef USE_CUDA - use_cuda = 1; - use_mlx = 0; -#else - fprintf(stderr, "--cuda not available (build with: make cuda)\n"); - return 1; -#endif - } - else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { - print_usage(argv[0]); - return 0; - } - else break; - arg_start++; - } - - pplx_verbose = verbose; - qwen_verbose = verbose; - - if (cuda_fast_gemm) { -#ifdef USE_CUDA - if (!use_cuda) { - fprintf(stderr, "--cuda-gemm-mode requires --cuda or --backend cuda\n"); - return 1; - } - if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { - fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); - return 1; - } -#else - fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); - return 1; -#endif - } - - if (cuda_weights) { -#ifdef USE_CUDA - if (!use_cuda) { - fprintf(stderr, "--cuda-weight-dtype requires --cuda or --backend cuda\n"); - return 1; - } - if (!strcmp(cuda_weights, "bf16")) { - pplx_cuda_set_weights_bf16(1); - } else if (!strcmp(cuda_weights, "f32")) { - pplx_cuda_set_weights_bf16(0); - } else { - fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); - return 1; - } -#else - fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); - return 1; -#endif - } - - if (!model_dir || !model_dir[0]) { - fprintf(stderr, "model directory required (-d )\n"); - print_usage(argv[0]); - return 1; - } - - if (dist_worker_mode && dist_remote) { - fprintf(stderr, "--dist-worker and --dist-remote are mutually exclusive\n"); - return 1; - } - if ((dist_worker_mode || dist_remote) && - (layer_start < 0 || layer_end < 0)) { - fprintf(stderr, "distributed mode requires --layers START:END\n"); - return 1; - } - if (dist_activation_bits != 32 && !dist_remote) { - fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n"); - return 1; - } - if (dist_remote && layer_start != 0) { - fprintf(stderr, "--dist-remote coordinator layers must start at zero\n"); - return 1; - } - if ((dist_worker_mode || dist_remote) && use_cuda) { - fprintf(stderr, "distributed mode is not available with --cuda yet\n"); - return 1; - } - - /* Threads (CPU backend) */ - if (!use_mlx && !use_cuda) { - if (n_threads <= 0) n_threads = qwen_get_num_cpus(); - qwen_set_threads(n_threads); - if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } else { - if (verbose >= 1) - fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); - } - - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quant-bits requires --mlx or --backend mlx\n"); - return 1; - } - - pplx_dist_options_t dist_opts = { - .mlx_quantize_bits = mlx_quantize_bits, - .mlx_quantize_group_size = mlx_quantize_group_size, - .activation_bits = dist_activation_bits, - }; - - if (dist_worker_mode) - return pplx_dist_run_worker_with_options( - model_dir, host, port, layer_start, layer_end, use_mlx, - &dist_opts); - - /* Tokenizer */ - char vocab_path[1024]; - snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); - double t0 = now_ms(); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); - if (!tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; } - if (verbose >= 1) fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0); - - /* Model */ - engine_t e = {0}; - e.tok = tok; - t0 = now_ms(); - -#ifdef USE_MLX - if (dist_remote) { - char remote_host[1024]; - int remote_port = 0; - if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host), - &remote_port) != 0) { - fprintf(stderr, "--dist-remote expects HOST:PORT\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dist = pplx_dist_client_open_with_options( - model_dir, remote_host, remote_port, layer_end, use_mlx, - &dist_opts); - if (!e.dist) { - fprintf(stderr, "failed to connect distributed worker\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_dist_client_config(e.dist)->hidden_size; - } else if (use_mlx) { - pplx_mlx_options_t mlx_opts = { - .quantize_bits = mlx_quantize_bits, - .quantize_group_size = mlx_quantize_group_size, - }; - e.mlx_ctx = pplx_mlx_load_with_options(model_dir, &mlx_opts); - if (!e.mlx_ctx) { - fprintf(stderr, "failed to load model\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_mlx_config(e.mlx_ctx)->hidden_size; - } else -#endif -#ifdef USE_CUDA - if (use_cuda) { - e.cuda_ctx = pplx_cuda_load(model_dir); - if (!e.cuda_ctx) { - fprintf(stderr, "failed to load CUDA model\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_cuda_config(e.cuda_ctx)->hidden_size; - } else -#endif - { - if (dist_remote) { - char remote_host[1024]; - int remote_port = 0; - if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host), - &remote_port) != 0) { - fprintf(stderr, "--dist-remote expects HOST:PORT\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dist = pplx_dist_client_open_with_options( - model_dir, remote_host, remote_port, layer_end, use_mlx, - &dist_opts); - if (!e.dist) { - fprintf(stderr, "failed to connect distributed worker\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_dist_client_config(e.dist)->hidden_size; - } else { - e.model = pplx_model_load(model_dir); - if (!e.model) { - fprintf(stderr, "failed to load model\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.workspace = pplx_workspace_new(e.model); - if (!e.workspace) { - fprintf(stderr, "failed to allocate workspace\n"); - pplx_model_free(e.model); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_model_config(e.model)->hidden_size; - } - } - if (verbose >= 1) - fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n", - e.dim, now_ms() - t0, - use_mlx ? " (MLX)" : "", - use_cuda ? " (CUDA)" : ""); - - e.tok_ws = qwen_tokenizer_workspace_new(); - if (!e.tok_ws) { - fprintf(stderr, "failed to allocate tokenizer workspace\n"); - if (e.workspace) pplx_workspace_free(e.workspace); - if (e.model) pplx_model_free(e.model); - if (e.dist) pplx_dist_client_free(e.dist); -#ifdef USE_MLX - if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); -#endif -#ifdef USE_CUDA - if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); -#endif - qwen_tokenizer_free(tok); - return 1; - } - - /* Run */ - int rc; - if (stdin_mode) - rc = run_stdin(&e, batch_size > 0 ? batch_size : 1); - else - rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size); - - /* Cleanup */ - if (e.workspace) pplx_workspace_free(e.workspace); - if (e.model) pplx_model_free(e.model); - if (e.dist) pplx_dist_client_free(e.dist); -#ifdef USE_MLX - if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); -#endif -#ifdef USE_CUDA - if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); -#endif - qwen_tokenizer_workspace_free(e.tok_ws); - qwen_tokenizer_free(tok); - return rc; -} diff --git a/pplx_distributed.c b/pplx_distributed.c deleted file mode 100644 index d4254f7..0000000 --- a/pplx_distributed.c +++ /dev/null @@ -1,888 +0,0 @@ -/* - * pplx_distributed.c - Stateless two-node layer-sharded inference. - * - * This first transport is deliberately synchronous: one coordinator sends a - * packed hidden-state batch to one final worker and receives pooled - * embeddings. The protocol is bounded and versioned so later route discovery, - * retries, and richer transport can extend it without changing model math. - */ - -#include "pplx_distributed.h" - -#ifdef USE_MLX -#include "pplx_embed_mlx.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define PPLX_DIST_MAGIC UINT32_C(0x504c5844) /* "PLXD" */ -#define PPLX_DIST_VERSION 2u - -#define PPLX_DIST_MSG_HELLO 1u -#define PPLX_DIST_MSG_WORK 2u -#define PPLX_DIST_MSG_RESULT 3u - -#define PPLX_DIST_MAX_FRAME_BYTES (512u * 1024u * 1024u) -#define PPLX_DIST_MAX_BATCH 512u -#define PPLX_DIST_MAX_TOKENS 120000u -#define PPLX_DIST_MAX_ERROR_BYTES 1024u -#define PPLX_DIST_SOCKET_TIMEOUT_SEC 60 -#define PPLX_DIST_ACTIVATION_BITS_DEFAULT 32u - -typedef struct { - uint32_t magic; - uint32_t version; - uint32_t type; - uint32_t body_bytes; -} dist_frame_header; - -typedef struct { - uint32_t signature_hi; - uint32_t signature_lo; - uint32_t hidden_size; - uint32_t n_layers; - uint32_t vocab_size; - uint32_t n_heads; - uint32_t n_kv_heads; - uint32_t head_dim; - uint32_t intermediate_size; - uint32_t layer_start; - uint32_t layer_end; -} dist_hello; - -typedef struct { - uint32_t request_hi; - uint32_t request_lo; - uint32_t batch; - uint32_t total_tokens; - uint32_t hidden_size; - uint32_t lengths_bytes; - uint32_t states_bytes; - uint32_t states_bits; -} dist_work; - -typedef struct { - uint32_t request_hi; - uint32_t request_lo; - uint32_t status; - uint32_t batch; - uint32_t dimensions; - uint32_t payload_bytes; -} dist_result; - -typedef struct { - pplx_model_t *model; - pplx_workspace_t *workspace; -#ifdef USE_MLX - pplx_mlx_ctx_t *mlx; -#endif - const pplx_config_t *cfg; -} dist_stage; - -struct pplx_dist_client { - dist_stage local; - int fd; - int local_layer_end; - uint64_t next_request_id; - uint32_t activation_bits; -}; - -static int host_is_little_endian(void) -{ - const uint16_t one = 1; - return *(const unsigned char *)&one == 1; -} - -static uint64_t fnv_u32(uint64_t h, uint32_t v) -{ - for (int i = 0; i < 4; i++) { - h ^= (unsigned char)(v >> (i * 8)); - h *= UINT64_C(1099511628211); - } - return h; -} - -static uint64_t config_signature(const pplx_config_t *cfg) -{ - uint64_t h = UINT64_C(1469598103934665603); - uint32_t eps = 0, theta = 0; - memcpy(&eps, &cfg->rms_norm_eps, sizeof(eps)); - memcpy(&theta, &cfg->rope_theta, sizeof(theta)); - h = fnv_u32(h, (uint32_t)cfg->hidden_size); - h = fnv_u32(h, (uint32_t)cfg->n_layers); - h = fnv_u32(h, (uint32_t)cfg->n_heads); - h = fnv_u32(h, (uint32_t)cfg->n_kv_heads); - h = fnv_u32(h, (uint32_t)cfg->head_dim); - h = fnv_u32(h, (uint32_t)cfg->intermediate_size); - h = fnv_u32(h, (uint32_t)cfg->vocab_size); - h = fnv_u32(h, eps); - h = fnv_u32(h, theta); - return h; -} - -static void u64_to_halves(uint64_t v, uint32_t *hi, uint32_t *lo) -{ - *hi = (uint32_t)(v >> 32); - *lo = (uint32_t)v; -} - -static uint64_t u64_from_halves(uint32_t hi, uint32_t lo) -{ - return ((uint64_t)hi << 32) | lo; -} - -static int mul_u32(uint32_t a, uint32_t b, uint32_t *out) -{ - if (a != 0 && b > UINT32_MAX / a) return -1; - *out = a * b; - return 0; -} - -static uint32_t activation_bits_or_default(uint32_t bits) -{ - return bits ? bits : PPLX_DIST_ACTIVATION_BITS_DEFAULT; -} - -static int activation_bits_valid(uint32_t bits) -{ - bits = activation_bits_or_default(bits); - return bits == 32u || bits == 16u; -} - -static int activation_wire_bytes(uint32_t bits, uint64_t values, uint32_t *out) -{ - bits = activation_bits_or_default(bits); - if (!activation_bits_valid(bits)) return -1; - uint64_t bytes = values * (uint64_t)(bits / 8u); - if (bytes > UINT32_MAX) return -1; - if (out) *out = (uint32_t)bytes; - return 0; -} - -static uint16_t f32_to_f16(float f) -{ - uint32_t bits; - memcpy(&bits, &f, sizeof(bits)); - - uint32_t sign = (bits >> 16) & 0x8000u; - int32_t exp = (int32_t)((bits >> 23) & 0xffu) - 127 + 15; - uint32_t mant = bits & 0x7fffffu; - - if (exp <= 0) { - if (exp < -10) return (uint16_t)sign; - mant |= 0x800000u; - uint32_t shift = (uint32_t)(14 - exp); - uint32_t half_mant = mant >> shift; - uint32_t round_bit = (mant >> (shift - 1)) & 1u; - uint32_t sticky = mant & ((1u << (shift - 1)) - 1u); - if (round_bit && (sticky || (half_mant & 1u))) half_mant++; - return (uint16_t)(sign | half_mant); - } - - if (exp >= 31) { - if (((bits >> 23) & 0xffu) == 0xffu && mant != 0) - return (uint16_t)(sign | 0x7e00u); - return (uint16_t)(sign | 0x7c00u); - } - - uint32_t half = sign | ((uint32_t)exp << 10) | (mant >> 13); - uint32_t round = mant & 0x1fffu; - if (round > 0x1000u || (round == 0x1000u && (half & 1u))) half++; - return (uint16_t)half; -} - -static float f16_to_f32(uint16_t h) -{ - uint32_t sign = (uint32_t)(h & 0x8000u) << 16; - int32_t exp = (int32_t)((h >> 10) & 0x1fu); - uint32_t mant = h & 0x03ffu; - uint32_t bits; - - if (exp == 0) { - if (mant == 0) { - bits = sign; - } else { - exp = 1; - while ((mant & 0x0400u) == 0) { - mant <<= 1; - exp--; - } - mant &= 0x03ffu; - bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13); - } - } else if (exp == 31) { - bits = sign | 0x7f800000u | (mant << 13); - } else { - bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13); - } - - float f; - memcpy(&f, &bits, sizeof(f)); - return f; -} - -static int encode_activations(void *dst, const float *src, uint64_t values, - uint32_t bits) -{ - bits = activation_bits_or_default(bits); - if (!dst || !src || !activation_bits_valid(bits)) return -1; - if (bits == 32u) { - memcpy(dst, src, values * sizeof(float)); - return 0; - } - uint16_t *out = dst; - for (uint64_t i = 0; i < values; i++) out[i] = f32_to_f16(src[i]); - return 0; -} - -static int decode_activations(float *dst, const void *wire, uint64_t values, - uint32_t bits) -{ - bits = activation_bits_or_default(bits); - if (!dst || !wire || !activation_bits_valid(bits)) return -1; - if (bits == 32u) { - memcpy(dst, wire, values * sizeof(float)); - return 0; - } - const uint16_t *src = wire; - for (uint64_t i = 0; i < values; i++) dst[i] = f16_to_f32(src[i]); - return 0; -} - -static void frame_to_wire(dist_frame_header *h) -{ - h->magic = htonl(h->magic); - h->version = htonl(h->version); - h->type = htonl(h->type); - h->body_bytes = htonl(h->body_bytes); -} - -static void hello_to_wire(dist_hello *h) -{ - uint32_t *v = (uint32_t *)h; - for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = htonl(v[i]); -} - -static void hello_from_wire(dist_hello *h) -{ - uint32_t *v = (uint32_t *)h; - for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = ntohl(v[i]); -} - -static void work_to_wire(dist_work *w) -{ - uint32_t *v = (uint32_t *)w; - for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = htonl(v[i]); -} - -static void work_from_wire(dist_work *w) -{ - uint32_t *v = (uint32_t *)w; - for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = ntohl(v[i]); -} - -static void result_to_wire(dist_result *r) -{ - uint32_t *v = (uint32_t *)r; - for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = htonl(v[i]); -} - -static void result_from_wire(dist_result *r) -{ - uint32_t *v = (uint32_t *)r; - for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = ntohl(v[i]); -} - -static int write_full(int fd, const void *buf, size_t len) -{ - const unsigned char *p = buf; - while (len) { -#ifdef MSG_NOSIGNAL - ssize_t n = send(fd, p, len, MSG_NOSIGNAL); -#else - ssize_t n = send(fd, p, len, 0); -#endif - if (n < 0 && errno == EINTR) continue; - if (n <= 0) return -1; - p += (size_t)n; - len -= (size_t)n; - } - return 0; -} - -static int read_full(int fd, void *buf, size_t len) -{ - unsigned char *p = buf; - while (len) { - ssize_t n = recv(fd, p, len, 0); - if (n < 0 && errno == EINTR) continue; - if (n <= 0) return -1; - p += (size_t)n; - len -= (size_t)n; - } - return 0; -} - -static int write_frame_header(int fd, uint32_t type, uint32_t body_bytes) -{ - dist_frame_header h = {PPLX_DIST_MAGIC, PPLX_DIST_VERSION, type, body_bytes}; - frame_to_wire(&h); - return write_full(fd, &h, sizeof(h)); -} - -static int read_frame_header(int fd, uint32_t *type, uint32_t *body_bytes) -{ - dist_frame_header h; - if (read_full(fd, &h, sizeof(h)) != 0) return -1; - h.magic = ntohl(h.magic); - h.version = ntohl(h.version); - h.type = ntohl(h.type); - h.body_bytes = ntohl(h.body_bytes); - if (h.magic != PPLX_DIST_MAGIC || h.version != PPLX_DIST_VERSION || - h.body_bytes > PPLX_DIST_MAX_FRAME_BYTES) - return -1; - *type = h.type; - *body_bytes = h.body_bytes; - return 0; -} - -static void set_socket_options(int fd) -{ - int yes = 1; - struct timeval tv = {PPLX_DIST_SOCKET_TIMEOUT_SEC, 0}; - setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); - setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)); - setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); -#ifdef SO_NOSIGPIPE - setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &yes, sizeof(yes)); -#endif -} - -static int open_listener(const char *host, int port) -{ - char service[16]; - snprintf(service, sizeof(service), "%d", port); - struct addrinfo hints; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_PASSIVE; - struct addrinfo *list = NULL; - if (getaddrinfo(host, service, &hints, &list) != 0) return -1; - - int listen_fd = -1; - for (struct addrinfo *ai = list; ai; ai = ai->ai_next) { - int fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) continue; - int yes = 1; - setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); - if (bind(fd, ai->ai_addr, ai->ai_addrlen) == 0 && - listen(fd, 16) == 0) { - listen_fd = fd; - break; - } - close(fd); - } - freeaddrinfo(list); - return listen_fd; -} - -static int connect_endpoint(const char *host, int port) -{ - char service[16]; - snprintf(service, sizeof(service), "%d", port); - struct addrinfo hints; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - struct addrinfo *list = NULL; - if (getaddrinfo(host, service, &hints, &list) != 0) return -1; - - int fd = -1; - for (struct addrinfo *ai = list; ai; ai = ai->ai_next) { - fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) continue; - set_socket_options(fd); - if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) break; - close(fd); - fd = -1; - } - freeaddrinfo(list); - return fd; -} - -static int stage_load(dist_stage *stage, const char *model_dir, - int layer_start, int layer_end, int use_mlx, - const pplx_dist_options_t *opts) -{ - memset(stage, 0, sizeof(*stage)); -#ifndef USE_MLX - (void)opts; -#endif -#ifdef USE_MLX - if (use_mlx) { - pplx_mlx_options_t mlx_opts = {0}; - if (opts) { - mlx_opts.quantize_bits = opts->mlx_quantize_bits; - mlx_opts.quantize_group_size = opts->mlx_quantize_group_size; - } - stage->mlx = pplx_mlx_load_slice_with_options( - model_dir, layer_start, layer_end, &mlx_opts); - if (!stage->mlx) return -1; - stage->cfg = pplx_mlx_config(stage->mlx); - return 0; - } -#else - if (use_mlx) return -1; -#endif - stage->model = pplx_model_load_slice(model_dir, layer_start, layer_end); - if (!stage->model) return -1; - stage->workspace = pplx_workspace_new(stage->model); - if (!stage->workspace) { - pplx_model_free(stage->model); - stage->model = NULL; - return -1; - } - stage->cfg = pplx_model_config(stage->model); - return 0; -} - -static void stage_free(dist_stage *stage) -{ - if (!stage) return; - if (stage->workspace) pplx_workspace_free(stage->workspace); - if (stage->model) pplx_model_free(stage->model); -#ifdef USE_MLX - if (stage->mlx) pplx_mlx_free(stage->mlx); -#endif - memset(stage, 0, sizeof(*stage)); -} - -static int stage_forward(dist_stage *stage, const pplx_input_t *inputs, - int batch, const float *input_states, - int layer_start, int layer_end, int apply_final_norm, - float *out_states) -{ -#ifdef USE_MLX - if (stage->mlx) - return pplx_mlx_forward_slice_batch(stage->mlx, inputs, batch, - input_states, layer_start, - layer_end, apply_final_norm, - out_states); -#endif - return pplx_model_forward_slice_batch(stage->model, stage->workspace, - inputs, batch, input_states, - layer_start, layer_end, - apply_final_norm, out_states); -} - -static int send_hello(int fd, const pplx_config_t *cfg, - int layer_start, int layer_end) -{ - uint64_t sig = config_signature(cfg); - dist_hello h; - u64_to_halves(sig, &h.signature_hi, &h.signature_lo); - h.hidden_size = (uint32_t)cfg->hidden_size; - h.n_layers = (uint32_t)cfg->n_layers; - h.vocab_size = (uint32_t)cfg->vocab_size; - h.n_heads = (uint32_t)cfg->n_heads; - h.n_kv_heads = (uint32_t)cfg->n_kv_heads; - h.head_dim = (uint32_t)cfg->head_dim; - h.intermediate_size = (uint32_t)cfg->intermediate_size; - h.layer_start = (uint32_t)layer_start; - h.layer_end = (uint32_t)layer_end; - hello_to_wire(&h); - if (write_frame_header(fd, PPLX_DIST_MSG_HELLO, sizeof(h)) != 0) - return -1; - return write_full(fd, &h, sizeof(h)); -} - -static int recv_hello(int fd, const pplx_config_t *cfg, int expected_start) -{ - uint32_t type = 0, bytes = 0; - dist_hello h; - if (read_frame_header(fd, &type, &bytes) != 0 || - type != PPLX_DIST_MSG_HELLO || bytes != sizeof(h) || - read_full(fd, &h, sizeof(h)) != 0) - return -1; - hello_from_wire(&h); - if (u64_from_halves(h.signature_hi, h.signature_lo) != - config_signature(cfg) || - h.hidden_size != (uint32_t)cfg->hidden_size || - h.n_layers != (uint32_t)cfg->n_layers || - h.layer_start != (uint32_t)expected_start || - h.layer_end != (uint32_t)cfg->n_layers) - return -1; - return 0; -} - -static int send_error_result(int fd, uint64_t request_id, const char *msg) -{ - size_t len = msg ? strlen(msg) : 0; - if (len > PPLX_DIST_MAX_ERROR_BYTES) len = PPLX_DIST_MAX_ERROR_BYTES; - dist_result result; - memset(&result, 0, sizeof(result)); - u64_to_halves(request_id, &result.request_hi, &result.request_lo); - result.status = 1; - result.payload_bytes = (uint32_t)len; - dist_result wire = result; - result_to_wire(&wire); - if (write_frame_header(fd, PPLX_DIST_MSG_RESULT, - (uint32_t)sizeof(wire) + (uint32_t)len) != 0 || - write_full(fd, &wire, sizeof(wire)) != 0) - return -1; - return len ? write_full(fd, msg, len) : 0; -} - -static int worker_process_work(int fd, dist_stage *stage, - int layer_start, int layer_end, - const void *body, uint32_t body_bytes) -{ - uint64_t request_id = 0; - if (body_bytes < sizeof(dist_work)) - return send_error_result(fd, request_id, "truncated WORK frame"); - - dist_work work; - memcpy(&work, body, sizeof(work)); - work_from_wire(&work); - request_id = u64_from_halves(work.request_hi, work.request_lo); - - uint32_t expected_lengths = 0, expected_state_values = 0; - uint32_t expected_state_f32_bytes = 0, expected_state_wire_bytes = 0; - uint32_t states_bits = activation_bits_or_default(work.states_bits); - if (work.batch == 0 || work.batch > PPLX_DIST_MAX_BATCH || - work.total_tokens == 0 || work.total_tokens > PPLX_DIST_MAX_TOKENS || - work.hidden_size != (uint32_t)stage->cfg->hidden_size || - !activation_bits_valid(states_bits) || - mul_u32(work.batch, sizeof(uint32_t), &expected_lengths) != 0 || - mul_u32(work.total_tokens, work.hidden_size, - &expected_state_values) != 0 || - mul_u32(expected_state_values, sizeof(float), - &expected_state_f32_bytes) != 0 || - activation_wire_bytes(states_bits, expected_state_values, - &expected_state_wire_bytes) != 0 || - work.lengths_bytes != expected_lengths || - work.states_bytes != expected_state_wire_bytes) - return send_error_result(fd, request_id, "invalid WORK frame"); - uint64_t expected_body = (uint64_t)sizeof(work) + expected_lengths + - expected_state_wire_bytes; - if (expected_body > UINT32_MAX || body_bytes != (uint32_t)expected_body) - return send_error_result(fd, request_id, "invalid WORK frame"); - - const unsigned char *p = (const unsigned char *)body + sizeof(work); - int *lengths = malloc((size_t)work.batch * sizeof(*lengths)); - pplx_input_t *inputs = calloc(work.batch, sizeof(*inputs)); - float *states = malloc(expected_state_f32_bytes); - float *out_states = malloc(expected_state_f32_bytes); - uint32_t emb_bytes = 0; - if (mul_u32(work.batch, work.hidden_size, &emb_bytes) != 0 || - mul_u32(emb_bytes, sizeof(float), &emb_bytes) != 0) { - free(lengths); free(inputs); free(states); free(out_states); - return send_error_result(fd, request_id, "embedding result is too large"); - } - float *embeddings = malloc(emb_bytes); - if (!lengths || !inputs || !states || !out_states || !embeddings) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "out of memory"); - } - - uint32_t total = 0; - for (uint32_t i = 0; i < work.batch; i++) { - uint32_t n = 0; - memcpy(&n, p + (size_t)i * sizeof(n), sizeof(n)); - n = ntohl(n); - if (n == 0 || n > PPLX_DIST_MAX_TOKENS - total) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "invalid sequence length"); - } - lengths[i] = (int)n; - inputs[i].n_tokens = (int)n; - total += n; - } - if (total != work.total_tokens) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "sequence lengths do not match token total"); - } - if (decode_activations(states, p + expected_lengths, - expected_state_values, states_bits) != 0) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "invalid activation payload"); - } - - int rc = stage_forward(stage, inputs, (int)work.batch, states, - layer_start, layer_end, 1, out_states); - if (rc == 0) - rc = pplx_pool_batch(stage->cfg, out_states, lengths, - (int)work.batch, embeddings); - free(lengths); free(inputs); free(states); free(out_states); - if (rc != 0) { - free(embeddings); - return send_error_result(fd, request_id, "worker inference failed"); - } - - dist_result result; - memset(&result, 0, sizeof(result)); - u64_to_halves(request_id, &result.request_hi, &result.request_lo); - result.batch = work.batch; - result.dimensions = work.hidden_size; - result.payload_bytes = emb_bytes; - dist_result wire = result; - result_to_wire(&wire); - rc = write_frame_header(fd, PPLX_DIST_MSG_RESULT, - (uint32_t)sizeof(wire) + emb_bytes); - if (rc == 0) rc = write_full(fd, &wire, sizeof(wire)); - if (rc == 0) rc = write_full(fd, embeddings, emb_bytes); - free(embeddings); - return rc; -} - -static int worker_serve_connection(int fd, dist_stage *stage, - int layer_start, int layer_end) -{ - if (send_hello(fd, stage->cfg, layer_start, layer_end) != 0) return -1; - for (;;) { - uint32_t type = 0, bytes = 0; - if (read_frame_header(fd, &type, &bytes) != 0) return -1; - if (type != PPLX_DIST_MSG_WORK || bytes == 0) return -1; - void *body = malloc(bytes); - if (!body) return -1; - int rc = read_full(fd, body, bytes); - if (rc == 0) - rc = worker_process_work(fd, stage, layer_start, layer_end, - body, bytes); - free(body); - if (rc != 0) return -1; - } -} - -int pplx_dist_run_worker(const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx) -{ - return pplx_dist_run_worker_with_options(model_dir, host, port, - layer_start, layer_end, use_mlx, - NULL); -} - -int pplx_dist_run_worker_with_options( - const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx, - const pplx_dist_options_t *opts) -{ - if (!model_dir || !host || port <= 0 || port > 65535 || - !host_is_little_endian()) { - fprintf(stderr, "pplx-dist: invalid worker configuration\n"); - return 1; - } - dist_stage stage; - if (stage_load(&stage, model_dir, layer_start, layer_end, use_mlx, - opts) != 0) { - fprintf(stderr, "pplx-dist: failed to load worker shard\n"); - return 1; - } - if (layer_end != stage.cfg->n_layers) { - fprintf(stderr, "pplx-dist: initial worker must own final layer %d\n", - stage.cfg->n_layers); - stage_free(&stage); - return 1; - } - int listen_fd = open_listener(host, port); - if (listen_fd < 0) { - fprintf(stderr, "pplx-dist: listen failed on %s:%d: %s\n", - host, port, strerror(errno)); - stage_free(&stage); - return 1; - } - fprintf(stderr, "pplx-dist: worker listening on %s:%d layers=[%d,%d)\n", - host, port, layer_start, layer_end); - for (;;) { - int fd = accept(listen_fd, NULL, NULL); - if (fd < 0) { - if (errno == EINTR) continue; - fprintf(stderr, "pplx-dist: accept failed: %s\n", strerror(errno)); - break; - } - set_socket_options(fd); - if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator connected\n"); - worker_serve_connection(fd, &stage, layer_start, layer_end); - close(fd); - if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator disconnected\n"); - } - close(listen_fd); - stage_free(&stage); - return 1; -} - -pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, - const char *host, int port, - int local_layer_end, int use_mlx) -{ - return pplx_dist_client_open_with_options(model_dir, host, port, - local_layer_end, use_mlx, NULL); -} - -pplx_dist_client_t *pplx_dist_client_open_with_options( - const char *model_dir, const char *host, int port, - int local_layer_end, int use_mlx, const pplx_dist_options_t *opts) -{ - if (!model_dir || !host || port <= 0 || port > 65535 || - local_layer_end <= 0 || !host_is_little_endian()) - return NULL; - - pplx_dist_client_t *client = calloc(1, sizeof(*client)); - if (!client) return NULL; - client->fd = -1; - if (stage_load(&client->local, model_dir, 0, local_layer_end, use_mlx, - opts) != 0) - goto fail; - client->activation_bits = activation_bits_or_default( - opts ? (uint32_t)opts->activation_bits : 0); - if (!activation_bits_valid(client->activation_bits)) - goto fail; - client->fd = connect_endpoint(host, port); - if (client->fd < 0 || - recv_hello(client->fd, client->local.cfg, local_layer_end) != 0) - goto fail; - client->local_layer_end = local_layer_end; - client->next_request_id = 1; - return client; - -fail: - pplx_dist_client_free(client); - return NULL; -} - -void pplx_dist_client_free(pplx_dist_client_t *client) -{ - if (!client) return; - if (client->fd >= 0) close(client->fd); - stage_free(&client->local); - free(client); -} - -const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client) -{ - return client ? client->local.cfg : NULL; -} - -static int client_transport_error(pplx_dist_client_t *client) -{ - if (client->fd >= 0) { - close(client->fd); - client->fd = -1; - } - return -1; -} - -int pplx_dist_client_embed_batch(pplx_dist_client_t *client, - const pplx_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!client || client->fd < 0 || !inputs || batch <= 0 || - batch > (int)PPLX_DIST_MAX_BATCH || !out_embeddings) - return -1; - - uint32_t total_tokens = 0; - for (int i = 0; i < batch; i++) { - if (!inputs[i].ids || inputs[i].n_tokens <= 0 || - (uint32_t)inputs[i].n_tokens > PPLX_DIST_MAX_TOKENS - total_tokens) - return -1; - total_tokens += (uint32_t)inputs[i].n_tokens; - } - - uint32_t state_values = 0, state_f32_bytes = 0, state_wire_bytes = 0; - uint32_t lengths_bytes = 0; - uint32_t hidden = (uint32_t)client->local.cfg->hidden_size; - uint32_t states_bits = activation_bits_or_default(client->activation_bits); - if (mul_u32(total_tokens, hidden, &state_values) != 0 || - mul_u32(state_values, sizeof(float), &state_f32_bytes) != 0 || - activation_wire_bytes(states_bits, state_values, &state_wire_bytes) != 0 || - mul_u32((uint32_t)batch, sizeof(uint32_t), &lengths_bytes) != 0) - return -1; - uint64_t body_bytes64 = (uint64_t)sizeof(dist_work) + lengths_bytes + - state_wire_bytes; - if (body_bytes64 > PPLX_DIST_MAX_FRAME_BYTES) - return -1; - uint32_t body_bytes = (uint32_t)body_bytes64; - float *states = malloc(state_f32_bytes); - void *wire_states = malloc(state_wire_bytes); - if (!states || !wire_states) { - free(states); free(wire_states); - return -1; - } - int rc = stage_forward(&client->local, inputs, batch, NULL, 0, - client->local_layer_end, 0, states); - if (rc != 0) { - free(states); free(wire_states); - return -1; - } - if (encode_activations(wire_states, states, state_values, states_bits) != 0) { - free(states); free(wire_states); - return -1; - } - - uint64_t request_id = client->next_request_id++; - dist_work work; - u64_to_halves(request_id, &work.request_hi, &work.request_lo); - work.batch = (uint32_t)batch; - work.total_tokens = total_tokens; - work.hidden_size = hidden; - work.lengths_bytes = lengths_bytes; - work.states_bytes = state_wire_bytes; - work.states_bits = states_bits; - dist_work wire = work; - work_to_wire(&wire); - rc = write_frame_header(client->fd, PPLX_DIST_MSG_WORK, body_bytes); - if (rc == 0) rc = write_full(client->fd, &wire, sizeof(wire)); - for (int i = 0; rc == 0 && i < batch; i++) { - uint32_t n = htonl((uint32_t)inputs[i].n_tokens); - rc = write_full(client->fd, &n, sizeof(n)); - } - if (rc == 0) rc = write_full(client->fd, wire_states, state_wire_bytes); - free(states); free(wire_states); - if (rc != 0) return client_transport_error(client); - - uint32_t type = 0, bytes = 0; - dist_result result; - if (read_frame_header(client->fd, &type, &bytes) != 0 || - type != PPLX_DIST_MSG_RESULT || bytes < sizeof(result) || - read_full(client->fd, &result, sizeof(result)) != 0) - return client_transport_error(client); - result_from_wire(&result); - uint32_t payload_bytes = bytes - (uint32_t)sizeof(result); - if (result.payload_bytes != payload_bytes || - u64_from_halves(result.request_hi, result.request_lo) != request_id) - return client_transport_error(client); - if (result.status != 0) { - char error[PPLX_DIST_MAX_ERROR_BYTES + 1]; - if (payload_bytes > PPLX_DIST_MAX_ERROR_BYTES || - read_full(client->fd, error, payload_bytes) != 0) - return client_transport_error(client); - error[payload_bytes] = '\0'; - fprintf(stderr, "pplx-dist: worker error: %s\n", error); - return client_transport_error(client); - } - uint32_t expected = 0; - if (result.batch != (uint32_t)batch || result.dimensions != hidden || - mul_u32((uint32_t)batch, hidden, &expected) != 0 || - mul_u32(expected, sizeof(float), &expected) != 0 || - payload_bytes != expected || - read_full(client->fd, out_embeddings, payload_bytes) != 0) - return client_transport_error(client); - return 0; -} diff --git a/pplx_distributed.h b/pplx_distributed.h deleted file mode 100644 index cd97147..0000000 --- a/pplx_distributed.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * pplx_distributed.h - Stateless two-node layer-sharded inference. - */ - -#ifndef PPLX_DISTRIBUTED_H -#define PPLX_DISTRIBUTED_H - -#include "pplx_embed.h" - -typedef struct pplx_dist_client pplx_dist_client_t; - -typedef struct { - int mlx_quantize_bits; - int mlx_quantize_group_size; - int activation_bits; /* 0 defaults to fp32; supported: 32, 16 */ -} pplx_dist_options_t; - -/* - * Open a coordinator-side connection. The coordinator owns layers - * [0, local_layer_end); the remote worker must own [local_layer_end, N). - */ -pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, - const char *host, int port, - int local_layer_end, int use_mlx); -pplx_dist_client_t *pplx_dist_client_open_with_options( - const char *model_dir, const char *host, int port, - int local_layer_end, int use_mlx, const pplx_dist_options_t *opts); -void pplx_dist_client_free(pplx_dist_client_t *client); - -const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client); - -int pplx_dist_client_embed_batch(pplx_dist_client_t *client, - const pplx_input_t *inputs, int batch, - float *out_embeddings); - -/* - * Run a final-stage worker until the process is terminated. The worker owns - * [layer_start, layer_end), and layer_end must be the model's final layer. - */ -int pplx_dist_run_worker(const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx); -int pplx_dist_run_worker_with_options( - const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx, - const pplx_dist_options_t *opts); - -#endif /* PPLX_DISTRIBUTED_H */ diff --git a/pplx_embed.c b/pplx_embed.c deleted file mode 100644 index 996be9c..0000000 --- a/pplx_embed.c +++ /dev/null @@ -1,1483 +0,0 @@ -#include "pplx_embed.h" -#include "pplx_embed_internal.h" -#include "qwen_kernels.h" -#include "qwen_safetensors.h" - -#include -#include -#include -#include -#include -#include - -struct pplx_late_model { - pplx_model_t *base; - safetensors_file_t *dense_sf; - pplx_weight_ref_t projection; /* [token_dim, hidden] */ - int token_dim; -}; - -struct pplx_late_workspace { - const pplx_late_model_t *model; - pplx_workspace_t *base_ws; - float *states; - int states_seq_cap; -}; - -/* ======================================================================== - * Globals - * ======================================================================== */ - -int pplx_verbose = 0; -int qwen_verbose = 0; - -#define PPLX_BF16_QKV_FUSE_MAX_SEQ 16 -#define PPLX_BF16_PAIR_FUSE_MAX_SEQ 4 -#define PPLX_MIN_WORKSPACE_SEQ_CAP 16 -#define PPLX_WORKSPACE_SEQ_GRANULARITY 16 - -/* ======================================================================== - * Size helpers - * ======================================================================== */ - -static int mul_size(size_t a, size_t b, size_t *out) -{ - if (a != 0 && b > SIZE_MAX / a) return -1; - *out = a * b; - return 0; -} - -static int add_size(size_t a, size_t b, size_t *out) -{ - if (b > SIZE_MAX - a) return -1; - *out = a + b; - return 0; -} - -static int grow_cap(int current, int needed, int *out) -{ - if (needed <= 0) return -1; - - int cap = needed; - if (cap < PPLX_MIN_WORKSPACE_SEQ_CAP) - cap = PPLX_MIN_WORKSPACE_SEQ_CAP; - - int rem = cap % PPLX_WORKSPACE_SEQ_GRANULARITY; - if (rem != 0) { - int add = PPLX_WORKSPACE_SEQ_GRANULARITY - rem; - if (cap > INT_MAX - add) return -1; - cap += add; - } - - if (cap < current) - cap = current; - - *out = cap; - return 0; -} - -static int realloc_floats(float **ptr, size_t count) -{ - size_t bytes; - if (mul_size(count, sizeof(float), &bytes) != 0) return -1; - - void *p = realloc(*ptr, bytes); - if (!p && bytes != 0) return -1; - *ptr = (float *)p; - return 0; -} - -static float *malloc_floats(size_t count) -{ - size_t bytes; - if (mul_size(count, sizeof(float), &bytes) != 0) return NULL; - return (float *)malloc(bytes); -} - -static int model_dir_has_file(const char *model_dir, const char *rel) -{ - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) return 0; - FILE *f = fopen(path, "rb"); - if (!f) return 0; - fclose(f); - return 1; -} - -static int model_dir_has_late_projection(const char *model_dir) -{ - return model_dir_has_file(model_dir, "1_Dense/config.json") && - model_dir_has_file(model_dir, "1_Dense/model.safetensors"); -} - -static int realloc_floats_2d(float **ptr, int rows, int cols) -{ - if (rows < 0 || cols < 0) return -1; - - size_t count; - if (mul_size((size_t)rows, (size_t)cols, &count) != 0) return -1; - return realloc_floats(ptr, count); -} - -static int ensure_late_state_buffer(pplx_late_workspace_t *ws, int n_tokens) -{ - if (!ws || !ws->model || n_tokens <= 0) return -1; - if (n_tokens <= ws->states_seq_cap) return 0; - int hidden = ws->model->base->config.hidden_size; - if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0) - return -1; - ws->states_seq_cap = n_tokens; - return 0; -} - -static int realloc_ints(int **ptr, size_t count) -{ - size_t bytes; - if (mul_size(count, sizeof(int), &bytes) != 0) return -1; - - void *p = realloc(*ptr, bytes); - if (!p && bytes != 0) return -1; - *ptr = (int *)p; - return 0; -} - -/* ======================================================================== - * Minimal JSON helpers for config.json parsing - * ======================================================================== */ - -static const char *skip_ws(const char *p) -{ - while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; - return p; -} - -/* Find "key": in a flat JSON object. Returns pointer to value start. */ -static const char *json_find_key(const char *json, const char *key) -{ - size_t klen = strlen(key); - const char *p = json; - while ((p = strstr(p, key)) != NULL) { - /* Check it's actually a quoted key: "key" */ - if (p > json && *(p - 1) == '"') { - const char *after = p + klen; - if (*after == '"') { - after = skip_ws(after + 1); - if (*after == ':') return skip_ws(after + 1); - } - } - p += klen; - } - return NULL; -} - -static int json_get_int(const char *json, const char *key, int fallback) -{ - const char *v = json_find_key(json, key); - if (!v) return fallback; - return atoi(v); -} - -static double json_get_double(const char *json, const char *key, double fallback) -{ - const char *v = json_find_key(json, key); - if (!v) return fallback; - return atof(v); -} - -static int parse_config(pplx_config_t *cfg, const char *model_dir) -{ - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); - if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "pplx_model_load: model path too long\n"); - return -1; - } - - FILE *f = fopen(path, "rb"); - if (!f) { - fprintf(stderr, "pplx_model_load: cannot open %s\n", path); - return -1; - } - - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return -1; - } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - char *buf = (char *)malloc((size_t)sz + 1); - if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); free(buf); return -1; - } - fclose(f); - buf[sz] = '\0'; - - cfg->hidden_size = json_get_int(buf, "hidden_size", 0); - cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0); - cfg->n_heads = json_get_int(buf, "num_attention_heads", 0); - cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0); - cfg->head_dim = json_get_int(buf, "head_dim", PPLX_HEAD_DIM); - cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0); - cfg->vocab_size = json_get_int(buf, "vocab_size", PPLX_VOCAB_SIZE); - cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); - cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); - - cfg->q_dim = 0; - cfg->kv_dim = 0; - if (cfg->n_heads > 0 && cfg->head_dim > 0 && - cfg->n_heads <= INT_MAX / cfg->head_dim) - cfg->q_dim = cfg->n_heads * cfg->head_dim; - if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && - cfg->n_kv_heads <= INT_MAX / cfg->head_dim) - cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; - - free(buf); - - /* Sanity checks */ - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || - cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || - cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || - cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || - (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || - !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { - fprintf(stderr, "pplx_model_load: invalid config in %s " - "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", - path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, - cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); - return -1; - } - if (cfg->n_layers > PPLX_MAX_LAYERS) { - fprintf(stderr, "pplx_model_load: too many layers (%d > %d)\n", - cfg->n_layers, PPLX_MAX_LAYERS); - return -1; - } - - if (pplx_verbose >= 1) - fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, " - "inter=%d, head_dim=%d\n", - cfg->hidden_size, cfg->n_layers, - cfg->n_heads, cfg->n_kv_heads, - cfg->intermediate_size, cfg->head_dim); - - return 0; -} - -/* ======================================================================== - * Weight loading (direct mmap pointers into safetensors) - * ======================================================================== */ - -static size_t dtype_size(safetensor_dtype_t dtype) -{ - switch (dtype) { - case DTYPE_F32: return sizeof(float); - case DTYPE_BF16: return sizeof(uint16_t); - default: return 0; - } -} - -static const char *dtype_name(safetensor_dtype_t dtype) -{ - switch (dtype) { - case DTYPE_F32: return "F32"; - case DTYPE_F16: return "F16"; - case DTYPE_BF16: return "BF16"; - case DTYPE_I32: return "I32"; - case DTYPE_I64: return "I64"; - case DTYPE_BOOL: return "BOOL"; - default: return "UNKNOWN"; - } -} - -static int tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, const char *name, - const int64_t *shape, int ndim, - int bf16_ok) -{ - if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) { - fprintf(stderr, "pplx: unsupported dtype for %s: got %s, expected %s\n", - name, dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "pplx: bad rank for %s: got %d, expected %d\n", - name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (t->shape[i] != shape[i]) { - fprintf(stderr, "pplx: bad shape for %s at dim %d: " - "got %lld, expected %lld\n", - name, i, (long long)t->shape[i], (long long)shape[i]); - return 0; - } - if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "pplx: shape too large for %s\n", name); - return 0; - } - numel *= (size_t)shape[i]; - } - - size_t elem_size = dtype_size(t->dtype); - if (elem_size == 0) { - fprintf(stderr, "pplx: unsupported dtype for %s: got %s\n", - name, dtype_name(t->dtype)); - return 0; - } - - size_t bytes; - if (mul_size(numel, elem_size, &bytes) != 0) { - fprintf(stderr, "pplx: tensor data too large for %s\n", name); - return 0; - } - if (t->data_size != bytes) { - fprintf(stderr, "pplx: bad data size for %s: got %zu, expected %zu\n", - name, t->data_size, bytes); - return 0; - } - - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || - t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "pplx: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static pplx_weight_ref_t load_weight_direct(multi_safetensors_t *ms, - const char *name, - const int64_t *shape, int ndim) -{ - pplx_weight_ref_t ref = { DTYPE_UNKNOWN, NULL }; - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "pplx: tensor not found: %s\n", name); - return ref; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) - return ref; - - ref.dtype = t->dtype; - ref.data = safetensors_data(sf, t); - return ref; -} - -static float *load_norm_f32(multi_safetensors_t *ms, const char *name, - const int64_t *shape, int ndim) -{ - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "pplx: tensor not found: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) - return NULL; - return safetensors_get_f32(sf, t); -} - -static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) -{ - for (int i = 0; i < n; i++) { - uint32_t u = ((uint32_t)src[i]) << 16; - memcpy(dst + i, &u, sizeof(float)); - } -} - -static void copy_weight_row(float *dst, const pplx_weight_ref_t *w, - size_t row, int row_width) -{ - if (w->dtype == DTYPE_F32) { - const float *src = (const float *)w->data + row * (size_t)row_width; - memcpy(dst, src, (size_t)row_width * sizeof(float)); - } else { - const uint16_t *src = (const uint16_t *)w->data + row * (size_t)row_width; - bf16_row_to_f32(dst, src, row_width); - } -} - -static void linear_nobias_weight(float *y, const float *x, - const pplx_weight_ref_t *w, - int seq_len, int in_dim, int out_dim) -{ - if (w->dtype == DTYPE_F32) { - qwen_linear_nobias(y, x, (const float *)w->data, - seq_len, in_dim, out_dim); - } else { - qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, - seq_len, in_dim, out_dim); - } -} - -static void linear_qkv_weight(float *q, float *k, float *v, const float *x, - const pplx_weight_ref_t *wq, - const pplx_weight_ref_t *wk, - const pplx_weight_ref_t *wv, - int seq_len, int in_dim, int q_dim, int kv_dim) -{ - if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && - wv->dtype == DTYPE_BF16 && - seq_len <= PPLX_BF16_QKV_FUSE_MAX_SEQ) { - qwen_linear_nobias_bf16_qkv(q, k, v, x, - (const uint16_t *)wq->data, - (const uint16_t *)wk->data, - (const uint16_t *)wv->data, - seq_len, in_dim, q_dim, kv_dim); - return; - } - - linear_nobias_weight(q, x, wq, seq_len, in_dim, q_dim); - linear_nobias_weight(k, x, wk, seq_len, in_dim, kv_dim); - linear_nobias_weight(v, x, wv, seq_len, in_dim, kv_dim); -} - -static void linear_pair_weight(float *a, float *b, const float *x, - const pplx_weight_ref_t *wa, - const pplx_weight_ref_t *wb, - int seq_len, int in_dim, int a_dim, int b_dim) -{ - if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && - seq_len <= PPLX_BF16_PAIR_FUSE_MAX_SEQ) { - qwen_linear_nobias_bf16_pair(a, b, x, - (const uint16_t *)wa->data, - (const uint16_t *)wb->data, - seq_len, in_dim, a_dim, b_dim); - return; - } - - linear_nobias_weight(a, x, wa, seq_len, in_dim, a_dim); - linear_nobias_weight(b, x, wb, seq_len, in_dim, b_dim); -} - -/* ======================================================================== - * Working buffer management - * ======================================================================== */ - -static int ensure_buffers(pplx_workspace_t *ws, const pplx_config_t *c, int seq) -{ - if (seq <= ws->buf_seq_cap) return 0; - - int cap; - if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) return -1; - -#define R(ptr, n) do { \ - if (realloc_floats_2d(&(ptr), cap, (n)) != 0) return -1; \ -} while (0) - - R(ws->x, c->hidden_size); - R(ws->x_norm, c->hidden_size); - R(ws->q, c->q_dim); - R(ws->k, c->kv_dim); - R(ws->v, c->kv_dim); - R(ws->attn_out, c->q_dim); - R(ws->proj_out, c->hidden_size); - R(ws->ffn_gate, c->intermediate_size); - R(ws->ffn_up, c->intermediate_size); - -#undef R - - ws->buf_seq_cap = cap; - return 0; -} - -static void ensure_attention_scores(pplx_workspace_t *ws, const int *offsets, - int batch) -{ - size_t bytes = - qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (bytes <= ws->attn_scores_bytes) return; - - void *p = realloc(ws->attn_scores, bytes); - if (!p) return; - ws->attn_scores = (float *)p; - ws->attn_scores_bytes = bytes; -} - -/* ======================================================================== - * RoPE cache - * ======================================================================== */ - -static int ensure_rope_cache(pplx_workspace_t *ws, const pplx_config_t *cfg, - int n_pos) -{ - if (n_pos <= ws->rope_cache_cap) return 0; - - int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; - while (cap < n_pos) { - if (cap > INT_MAX / 2) return -1; - cap *= 2; - } - - int head_dim = cfg->head_dim; - int half = head_dim / 2; - float theta = cfg->rope_theta; - - size_t count; - if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) - return -1; - if (realloc_floats(&ws->rope_cos, count) != 0) - return -1; - if (realloc_floats(&ws->rope_sin, count) != 0) - return -1; - - for (int pos = ws->rope_cache_cap; pos < cap; pos++) { - float fp = (float)pos; - float *c = ws->rope_cos + (size_t)pos * head_dim; - float *s = ws->rope_sin + (size_t)pos * head_dim; - for (int d = 0; d < half; d++) { - float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); - float angle = fp * inv; - float cv = cosf(angle), sv = sinf(angle); - c[d] = cv; c[half + d] = cv; - s[d] = sv; s[half + d] = sv; - } - } - - ws->rope_cache_cap = cap; - return 0; -} - -static int ensure_offsets(pplx_workspace_t *ws, int batch) -{ - if (!ws || batch <= 0 || batch == INT_MAX) return -1; - - int needed = batch + 1; - if (needed <= ws->offsets_cap) return 0; - - if (realloc_ints(&ws->offsets, (size_t)needed) != 0) - return -1; - ws->offsets_cap = needed; - return 0; -} - -/* ======================================================================== - * Model / workspace load and free - * ======================================================================== */ - -static pplx_model_t *model_load_range_ex(const char *model_dir, - int layer_start, int layer_end, - int allow_late) -{ - if (!allow_late && model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "pplx_model_load: late-interaction models require " - "token-level MaxSim support and are not valid pooled " - "embedding models yet\n"); - return NULL; - } - - pplx_model_t *model = (pplx_model_t *)calloc(1, sizeof(pplx_model_t)); - if (!model) return NULL; - - /* Parse config.json */ - if (parse_config(&model->config, model_dir) != 0) { - free(model); - return NULL; - } - - const pplx_config_t *cfg = &model->config; - if (layer_end == -1) - layer_end = cfg->n_layers; - if (layer_start < 0 || layer_start >= layer_end || - layer_end > cfg->n_layers) { - fprintf(stderr, "pplx_model_load: invalid layer range [%d, %d) " - "for %d-layer model\n", layer_start, layer_end, cfg->n_layers); - free(model); - return NULL; - } - model->layer_start = layer_start; - model->layer_end = layer_end; - - /* Open safetensors (handles single file or multi-shard) */ - multi_safetensors_t *ms = multi_safetensors_open(model_dir); - if (!ms) { - fprintf(stderr, "pplx_model_load: failed to open safetensors in %s\n", model_dir); - free(model); - return NULL; - } - model->safetensors = ms; - - if (pplx_verbose >= 1) - fprintf(stderr, "pplx_model_load: loading weights from %s\n", model_dir); - - /* Only the first stage needs token embeddings. */ - pplx_weights_t *w = &model->weights; - if (layer_start == 0) { - const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size }; - w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", - embed_shape, 2); - if (!w->embed_tokens.data) goto fail; - } - - /* Allocate layer array */ - w->layers = (pplx_layer_t *)calloc(cfg->n_layers, sizeof(pplx_layer_t)); - if (!w->layers) goto fail; - - /* Per-layer weights */ - char name[256]; - for (int i = layer_start; i < layer_end; i++) { - pplx_layer_t *l = &w->layers[i]; - -#define LOAD_NORM(field, fmt, d0) do { \ - const int64_t expect[1] = { (d0) }; \ - snprintf(name, sizeof(name), fmt, i); \ - l->field = load_norm_f32(ms, name, expect, 1); \ - if (!l->field) goto fail; \ -} while (0) - -#define LOAD2(field, fmt, d0, d1) do { \ - const int64_t expect[2] = { (d0), (d1) }; \ - snprintf(name, sizeof(name), fmt, i); \ - l->field = load_weight_direct(ms, name, expect, 2); \ - if (!l->field.data) goto fail; \ -} while (0) - - LOAD2(wq, "layers.%d.self_attn.q_proj.weight", - cfg->q_dim, cfg->hidden_size); - LOAD2(wk, "layers.%d.self_attn.k_proj.weight", - cfg->kv_dim, cfg->hidden_size); - LOAD2(wv, "layers.%d.self_attn.v_proj.weight", - cfg->kv_dim, cfg->hidden_size); - LOAD2(wo, "layers.%d.self_attn.o_proj.weight", - cfg->hidden_size, cfg->q_dim); - LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight", - cfg->head_dim); - LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight", - cfg->head_dim); - LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight", - cfg->hidden_size); - LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight", - cfg->hidden_size); - LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight", - cfg->intermediate_size, cfg->hidden_size); - LOAD2(up_proj, "layers.%d.mlp.up_proj.weight", - cfg->intermediate_size, cfg->hidden_size); - LOAD2(down_proj, "layers.%d.mlp.down_proj.weight", - cfg->hidden_size, cfg->intermediate_size); - -#undef LOAD2 -#undef LOAD_NORM - - if (pplx_verbose >= 2) - fprintf(stderr, " layer %d loaded\n", i); - } - - /* Only the final stage applies the output RMSNorm. */ - if (layer_end == cfg->n_layers) { - const int64_t norm_shape[1] = { cfg->hidden_size }; - w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1); - if (!w->norm) goto fail; - } - - if (pplx_verbose >= 1) - fprintf(stderr, "pplx_model_load: layers [%d, %d) loaded " - "(%d-dim embeddings)\n", - layer_start, layer_end, cfg->hidden_size); - - return model; - -fail: - pplx_model_free(model); - return NULL; -} - -pplx_model_t *pplx_model_load(const char *model_dir) -{ - return model_load_range_ex(model_dir, 0, -1, 0); -} - -pplx_model_t *pplx_model_load_slice(const char *model_dir, - int layer_start, int layer_end) -{ - return model_load_range_ex(model_dir, layer_start, layer_end, 0); -} - -void pplx_model_free(pplx_model_t *model) -{ - if (!model) return; - if (model->weights.layers) { - for (int i = 0; i < model->config.n_layers; i++) { - pplx_layer_t *l = &model->weights.layers[i]; - free((void *)l->q_norm); - free((void *)l->k_norm); - free((void *)l->input_norm); - free((void *)l->post_attn_norm); - } - } - free((void *)model->weights.norm); - if (model->safetensors) - multi_safetensors_close((multi_safetensors_t *)model->safetensors); - free(model->weights.layers); - free(model); -} - -pplx_workspace_t *pplx_workspace_new(const pplx_model_t *model) -{ - if (!model) return NULL; - pplx_workspace_t *ws = (pplx_workspace_t *)calloc(1, sizeof(pplx_workspace_t)); - if (!ws) return NULL; - ws->model = model; - return ws; -} - -void pplx_workspace_free(pplx_workspace_t *ws) -{ - if (!ws) return; - free(ws->x); free(ws->x_norm); - free(ws->q); free(ws->k); free(ws->v); - free(ws->attn_out); free(ws->proj_out); - free(ws->ffn_gate); free(ws->ffn_up); - free(ws->attn_scores); - free(ws->offsets); - free(ws->rope_cos); free(ws->rope_sin); - free(ws); -} - -const pplx_config_t *pplx_model_config(const pplx_model_t *model) -{ - return model ? &model->config : NULL; -} - -static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, - const char *name) -{ - if (!sf || !name) return NULL; - for (int i = 0; i < sf->num_tensors; i++) { - if (strcmp(sf->tensors[i].name, name) == 0) - return &sf->tensors[i]; - } - return NULL; -} - -pplx_late_model_t *pplx_late_model_load(const char *model_dir) -{ - if (!model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "pplx_late_model_load: missing 1_Dense projection\n"); - return NULL; - } - - pplx_late_model_t *late = (pplx_late_model_t *)calloc(1, sizeof(*late)); - if (!late) return NULL; - - late->base = model_load_range_ex(model_dir, 0, -1, 1); - if (!late->base) goto fail; - - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", - model_dir); - if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "pplx_late_model_load: model path too long\n"); - goto fail; - } - - late->dense_sf = safetensors_open(path); - if (!late->dense_sf) { - fprintf(stderr, "pplx_late_model_load: cannot open %s\n", path); - goto fail; - } - - const safetensor_t *t = find_single_safetensor(late->dense_sf, - "linear.weight"); - if (!t) { - fprintf(stderr, "pplx_late_model_load: missing linear.weight\n"); - goto fail; - } - if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) { - fprintf(stderr, "pplx_late_model_load: bad projection shape\n"); - goto fail; - } - - int64_t shape[2] = { t->shape[0], late->base->config.hidden_size }; - if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", - shape, 2, 1)) - goto fail; - - late->projection.dtype = t->dtype; - late->projection.data = safetensors_data(late->dense_sf, t); - late->token_dim = (int)t->shape[0]; - return late; - -fail: - pplx_late_model_free(late); - return NULL; -} - -void pplx_late_model_free(pplx_late_model_t *model) -{ - if (!model) return; - pplx_model_free(model->base); - safetensors_close(model->dense_sf); - free(model); -} - -pplx_late_workspace_t *pplx_late_workspace_new(const pplx_late_model_t *model) -{ - if (!model || !model->base) return NULL; - pplx_late_workspace_t *ws = (pplx_late_workspace_t *)calloc(1, sizeof(*ws)); - if (!ws) return NULL; - ws->model = model; - ws->base_ws = pplx_workspace_new(model->base); - if (!ws->base_ws) { - free(ws); - return NULL; - } - return ws; -} - -void pplx_late_workspace_free(pplx_late_workspace_t *ws) -{ - if (!ws) return; - pplx_workspace_free(ws->base_ws); - free(ws->states); - free(ws); -} - -const pplx_config_t *pplx_late_model_config(const pplx_late_model_t *model) -{ - return model && model->base ? &model->base->config : NULL; -} - -int pplx_late_model_token_dim(const pplx_late_model_t *model) -{ - return model ? model->token_dim : 0; -} - -int pplx_late_model_encode_tokens(const pplx_late_model_t *model, - pplx_late_workspace_t *ws, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors) -{ - if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || - !out_vectors || model->token_dim <= 0) - return -1; - - int hidden = model->base->config.hidden_size; - int dim = model->token_dim; - if (ensure_late_state_buffer(ws, n_tokens) != 0) - return -1; - if (pplx_model_forward_into(model->base, ws->base_ws, token_ids, - n_tokens, ws->states) != 0) - return -1; - - linear_nobias_weight(out_vectors, ws->states, &model->projection, - n_tokens, hidden, dim); - if (normalize) { - for (int i = 0; i < n_tokens; i++) { - if (pplx_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) - return -1; - } - } - return 0; -} - -float pplx_late_maxsim(const float *query_vectors, int query_tokens, - const float *doc_vectors, int doc_tokens, - int dim) -{ - if (!query_vectors || !doc_vectors || query_tokens <= 0 || - doc_tokens <= 0 || dim <= 0) - return 0.0f; - - float score = 0.0f; - for (int qi = 0; qi < query_tokens; qi++) { - const float *q = query_vectors + (size_t)qi * dim; - float best = -FLT_MAX; - for (int di = 0; di < doc_tokens; di++) { - const float *d = doc_vectors + (size_t)di * dim; - float dot = qwen_dot_f32(q, d, dim); - if (dot > best) best = dot; - } - score += best; - } - return score; -} - -int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens, - const float *doc_vectors, - const int *doc_offsets, int docs, - int dim, float *scores) -{ - if (!query_vectors || !doc_vectors || !doc_offsets || !scores || - query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0) - return -1; - - for (int i = 0; i < docs; i++) { - int start = doc_offsets[i]; - int end = doc_offsets[i + 1]; - if (start < 0 || end <= start) - return -1; - scores[i] = pplx_late_maxsim(query_vectors, query_tokens, - doc_vectors + (size_t)start * dim, - end - start, dim); - } - return 0; -} - -size_t pplx_workspace_nbytes(const pplx_workspace_t *ws) -{ - if (!ws) return 0; - - size_t total = sizeof(*ws); - const pplx_model_t *model = ws->model; - if (!model) return total; - - const pplx_config_t *c = &model->config; - size_t row_floats = 0; - const int dims[] = { - c->hidden_size, /* x */ - c->hidden_size, /* x_norm */ - c->q_dim, /* q */ - c->kv_dim, /* k */ - c->kv_dim, /* v */ - c->q_dim, /* attn_out */ - c->hidden_size, /* proj_out */ - c->intermediate_size, /* ffn_gate */ - c->intermediate_size, /* ffn_up */ - }; - - for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { - if (dims[i] < 0 || - add_size(row_floats, (size_t)dims[i], &row_floats) != 0) - return SIZE_MAX; - } - - size_t buf_floats = 0; - if (ws->buf_seq_cap > 0 && - mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) - return SIZE_MAX; - - size_t rope_floats = 0; - if (ws->rope_cache_cap > 0) { - size_t rope_rows = 0; - if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, - &rope_rows) != 0 || - mul_size(rope_rows, 2, &rope_floats) != 0) - return SIZE_MAX; - } - - size_t all_floats = 0, bytes = 0; - if (add_size(buf_floats, rope_floats, &all_floats) != 0 || - mul_size(all_floats, sizeof(float), &bytes) != 0 || - add_size(total, bytes, &total) != 0) - return SIZE_MAX; - - size_t offset_bytes = 0; - if (ws->offsets_cap > 0 && - mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) - return SIZE_MAX; - if (add_size(total, offset_bytes, &total) != 0) - return SIZE_MAX; - if (add_size(total, ws->attn_scores_bytes, &total) != 0) - return SIZE_MAX; - - return total; -} - -/* ======================================================================== - * Packed forward pass - * ======================================================================== */ - -static int build_offsets(const pplx_input_t *inputs, int batch, int require_ids, - int *offsets, int *total_out, int *max_out) -{ - if (!inputs || batch <= 0 || !offsets || !total_out || !max_out) - return -1; - - int total = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - if ((require_ids && !inputs[b].ids) || inputs[b].n_tokens <= 0) - return -1; - if (total > INT32_MAX - inputs[b].n_tokens) - return -1; - offsets[b] = total; - total += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) - max_seq = inputs[b].n_tokens; - } - offsets[batch] = total; - *total_out = total; - *max_out = max_seq; - return 0; -} - -static int forward_packed_slice_inplace(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_input_t *inputs, - int batch, const int *offsets, - int total_seq, int max_seq, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm) -{ - if (!model || !ws || ws->model != model || !inputs || - batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0) - return -1; - - const pplx_config_t *cfg = &model->config; - const pplx_weights_t *w = &model->weights; - if (layer_start < 0 || layer_start > layer_end || - layer_end > cfg->n_layers || - layer_start < model->layer_start || layer_end > model->layer_end || - (layer_start == 0 && input_states) || - (layer_start != 0 && !input_states) || - (layer_start == 0 && !w->embed_tokens.data) || - (apply_final_norm && (layer_end != cfg->n_layers || !w->norm))) - return -1; - - int hidden = cfg->hidden_size; - int n_heads = cfg->n_heads; - int n_kv_heads = cfg->n_kv_heads; - int head_dim = cfg->head_dim; - int q_dim = cfg->q_dim; - int kv_dim = cfg->kv_dim; - int inter = cfg->intermediate_size; - float eps = cfg->rms_norm_eps; - - if (ensure_buffers(ws, cfg, total_seq) != 0) return -1; - if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1; - ensure_attention_scores(ws, offsets, batch); - - const float *rope_cos = ws->rope_cos; - const float *rope_sin = ws->rope_sin; - - float *x = ws->x; - float *x_norm = ws->x_norm; - float *q_buf = ws->q; - float *k_buf = ws->k; - float *v_buf = ws->v; - float *attn_out = ws->attn_out; - float *proj_out = ws->proj_out; - float *ffn_gate = ws->ffn_gate; - float *ffn_up = ws->ffn_up; - - /* A first stage performs embedding lookup. Later stages resume from a - * packed hidden-state tensor received from the preceding stage. */ - if (layer_start == 0) { - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - for (int i = 0; i < inputs[b].n_tokens; i++) { - int id = inputs[b].ids[i]; - if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "pplx: invalid token id %d at input %d position %d\n", - id, b, i); - return -1; - } - copy_weight_row(x + (size_t)(start + i) * hidden, - &w->embed_tokens, (size_t)id, hidden); - } - } - } else { - memcpy(x, input_states, (size_t)total_seq * hidden * sizeof(float)); - } - - float scale = 1.0f / sqrtf((float)head_dim); - - /* 2. Transformer layers. Linear/MLP ops run over all packed tokens; - * attention is explicitly block-diagonal by sequence offsets. */ - for (int layer = layer_start; layer < layer_end; layer++) { - const pplx_layer_t *l = &w->layers[layer]; - - qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - - linear_qkv_weight(q_buf, k_buf, v_buf, x_norm, - &l->wq, &l->wk, &l->wv, - total_seq, hidden, q_dim, kv_dim); - - qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); - qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); - - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - int len = offsets[b + 1] - start; - qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, - rope_cos, rope_sin, len, n_heads, head_dim); - qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, - rope_cos, rope_sin, len, n_kv_heads, head_dim); - } - - qwen_bidirectional_gqa_attention_packed_with_scratch( - attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, - n_kv_heads, head_dim, scale, ws->attn_scores, - ws->attn_scores_bytes); - - linear_nobias_weight(proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); - qwen_add_inplace(x, proj_out, total_seq * hidden); - - qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); - - linear_pair_weight(ffn_gate, ffn_up, x_norm, - &l->gate_proj, &l->up_proj, - total_seq, hidden, inter, inter); - qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); - linear_nobias_weight(proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); - qwen_add_inplace(x, proj_out, total_seq * hidden); - } - - if (apply_final_norm) - qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps); - return 0; -} - -static int forward_packed_inplace(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_input_t *inputs, - int batch, const int *offsets, - int total_seq, int max_seq, - int apply_final_norm) -{ - return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, - total_seq, max_seq, NULL, 0, - model->config.n_layers, - apply_final_norm); -} - -static void pool_embeddings(const pplx_model_t *model, - const pplx_workspace_t *ws, - const int *offsets, int batch, - float *out_embeddings) -{ - const pplx_config_t *cfg = &model->config; - int hidden = cfg->hidden_size; - float eps = cfg->rms_norm_eps; - const float *norm_weight = model->weights.norm; - const float *x = ws->x; - - for (int b = 0; b < batch; b++) { - int start = offsets[b]; - int end = offsets[b + 1]; - int len = end - start; - float *emb = out_embeddings + (size_t)b * hidden; - - memset(emb, 0, (size_t)hidden * sizeof(float)); - for (int i = start; i < end; i++) { - const float *row = x + (size_t)i * hidden; - - float sum_sq = 0.0f; - for (int d = 0; d < hidden; d++) - sum_sq += row[d] * row[d]; - - float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps); - for (int d = 0; d < hidden; d++) - emb[d] += row[d] * rms_inv * norm_weight[d]; - } - - float inv_len = 1.0f / (float)len; - for (int d = 0; d < hidden; d++) - emb[d] *= inv_len; - - } -} - -/* ======================================================================== - * Public forward/embed APIs - * ======================================================================== */ - -int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_states) -{ - if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) return -1; - - size_t count, bytes; - if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, - &count) != 0 || - mul_size(count, sizeof(float), &bytes) != 0) - return -1; - - pplx_input_t input = { token_ids, n_tokens }; - int offsets[2] = { 0, n_tokens }; - if (forward_packed_inplace(model, ws, &input, 1, offsets, - n_tokens, n_tokens, 1) != 0) - return -1; - - memcpy(out_states, ws->x, bytes); - return 0; -} - -int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, - const pplx_span_t *spans, int n_spans, - float *out_embeddings) -{ - if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || - !out_embeddings) - return -1; - - int hidden = cfg->hidden_size; - if (hidden <= 0) return -1; - - for (int s = 0; s < n_spans; s++) { - int start = spans[s].start; - int len = spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) - return -1; - - float *emb = out_embeddings + (size_t)s * hidden; - memset(emb, 0, (size_t)hidden * sizeof(float)); - - for (int t = 0; t < len; t++) { - const float *row = states + (size_t)(start + t) * hidden; - for (int d = 0; d < hidden; d++) - emb[d] += row[d]; - } - - float inv_len = 1.0f / (float)len; - for (int d = 0; d < hidden; d++) { - emb[d] *= inv_len; - } - } - return 0; -} - -int pplx_model_embed_spans_batch(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_context_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) - return -1; - - if (ensure_offsets(ws, batch) != 0) return -1; - if ((size_t)batch > SIZE_MAX / sizeof(pplx_input_t)) return -1; - pplx_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); - if (!packed_inputs) return -1; - int *offsets = ws->offsets; - int total_seq = 0; - int max_seq = 0; - int total_spans = 0; - for (int b = 0; b < batch; b++) { - const pplx_context_input_t *input = &inputs[b]; - int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_seq > INT_MAX - n_tokens || - total_spans > INT_MAX - input->n_spans) { - free(packed_inputs); - return -1; - } - packed_inputs[b] = input->input; - offsets[b] = total_seq; - total_seq += n_tokens; - total_spans += input->n_spans; - if (n_tokens > max_seq) max_seq = n_tokens; - for (int s = 0; s < input->n_spans; s++) { - int start = input->spans[s].start; - int len = input->spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || - len > n_tokens - start) { - free(packed_inputs); - return -1; - } - } - } - offsets[batch] = total_seq; - - if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, - total_seq, max_seq, 1) != 0) { - free(packed_inputs); - return -1; - } - free(packed_inputs); - - int span_offset = 0; - int hidden = model->config.hidden_size; - for (int b = 0; b < batch; b++) { - const pplx_context_input_t *input = &inputs[b]; - if (pplx_pool_spans(&model->config, - ws->x + (size_t)offsets[b] * hidden, - input->input.n_tokens, input->spans, input->n_spans, - out_embeddings + (size_t)span_offset * hidden) != 0) - return -1; - span_offset += input->n_spans; - } - return 0; -} - -int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens, - const pplx_span_t *spans, int n_spans, - float *out_embeddings) -{ - pplx_context_input_t input = { - .input = { token_ids, n_tokens }, - .spans = spans, - .n_spans = n_spans, - }; - return pplx_model_embed_spans_batch(model, ws, &input, 1, out_embeddings); -} - -float *pplx_model_forward(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens) -{ - if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; - - size_t n; - if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0) - return NULL; - float *out = malloc_floats(n); - if (!out) return NULL; - - if (pplx_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) { - free(out); - return NULL; - } - return out; -} - -int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws, - const pplx_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; - - if (ensure_offsets(ws, batch) != 0) return -1; - int *offsets = ws->offsets; - - int total_seq = 0; - int max_seq = 0; - if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0) - return -1; - - int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, - total_seq, max_seq, 0); - if (rc == 0) - pool_embeddings(model, ws, offsets, batch, out_embeddings); - - return rc; -} - -int pplx_model_forward_slice_batch(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states) -{ - if (!model || !ws || !inputs || batch <= 0 || !out_states || - layer_start < 0 || layer_start > layer_end || - layer_end > model->config.n_layers) - return -1; - - if (ensure_offsets(ws, batch) != 0) return -1; - int *offsets = ws->offsets; - - int total_seq = 0; - int max_seq = 0; - if (build_offsets(inputs, batch, layer_start == 0, offsets, - &total_seq, &max_seq) != 0) - return -1; - - int rc = forward_packed_slice_inplace(model, ws, inputs, batch, offsets, - total_seq, max_seq, input_states, - layer_start, layer_end, - apply_final_norm); - if (rc == 0) { - size_t count; - if (mul_size((size_t)total_seq, - (size_t)model->config.hidden_size, &count) != 0) - return -1; - memcpy(out_states, ws->x, count * sizeof(float)); - } - return rc; -} - -int pplx_pool_batch(const pplx_config_t *cfg, const float *states, - const int *seq_lengths, int batch, - float *out_embeddings) -{ - if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || - cfg->hidden_size <= 0) - return -1; - - int offset = 0; - for (int b = 0; b < batch; b++) { - int len = seq_lengths[b]; - if (len <= 0 || offset > INT_MAX - len) - return -1; - - float *emb = out_embeddings + (size_t)b * cfg->hidden_size; - memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float)); - for (int t = 0; t < len; t++) { - const float *row = states + (size_t)(offset + t) * cfg->hidden_size; - for (int d = 0; d < cfg->hidden_size; d++) - emb[d] += row[d]; - } - - float inv_len = 1.0f / (float)len; - for (int d = 0; d < cfg->hidden_size; d++) - emb[d] *= inv_len; - offset += len; - } - return 0; -} - -int pplx_model_embed_into(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_embedding) -{ - pplx_input_t input = { token_ids, n_tokens }; - return pplx_model_embed_batch(model, ws, &input, 1, out_embedding); -} - -float *pplx_model_embed(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens) -{ - if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; - - int hidden = model->config.hidden_size; - float *emb = malloc_floats((size_t)hidden); - if (!emb) return NULL; - - if (pplx_model_embed_into(model, ws, token_ids, n_tokens, emb) != 0) { - free(emb); - return NULL; - } - return emb; -} - -/* ======================================================================== - * Vector helpers - * ======================================================================== */ - -int pplx_l2_normalize(float *vec, int dim) -{ - if (!vec || dim <= 0) return -1; - - float norm_sq = 0.0f; - for (int i = 0; i < dim; i++) - norm_sq += vec[i] * vec[i]; - - if (!(norm_sq > 0.0f) || !isfinite(norm_sq)) return -1; - - float inv_norm = 1.0f / sqrtf(norm_sq); - for (int i = 0; i < dim; i++) - vec[i] *= inv_norm; - return 0; -} - -float pplx_cosine_similarity(const float *a, const float *b, int dim) -{ - if (!a || !b || dim <= 0) return 0.0f; - - float dot = 0.0f; - float norm_a_sq = 0.0f; - float norm_b_sq = 0.0f; - for (int i = 0; i < dim; i++) { - dot += a[i] * b[i]; - norm_a_sq += a[i] * a[i]; - norm_b_sq += b[i] * b[i]; - } - - if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || - !isfinite(norm_a_sq) || !isfinite(norm_b_sq)) - return 0.0f; - return dot / sqrtf(norm_a_sq * norm_b_sq); -} diff --git a/pplx_embed.h b/pplx_embed.h deleted file mode 100644 index 5a2501b..0000000 --- a/pplx_embed.h +++ /dev/null @@ -1,293 +0,0 @@ -/* - * pplx_embed.h - Pure C inference for Perplexity AI's pplx-embed models. - * - * Supports all pplx-embed-v1 and pplx-embed-context-v1 variants (0.6B, 4B). - */ - -#ifndef PPLX_EMBED_H -#define PPLX_EMBED_H - -#include -#include - -/* ======================================================================== - * Shared constants (identical across all models) - * ======================================================================== */ - -#define PPLX_VOCAB_SIZE 151936 -#define PPLX_HEAD_DIM 128 -#define PPLX_MAX_LAYERS 64 /* upper bound for stack arrays */ -#define PPLX_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ - -/* ======================================================================== - * Model Configuration (populated from config.json at load time) - * ======================================================================== */ - -typedef struct { - int hidden_size; /* 0.6B: 1024 4B: 2560 */ - int n_layers; /* 0.6B: 28 4B: 36 */ - int n_heads; /* 0.6B: 16 4B: 32 */ - int n_kv_heads; /* 0.6B: 8 4B: 8 */ - int head_dim; /* always 128 */ - int q_dim; /* n_heads * head_dim */ - int kv_dim; /* n_kv_heads * head_dim */ - int intermediate_size; /* 0.6B: 3072 4B: 9728 */ - int vocab_size; /* 151936 */ - float rms_norm_eps; /* 1e-6 */ - float rope_theta; /* 1e6 */ -} pplx_config_t; - -/* ======================================================================== - * Model + workspace ownership - * ======================================================================== */ - -typedef struct pplx_model pplx_model_t; -typedef struct pplx_workspace pplx_workspace_t; -typedef struct pplx_late_model pplx_late_model_t; -typedef struct pplx_late_workspace pplx_late_workspace_t; - -typedef struct { - const int *ids; - int n_tokens; -} pplx_input_t; - -typedef struct { - int start; - int n_tokens; -} pplx_span_t; - -typedef struct { - pplx_input_t input; - const pplx_span_t *spans; - int n_spans; -} pplx_context_input_t; - -/* ======================================================================== - * API - * ======================================================================== */ - -/* - * Load immutable model data from a directory containing: - * config.json - model hyperparameters - * model.safetensors (or sharded) - F32 or BF16 weights - * vocab.json + merges.txt - BPE tokenizer - * - * Reads config.json to determine model size (0.6B vs 4B). - * Returns NULL on error. - */ -pplx_model_t *pplx_model_load(const char *model_dir); - -/* - * Load one contiguous transformer layer range for distributed execution. - * - * layer_start is inclusive and layer_end is exclusive. A range beginning at - * zero also loads the token embedding table. A range ending at n_layers also - * loads the final RMSNorm. Safetensors remain mmap-backed, so unselected - * weights are not touched. - */ -pplx_model_t *pplx_model_load_slice(const char *model_dir, - int layer_start, int layer_end); - -void pplx_model_free(pplx_model_t *model); - -/* - * Allocate/free mutable scratch buffers for a model. The model must outlive - * workspaces created from it. - */ -pplx_workspace_t *pplx_workspace_new(const pplx_model_t *model); -void pplx_workspace_free(pplx_workspace_t *ws); - -/* Config accessor for opaque model ownership. */ -const pplx_config_t *pplx_model_config(const pplx_model_t *model); - -/* - * Current mutable workspace memory owned by this workspace, in bytes. - * - * This includes the workspace struct, scratch buffers, and RoPE cache capacity. - * It does not include immutable mmap'd model weights. - */ -size_t pplx_workspace_nbytes(const pplx_workspace_t *ws); - -/* - * Compute embedding for a token sequence. - * - * 1. Token embedding lookup - * 2. N transformer layers (bidirectional GQA + RoPE + SwiGLU) - * 3. Final RMSNorm - * 4. Mean pooling over all positions - * - * Perplexity embeddings are intentionally not L2-normalized. Use cosine - * similarity for comparisons, or normalize explicitly before storing in vector - * databases that only support inner product. - * - * Returns malloc'd float[hidden_size] (caller frees). NULL on error. - */ -float *pplx_model_embed(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens); - -/* - * Compute one embedding into caller-provided out[hidden_size]. - * Returns 0 on success, -1 on error. - */ -int pplx_model_embed_into(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_embedding); - -/* - * Compute embeddings for a true packed/ragged batch. - * - * out_embeddings is caller-provided [batch, hidden_size]. - * Each sequence attends only to tokens from the same input, and RoPE - * positions restart at zero for every input. - * - * Returns 0 on success, -1 on error. - */ -int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws, - const pplx_input_t *inputs, int batch, - float *out_embeddings); - -/* - * Run a contiguous transformer layer range for a packed/ragged batch. - * - * layer_start is inclusive and layer_end is exclusive. For a range beginning - * at layer 0, input_states must be NULL and every input must provide token - * ids. For a later range, input_states must contain the preceding range's - * packed [sum(n_tokens), hidden_size] output; token ids are not required. - * - * apply_final_norm is valid only for a range ending at the final transformer - * layer. out_states receives packed [sum(n_tokens), hidden_size] states. - * - * This is the core execution boundary used by layer-sharded inference. - */ -int pplx_model_forward_slice_batch(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states); - -/* - * Mean-pool packed final hidden states for a ragged batch. - * - * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive - * token count per sequence. The states must already include final RMSNorm. - */ -int pplx_pool_batch(const pplx_config_t *cfg, const float *states, - const int *seq_lengths, int batch, - float *out_embeddings); - -/* - * Run the transformer forward pass WITHOUT pooling. - * Returns the full [n_tokens * hidden_size] output after final RMSNorm. - * Caller frees the returned buffer. NULL on error. - * - * Useful for contextual (late-chunking) models where you need - * per-token embeddings before splitting by separator positions. - */ -float *pplx_model_forward(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens); - -/* - * Run the transformer forward pass into caller-provided - * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error. - */ -int pplx_model_forward_into(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens, - float *out_states); - -/* - * Pool embeddings from final hidden states. - * - * states is [n_tokens, hidden_size], normally produced by - * pplx_model_forward_into(). Each span is mean pooled into - * out_embeddings[n_spans, hidden_size] without L2 normalization. - */ -int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, - const pplx_span_t *spans, int n_spans, - float *out_embeddings); - -/* - * Run one contextual sequence and pool selected token spans. - * Returns 0 on success, -1 on error. - */ -int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens, - const pplx_span_t *spans, int n_spans, - float *out_embeddings); - -/* - * Run a packed/ragged contextual document batch and pool every selected span. - * - * Documents attend independently, RoPE positions restart for each document, - * and out_embeddings contains spans in document order. - */ -int pplx_model_embed_spans_batch(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_context_input_t *inputs, int batch, - float *out_embeddings); - -/* - * L2-normalize vec[dim] in place. - * - * Returns 0 on success. Returns -1 for invalid arguments or a zero-length - * vector, which cannot be normalized. - */ -int pplx_l2_normalize(float *vec, int dim); - -/* - * Cosine similarity between two vectors. - * Returns 0 for invalid arguments or if either vector has zero length. - */ -float pplx_cosine_similarity(const float *a, const float *b, int dim); - -/* - * Late-interaction model API. - * - * A late model produces one 128D vector per retained token. The vectors are - * projected by the snapshot's 1_Dense head and L2-normalized per token by - * default, matching ColBERT/MaxSim scoring semantics. - */ -pplx_late_model_t *pplx_late_model_load(const char *model_dir); -void pplx_late_model_free(pplx_late_model_t *model); - -pplx_late_workspace_t *pplx_late_workspace_new(const pplx_late_model_t *model); -void pplx_late_workspace_free(pplx_late_workspace_t *ws); - -const pplx_config_t *pplx_late_model_config(const pplx_late_model_t *model); -int pplx_late_model_token_dim(const pplx_late_model_t *model); - -/* - * Encode token_ids into out_vectors[n_tokens, token_dim]. - * - * If normalize is non-zero, every non-zero token vector is L2-normalized. - */ -int pplx_late_model_encode_tokens(const pplx_late_model_t *model, - pplx_late_workspace_t *ws, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors); - -/* - * Sum over query tokens of max dot product against document tokens. - */ -float pplx_late_maxsim(const float *query_vectors, int query_tokens, - const float *doc_vectors, int doc_tokens, - int dim); - -/* - * Score one query against a packed/ragged batch of documents. - * - * doc_offsets has docs + 1 entries. Document i is stored in - * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim). - * scores is caller-provided [docs]. - */ -int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens, - const float *doc_vectors, - const int *doc_offsets, int docs, - int dim, float *scores); - -/* Verbose level: 0=quiet, 1=info, 2=debug */ -extern int pplx_verbose; -extern int qwen_verbose; - -#endif /* PPLX_EMBED_H */ diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu deleted file mode 100644 index a00fc84..0000000 --- a/pplx_embed_cuda.cu +++ /dev/null @@ -1,1204 +0,0 @@ -extern "C" { -#include "pplx_embed_cuda.h" -#include "pplx_embed_internal.h" -} - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -typedef struct { - float *d; // device buffer; holds __nv_bfloat16 elements when bf16 - int rows; - int cols; - int bf16; // 1 if d holds __nv_bfloat16, 0 if F32 -} cuda_matrix_t; - -typedef struct { - cuda_matrix_t qkv, wo; - float *q_norm; - float *k_norm; - float *input_norm; - float *post_attn_norm; - cuda_matrix_t gate_up_proj, down_proj; -} cuda_layer_t; - -struct pplx_cuda_ctx { - pplx_model_t *cpu; - pplx_config_t config; - cudaStream_t stream; - cublasHandle_t blas; - cuda_matrix_t embed_tokens; - cuda_layer_t *layers; - float *norm; - - int seq_cap; - int batch_cap; - int max_seq_cap; - float *x; - float *x_norm; - float *qkv; - float *attn_out; - float *ffn_gate_up; - void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights) - float *pooled_out; - float *rope_cos; - float *rope_sin; - int *token_ids; - int *offsets; - int *offsets_host; - int *positions; - float *kexp; - float *vexp; - float *attn_scores; - int *span_starts; - int *span_lens; - int pooled_rows_cap; - int span_cap; -}; - -#define CUDA_CHECK(expr) do { \ - cudaError_t _e = (expr); \ - if (_e != cudaSuccess) { \ - fprintf(stderr, "cuda: %s failed: %s\n", #expr, \ - cudaGetErrorString(_e)); \ - return -1; \ - } \ -} while (0) - -#define CUBLAS_CHECK(expr) do { \ - cublasStatus_t _s = (expr); \ - if (_s != CUBLAS_STATUS_SUCCESS) { \ - fprintf(stderr, "cuda: %s failed: cublas status %d\n", \ - #expr, (int)_s); \ - return -1; \ - } \ -} while (0) - -static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) -{ - if (!mode || !mode[0] || !strcmp(mode, "f32")) { - *out = CUBLAS_COMPUTE_32F; - return 0; - } - if (!strcmp(mode, "tf32")) { - *out = CUBLAS_COMPUTE_32F_FAST_TF32; - return 0; - } - if (!strcmp(mode, "bf16")) { - *out = CUBLAS_COMPUTE_32F_FAST_16BF; - return 0; - } - if (!strcmp(mode, "16f")) { - *out = CUBLAS_COMPUTE_32F_FAST_16F; - return 0; - } - return -1; -} - -static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F; -static int g_gemm_compute_set = 0; - -int pplx_cuda_set_fast_gemm(const char *mode) -{ - cublasComputeType_t ct; - if (parse_gemm_compute(mode, &ct) != 0) - return -1; - g_gemm_compute = ct; - g_gemm_compute_set = 1; - return 0; -} - -static void cuda_matrix_free(cuda_matrix_t *m) -{ - if (!m) return; - cudaFree(m->d); - memset(m, 0, sizeof(*m)); -} - -static float bf16_to_f32(uint16_t v) -{ - uint32_t u = ((uint32_t)v) << 16; - float f; - memcpy(&f, &u, sizeof(f)); - return f; -} - -// Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are -// preserved. BF16 shares F32's 8-bit exponent, so this never overflows. -static uint16_t f32_to_bf16(float f) -{ - uint32_t x; - memcpy(&x, &f, sizeof(x)); - if ((x & 0x7fffffffu) > 0x7f800000u) // NaN - return (uint16_t)((x >> 16) | 0x0040u); - x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even - return (uint16_t)(x >> 16); -} - -// Store weights as native BF16 on the device when set (halves weight memory and -// feeds BF16 tensor cores); default keeps exact F32. Read at load time. -static int g_weights_bf16 = 0; - -int pplx_cuda_set_weights_bf16(int on) -{ - g_weights_bf16 = on ? 1 : 0; - return 0; -} - -static int copy_weight_host_f32(float *dst, const pplx_weight_ref_t *w, - size_t count) -{ - if (!dst || !w || !w->data) return -1; - if (w->dtype == DTYPE_F32) { - memcpy(dst, w->data, count * sizeof(float)); - return 0; - } - if (w->dtype == DTYPE_BF16) { - const uint16_t *src = (const uint16_t *)w->data; - for (size_t i = 0; i < count; i++) - dst[i] = bf16_to_f32(src[i]); - return 0; - } - return -1; -} - -// Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is -// set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16. -static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, - int rows, int cols) -{ - cudaError_t e; - if (g_weights_bf16) { - uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t)); - if (!tmp16) return -1; - for (size_t i = 0; i < count; i++) - tmp16[i] = f32_to_bf16(src[i]); - e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16)); - if (e != cudaSuccess) { - fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e)); - free(tmp16); - return -1; - } - e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), - cudaMemcpyHostToDevice); - free(tmp16); - m->bf16 = 1; - } else { - e = cudaMalloc((void **)&m->d, count * sizeof(float)); - if (e != cudaSuccess) { - fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e)); - return -1; - } - e = cudaMemcpy(m->d, src, count * sizeof(float), cudaMemcpyHostToDevice); - m->bf16 = 0; - } - if (e != cudaSuccess) { - fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e)); - cuda_matrix_free(m); - return -1; - } - m->rows = rows; - m->cols = cols; - return 0; -} - -static int load_matrix(cuda_matrix_t *m, const pplx_weight_ref_t *w, - int rows, int cols) -{ - if (!m || !w || rows <= 0 || cols <= 0) return -1; - size_t count = (size_t)rows * (size_t)cols; - float *tmp = (float *)malloc(count * sizeof(float)); - if (!tmp) return -1; - if (copy_weight_host_f32(tmp, w, count) != 0) { - free(tmp); - return -1; - } - int r = upload_weight(m, tmp, count, rows, cols); - free(tmp); - return r; -} - -static int load_qkv_matrix(cuda_matrix_t *m, const pplx_layer_t *src, - const pplx_config_t *c) -{ - if (!m || !src || !c) return -1; - int rows = c->q_dim + 2 * c->kv_dim; - int cols = c->hidden_size; - size_t q_count = (size_t)c->q_dim * cols; - size_t kv_count = (size_t)c->kv_dim * cols; - size_t total = q_count + 2 * kv_count; - float *tmp = (float *)malloc(total * sizeof(float)); - if (!tmp) return -1; - if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 || - copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 || - copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) { - free(tmp); - return -1; - } - int r = upload_weight(m, tmp, total, rows, cols); - free(tmp); - return r; -} - -static int load_gate_up_matrix(cuda_matrix_t *m, const pplx_layer_t *src, - const pplx_config_t *c) -{ - if (!m || !src || !c) return -1; - int rows = 2 * c->intermediate_size; - int cols = c->hidden_size; - size_t proj_count = (size_t)c->intermediate_size * cols; - size_t total = 2 * proj_count; - float *tmp = (float *)malloc(total * sizeof(float)); - if (!tmp) return -1; - if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 || - copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) { - free(tmp); - return -1; - } - int r = upload_weight(m, tmp, total, rows, cols); - free(tmp); - return r; -} - -static int load_vector(float **out, const float *src, int n) -{ - if (!out || !src || n <= 0) return -1; - cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float)); - if (e != cudaSuccess) { - fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e)); - return -1; - } - e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice); - if (e != cudaSuccess) { - fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e)); - cudaFree(*out); - *out = NULL; - return -1; - } - return 0; -} - -__global__ static void embed_lookup_kernel(float *x, const int *ids, - const void *emb, int emb_bf16, - int total, int hidden, int vocab) -{ - int idx = blockIdx.x * blockDim.x + threadIdx.x; - int count = total * hidden; - if (idx >= count) return; - int tok = idx / hidden; - int d = idx - tok * hidden; - int id = ids[tok]; - if (id < 0 || id >= vocab) { - x[idx] = 0.0f; - return; - } - size_t off = (size_t)id * hidden + d; - x[idx] = emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) - : ((const float *)emb)[off]; -} - -// Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs). -__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, - const float *in, size_t n) -{ - size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i < n) out[i] = __float2bfloat16(in[i]); -} - -__device__ static float block_sum(float v) -{ - __shared__ float partial[8]; - __shared__ float total; - int lane = threadIdx.x & 31; - int warp = threadIdx.x >> 5; - int warps = blockDim.x >> 5; - - for (int offset = 16; offset > 0; offset >>= 1) - v += __shfl_down_sync(0xffffffff, v, offset); - if (lane == 0) partial[warp] = v; - __syncthreads(); - - if (warp == 0) { - v = lane < warps ? partial[lane] : 0.0f; - for (int offset = 16; offset > 0; offset >>= 1) - v += __shfl_down_sync(0xffffffff, v, offset); - if (lane == 0) total = v; - } - __syncthreads(); - return total; -} - -__device__ static inline void store_act(float *out, size_t i, float v) -{ - out[i] = v; -} - -__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) -{ - out[i] = __float2bfloat16(v); -} - -// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight -// projection GEMM directly (same rounding as a separate cast kernel). -template -__global__ static void rms_norm_kernel(OUT_T *out, const float *x, - const float *weight, int rows, - int dim, float eps) -{ - int row = blockIdx.x; - int tid = threadIdx.x; - float sum = 0.0f; - for (int d = tid; d < dim; d += blockDim.x) { - float v = x[(size_t)row * dim + d]; - sum += v * v; - } - float total = block_sum(sum); - float inv = rsqrtf(total / (float)dim + eps); - for (int d = tid; d < dim; d += blockDim.x) - store_act(out, (size_t)row * dim + d, - x[(size_t)row * dim + d] * inv * weight[d]); -} - -__global__ static void rms_norm_rope_head_kernel(float *x, const float *weight, - const int *positions, - const float *cosv, - const float *sinv, - int rows, int heads, - int head_dim, - int row_stride, - int base_offset, - float eps) -{ - __shared__ float partial[4]; - __shared__ float total; - int row = blockIdx.x; - int head = blockIdx.y; - int tid = threadIdx.x; - int lane = tid & 31; - int warp = tid >> 5; - int half = head_dim / 2; - int pair_d = tid < half ? tid + half : tid - half; - float sign = tid < half ? -1.0f : 1.0f; - size_t base = (size_t)row * row_stride + base_offset + head * head_dim; - float v = tid < head_dim ? x[base + tid] : 0.0f; - float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f; - float sum = v * v; - - for (int offset = 16; offset > 0; offset >>= 1) - sum += __shfl_down_sync(0xffffffff, sum, offset); - if (lane == 0) partial[warp] = sum; - __syncthreads(); - if (warp == 0) { - sum = lane < 4 ? partial[lane] : 0.0f; - for (int offset = 16; offset > 0; offset >>= 1) - sum += __shfl_down_sync(0xffffffff, sum, offset); - if (lane == 0) total = sum; - } - __syncthreads(); - - if (tid < head_dim) { - float inv = rsqrtf(total / (float)head_dim + eps); - int pos = positions[row]; - float c = cosv[(size_t)pos * head_dim + tid]; - float s = sinv[(size_t)pos * head_dim + tid]; - float a = v * inv * weight[tid]; - float b = pair_v * inv * weight[pair_d]; - x[base + tid] = a * c + sign * b * s; - } -} - -// Writes SiLU(gate)*up to out with out_stride. The F32 path writes in place -// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16 -// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM. -template -__global__ static void silu_mul_packed_kernel(OUT_T *out, int out_stride, - const float *gate_up, int rows, - int intermediate) -{ - int i = blockIdx.x * blockDim.x + threadIdx.x; - int n = rows * intermediate; - if (i < n) { - int row = i / intermediate; - int d = i - row * intermediate; - size_t base = (size_t)row * (2 * intermediate); - float g = gate_up[base + d]; - float u = gate_up[base + intermediate + d]; - store_act(out, (size_t)row * out_stride + d, - (g / (1.0f + expf(-g))) * u); - } -} - -// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim] -// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times) -// so the attention GEMMs can stride uniformly over query heads. -__global__ static void attn_expand_kv_kernel( - float *kexp, float *vexp, const float *qkv, int total, int n_heads, - int n_kv_heads, int head_dim, int qkv_dim, int k_offset, int v_offset) -{ - int q_dim = n_heads * head_dim; - size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - size_t count = (size_t)total * q_dim; - if (idx >= count) return; - int t = (int)(idx / q_dim); - int rem = (int)(idx - (size_t)t * q_dim); - int h = rem / head_dim; - int e = rem - h * head_dim; - int kv = h / (n_heads / n_kv_heads); - const float *base = qkv + (size_t)t * qkv_dim; - kexp[idx] = base[k_offset + kv * head_dim + e]; - vexp[idx] = base[v_offset + kv * head_dim + e]; -} - -// Row softmax over the key dimension for one sequence's score tensor laid out -// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i). -// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM. -__global__ static void attn_softmax_kernel(float *scores, int L) -{ - float *s = scores + (size_t)blockIdx.x * L; - int tid = threadIdx.x; - __shared__ float red[256]; - - float m = -3.402823466e+38F; - for (int j = tid; j < L; j += blockDim.x) m = fmaxf(m, s[j]); - red[tid] = m; - __syncthreads(); - for (int o = blockDim.x / 2; o > 0; o >>= 1) { - if (tid < o) red[tid] = fmaxf(red[tid], red[tid + o]); - __syncthreads(); - } - m = red[0]; - __syncthreads(); - - float sum = 0.0f; - for (int j = tid; j < L; j += blockDim.x) { - float e = __expf(s[j] - m); - s[j] = e; - sum += e; - } - red[tid] = sum; - __syncthreads(); - for (int o = blockDim.x / 2; o > 0; o >>= 1) { - if (tid < o) red[tid] += red[tid + o]; - __syncthreads(); - } - float inv = 1.0f / red[0]; - for (int j = tid; j < L; j += blockDim.x) s[j] *= inv; -} - -__global__ static void mean_pool_kernel(float *out, const float *x, - const int *offsets, int batch, - int hidden) -{ - int b = blockIdx.x; - int tid = threadIdx.x; - int start = offsets[b]; - int end = offsets[b + 1]; - int len = end - start; - for (int d = tid; d < hidden; d += blockDim.x) { - float sum = 0.0f; - for (int t = start; t < end; t++) - sum += x[(size_t)t * hidden + d]; - out[(size_t)b * hidden + d] = sum / (float)len; - } -} - -__global__ static void span_pool_kernel(float *out, const float *x, - const int *starts, const int *lens, - int n_spans, int hidden) -{ - int s = blockIdx.x; - int tid = threadIdx.x; - int start = starts[s]; - int len = lens[s]; - for (int d = tid; d < hidden; d += blockDim.x) { - float sum = 0.0f; - for (int t = 0; t < len; t++) - sum += x[(size_t)(start + t) * hidden + d]; - out[(size_t)s * hidden + d] = sum / (float)len; - } -} - -static int launch_check(void) -{ - cudaError_t e = cudaGetLastError(); - if (e != cudaSuccess) { - fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e)); - return -1; - } - return 0; -} - -// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm). -// PPLX_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core -// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands -// internally. -static cublasComputeType_t gemm_compute(void) -{ - static int env_init = 0; - if (!g_gemm_compute_set && !env_init) { - const char *e = getenv("PPLX_CUDA_GEMM_MODE"); - if (e && parse_gemm_compute(e, &g_gemm_compute) == 0) - g_gemm_compute_set = 1; - env_init = 1; - } - return g_gemm_compute; -} - -// y = w @ x (+ beta*y). With BF16 weights both GEMM inputs are BF16 and -// accumulation/output stay F32: when x_is_bf16 the activation operand was -// already produced in BF16 by the upstream kernel, otherwise the F32 operand -// is cast into ctx->act_bf16 first. With F32 weights this is the exact path, -// honoring gemm_compute(). -static int linear_ex(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const void *x, int x_is_bf16, float *y, int rows, - int in_dim, int out_dim, int x_stride, float beta) -{ - const float alpha = 1.0f; - if (w->bf16) { - const void *xb = x; - if (!x_is_bf16) { - size_t n = (size_t)rows * (size_t)x_stride; - int threads = 256; - cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, - ctx->stream>>>( - (__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n); - if (launch_check() != 0) return -1; - xb = ctx->act_bf16; - } - CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, - out_dim, rows, in_dim, - &alpha, w->d, CUDA_R_16BF, in_dim, - xb, CUDA_R_16BF, x_stride, - &beta, y, CUDA_R_32F, out_dim, - CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); - return 0; - } - CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, - out_dim, rows, in_dim, - &alpha, w->d, CUDA_R_32F, in_dim, - x, CUDA_R_32F, x_stride, - &beta, y, CUDA_R_32F, out_dim, - gemm_compute(), CUBLAS_GEMM_DEFAULT)); - return 0; -} - -static int linear(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const float *x, float *y, int rows, int in_dim, int out_dim) -{ - return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f); -} - -// x points at activations already stored as BF16 (e.g. ctx->act_bf16). -static int linear_bf16x(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const void *x, float *y, int rows, int in_dim, - int out_dim, int x_stride, float beta) -{ - return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta); -} - -static int linear_accum(pplx_cuda_ctx_t *ctx, const cuda_matrix_t *w, - const float *x, float *y, - int rows, int in_dim, int out_dim) -{ - return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f); -} - -static int gemm_strided_batched(cublasHandle_t blas, cublasOperation_t transa, - cublasOperation_t transb, int m, int n, int k, - const float *alpha, const float *A, int lda, - long long strideA, const float *B, int ldb, - long long strideB, const float *beta, - float *C, int ldc, long long strideC, - int batch_count) -{ - cublasComputeType_t ct = gemm_compute(); - if (ct == CUBLAS_COMPUTE_32F) { - CUBLAS_CHECK(cublasSgemmStridedBatched( - blas, transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb, - strideB, beta, C, ldc, strideC, batch_count)); - } else { - CUBLAS_CHECK(cublasGemmStridedBatchedEx( - blas, transa, transb, m, n, k, - alpha, A, CUDA_R_32F, lda, strideA, - B, CUDA_R_32F, ldb, strideB, - beta, C, CUDA_R_32F, ldc, strideC, - batch_count, ct, CUBLAS_GEMM_DEFAULT)); - } - return 0; -} - -// Attention for one micro-batch. Expands K/V to the per-query-head layout, -// then for each sequence runs batched-over-heads Q@K^T (scaled) -> row softmax -// -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and -// ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets -// is the host copy of the packed sequence boundaries. -static int cuda_attention_gemm(pplx_cuda_ctx_t *ctx, const int *h_offsets, - int batch, int q_offset, int k_offset, - int v_offset, int qkv_dim, float scale) -{ - const pplx_config_t *c = &ctx->config; - int q_dim = c->q_dim; - int hd = c->head_dim; - int H = c->n_heads; - int total = h_offsets[batch]; - const float alpha = scale, beta = 0.0f, one = 1.0f; - - int threads = 256; - size_t exp_count = (size_t)total * q_dim; - attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, - 0, ctx->stream>>>( - ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, - k_offset, v_offset); - if (launch_check() != 0) return -1; - - for (int b = 0; b < batch; b++) { - int start = h_offsets[b]; - int L = h_offsets[b + 1] - start; - const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset; - const float *K = ctx->kexp + (size_t)start * q_dim; - const float *V = ctx->vexp + (size_t)start * q_dim; - float *O = ctx->attn_out + (size_t)start * q_dim; - /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */ - if (gemm_strided_batched( - ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, - &alpha, K, q_dim, hd, Q, qkv_dim, hd, - &beta, ctx->attn_scores, L, (long long)L * L, H) != 0) - return -1; - attn_softmax_kernel<<stream>>>( - ctx->attn_scores, L); - if (launch_check() != 0) return -1; - /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ - if (gemm_strided_batched( - ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, - &one, V, q_dim, hd, ctx->attn_scores, L, (long long)L * L, - &beta, O, q_dim, hd, H) != 0) - return -1; - } - return 0; -} - -static int ensure_buffers(pplx_cuda_ctx_t *ctx, int total, int batch, int max_seq) -{ - const pplx_config_t *c = &ctx->config; - if (total > ctx->seq_cap) { - cudaFree(ctx->x); - cudaFree(ctx->x_norm); - cudaFree(ctx->qkv); - cudaFree(ctx->attn_out); - cudaFree(ctx->ffn_gate_up); - cudaFree(ctx->token_ids); - cudaFree(ctx->positions); - cudaFree(ctx->kexp); - cudaFree(ctx->vexp); - cudaFree(ctx->act_bf16); - ctx->x = ctx->x_norm = ctx->qkv = NULL; - ctx->attn_out = ctx->ffn_gate_up = NULL; - ctx->token_ids = ctx->positions = NULL; - ctx->kexp = ctx->vexp = NULL; - ctx->act_bf16 = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up, - (size_t)total * 2 * c->intermediate_size * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int))); - CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int))); - CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float))); - if (g_weights_bf16) - CUDA_CHECK(cudaMalloc(&ctx->act_bf16, - (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16))); - ctx->seq_cap = total; - } - if (batch + 1 > ctx->batch_cap) { - cudaFree(ctx->offsets); - free(ctx->offsets_host); - CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int))); - ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int)); - if (!ctx->offsets_host) return -1; - ctx->batch_cap = batch + 1; - } - if (max_seq > ctx->max_seq_cap) { - cudaFree(ctx->rope_cos); - cudaFree(ctx->rope_sin); - cudaFree(ctx->attn_scores); - ctx->attn_scores = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, - (size_t)c->n_heads * max_seq * max_seq * sizeof(float))); - float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); - float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float)); - if (!hcos || !hsin) { - free(hcos); - free(hsin); - return -1; - } - int half = c->head_dim / 2; - for (int pos = 0; pos < max_seq; pos++) { - for (int d = 0; d < half; d++) { - float inv = 1.0f / powf(c->rope_theta, - (float)(2 * d) / (float)c->head_dim); - float angle = (float)pos * inv; - float cv = cosf(angle); - float sv = sinf(angle); - hcos[(size_t)pos * c->head_dim + d] = cv; - hcos[(size_t)pos * c->head_dim + half + d] = cv; - hsin[(size_t)pos * c->head_dim + d] = sv; - hsin[(size_t)pos * c->head_dim + half + d] = sv; - } - } - CUDA_CHECK(cudaMalloc((void **)&ctx->rope_cos, - (size_t)max_seq * c->head_dim * sizeof(float))); - CUDA_CHECK(cudaMalloc((void **)&ctx->rope_sin, - (size_t)max_seq * c->head_dim * sizeof(float))); - CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, - (size_t)max_seq * c->head_dim * sizeof(float), - cudaMemcpyHostToDevice, ctx->stream)); - CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin, - (size_t)max_seq * c->head_dim * sizeof(float), - cudaMemcpyHostToDevice, ctx->stream)); - free(hcos); - free(hsin); - ctx->max_seq_cap = max_seq; - } - return 0; -} - -static int ensure_pooled_rows(pplx_cuda_ctx_t *ctx, int rows) -{ - if (rows <= ctx->pooled_rows_cap) return 0; - const pplx_config_t *c = &ctx->config; - cudaFree(ctx->pooled_out); - ctx->pooled_out = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out, - (size_t)rows * c->hidden_size * sizeof(float))); - ctx->pooled_rows_cap = rows; - return 0; -} - -static int ensure_span_buffers(pplx_cuda_ctx_t *ctx, int n_spans) -{ - if (n_spans <= ctx->span_cap) return 0; - cudaFree(ctx->span_starts); - cudaFree(ctx->span_lens); - ctx->span_starts = NULL; - ctx->span_lens = NULL; - CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, - (size_t)n_spans * sizeof(int))); - CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, - (size_t)n_spans * sizeof(int))); - ctx->span_cap = n_spans; - return 0; -} - -static int load_layer(cuda_layer_t *dst, const pplx_layer_t *src, - const pplx_config_t *c) -{ - return - load_qkv_matrix(&dst->qkv, src, c) || - load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || - load_vector(&dst->q_norm, src->q_norm, c->head_dim) || - load_vector(&dst->k_norm, src->k_norm, c->head_dim) || - load_vector(&dst->input_norm, src->input_norm, c->hidden_size) || - load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) || - load_gate_up_matrix(&dst->gate_up_proj, src, c) || - load_matrix(&dst->down_proj, &src->down_proj, - c->hidden_size, c->intermediate_size); -} - -static void free_layer(cuda_layer_t *l) -{ - cuda_matrix_free(&l->qkv); - cuda_matrix_free(&l->wo); - cudaFree(l->q_norm); - cudaFree(l->k_norm); - cudaFree(l->input_norm); - cudaFree(l->post_attn_norm); - cuda_matrix_free(&l->gate_up_proj); - cuda_matrix_free(&l->down_proj); -} - -pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir) -{ - pplx_model_t *cpu = pplx_model_load(model_dir); - if (!cpu) return NULL; - - pplx_cuda_ctx_t *ctx = (pplx_cuda_ctx_t *)calloc(1, sizeof(*ctx)); - if (!ctx) { - pplx_model_free(cpu); - return NULL; - } - ctx->cpu = cpu; - ctx->config = cpu->config; - - if (cudaStreamCreate(&ctx->stream) != cudaSuccess || - cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS || - cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) { - pplx_cuda_free(ctx); - return NULL; - } - - const pplx_config_t *c = &ctx->config; - ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers)); - if (!ctx->layers) { - pplx_cuda_free(ctx); - return NULL; - } - if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, - c->vocab_size, c->hidden_size) != 0 || - load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) { - pplx_cuda_free(ctx); - return NULL; - } - for (int i = 0; i < c->n_layers; i++) { - if (load_layer(&ctx->layers[i], &cpu->weights.layers[i], c) != 0) { - fprintf(stderr, "cuda: failed to load layer %d\n", i); - pplx_cuda_free(ctx); - return NULL; - } - } - cudaStreamSynchronize(ctx->stream); - return ctx; -} - -void pplx_cuda_free(pplx_cuda_ctx_t *ctx) -{ - if (!ctx) return; - cuda_matrix_free(&ctx->embed_tokens); - if (ctx->layers) { - for (int i = 0; i < ctx->config.n_layers; i++) - free_layer(&ctx->layers[i]); - free(ctx->layers); - } - cudaFree(ctx->norm); - cudaFree(ctx->x); - cudaFree(ctx->x_norm); - cudaFree(ctx->qkv); - cudaFree(ctx->attn_out); - cudaFree(ctx->ffn_gate_up); - cudaFree(ctx->pooled_out); - cudaFree(ctx->rope_cos); - cudaFree(ctx->rope_sin); - cudaFree(ctx->token_ids); - cudaFree(ctx->offsets); - free(ctx->offsets_host); - cudaFree(ctx->positions); - cudaFree(ctx->kexp); - cudaFree(ctx->vexp); - cudaFree(ctx->act_bf16); - cudaFree(ctx->attn_scores); - cudaFree(ctx->span_starts); - cudaFree(ctx->span_lens); - if (ctx->blas) cublasDestroy(ctx->blas); - if (ctx->stream) cudaStreamDestroy(ctx->stream); - pplx_model_free(ctx->cpu); - free(ctx); -} - -const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx) -{ - return ctx ? &ctx->config : NULL; -} - -static int cuda_forward_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, - int batch) -{ - if (!ctx || !inputs || batch <= 0) return -1; - const pplx_config_t *c = &ctx->config; - int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); - if (!h_offsets) return -1; - int total = 0, max_seq = 0; - for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) { - free(h_offsets); - return -1; - } - h_offsets[b] = total; - total += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - } - h_offsets[batch] = total; - int *h_ids = (int *)malloc((size_t)total * sizeof(int)); - int *h_pos = (int *)malloc((size_t)total * sizeof(int)); - if (!h_offsets || !h_ids || !h_pos) { - free(h_offsets); free(h_ids); free(h_pos); - return -1; - } - - for (int b = 0; b < batch; b++) { - int off = h_offsets[b]; - memcpy(h_ids + off, inputs[b].ids, - (size_t)inputs[b].n_tokens * sizeof(int)); - for (int i = 0; i < inputs[b].n_tokens; i++) - h_pos[off + i] = i; - } - if (ensure_buffers(ctx, total, batch, max_seq) != 0) { - free(h_offsets); free(h_ids); free(h_pos); - return -1; - } - memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int)); - cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, - (size_t)total * sizeof(int), - cudaMemcpyHostToDevice, ctx->stream); - if (ce == cudaSuccess) - ce = cudaMemcpyAsync(ctx->positions, h_pos, - (size_t)total * sizeof(int), - cudaMemcpyHostToDevice, ctx->stream); - if (ce == cudaSuccess) - ce = cudaMemcpyAsync(ctx->offsets, h_offsets, - (size_t)(batch + 1) * sizeof(int), - cudaMemcpyHostToDevice, ctx->stream); - free(h_offsets); free(h_ids); free(h_pos); - if (ce != cudaSuccess) { - fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce)); - return -1; - } - - int threads = 256; - int blocks_hidden = (total * c->hidden_size + threads - 1) / threads; - embed_lookup_kernel<<stream>>>( - ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, - total, c->hidden_size, c->vocab_size); - if (launch_check() != 0) return -1; - - float scale = 1.0f / sqrtf((float)c->head_dim); - int q_offset = 0; - int k_offset = c->q_dim; - int v_offset = c->q_dim + c->kv_dim; - int qkv_dim = c->q_dim + 2 * c->kv_dim; - // With BF16 weights the norm and SiLU kernels store their results as BF16 - // directly into ctx->act_bf16, which the next projection GEMM consumes - // before any later producer reuses the buffer (single stream). This - // removes three cast launches per layer; only the attention output still - // needs a cast because cuBLAS cannot emit a BF16 C from F32 A/B operands. - __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16; - for (int layer = 0; layer < c->n_layers; layer++) { - cuda_layer_t *l = &ctx->layers[layer]; - int wbf16 = l->qkv.bf16; - if (wbf16) { - rms_norm_kernel<<stream>>>( - act, ctx->x, l->input_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, - total, c->hidden_size, qkv_dim, - c->hidden_size, 0.0f) != 0) - return -1; - } else { - rms_norm_kernel<<stream>>>( - ctx->x_norm, ctx->x, l->input_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, - total, c->hidden_size, qkv_dim) != 0) - return -1; - } - - rms_norm_rope_head_kernel<<n_heads), 128, 0, ctx->stream>>>( - ctx->qkv, l->q_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, - total, c->n_heads, c->head_dim, qkv_dim, q_offset, - c->rms_norm_eps); - rms_norm_rope_head_kernel<<n_kv_heads), 128, 0, ctx->stream>>>( - ctx->qkv, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, - total, c->n_kv_heads, c->head_dim, qkv_dim, k_offset, - c->rms_norm_eps); - if (launch_check() != 0) return -1; - - if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, - k_offset, v_offset, qkv_dim, scale) != 0) - return -1; - - if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, - total, c->q_dim, c->hidden_size) != 0) - return -1; - - int inter_count = total * c->intermediate_size; - if (wbf16) { - rms_norm_kernel<<stream>>>( - act, ctx->x, l->post_attn_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, - total, c->hidden_size, 2 * c->intermediate_size, - c->hidden_size, 0.0f) != 0) - return -1; - silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, - threads, 0, ctx->stream>>>( - act, c->intermediate_size, ctx->ffn_gate_up, total, - c->intermediate_size); - if (launch_check() != 0) return -1; - if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, - total, c->intermediate_size, c->hidden_size, - c->intermediate_size, 1.0f) != 0) - return -1; - } else { - rms_norm_kernel<<stream>>>( - ctx->x_norm, ctx->x, l->post_attn_norm, total, - c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - if (linear(ctx, &l->gate_up_proj, ctx->x_norm, - ctx->ffn_gate_up, total, c->hidden_size, - 2 * c->intermediate_size) != 0) - return -1; - silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, - threads, 0, ctx->stream>>>( - ctx->ffn_gate_up, 2 * c->intermediate_size, ctx->ffn_gate_up, - total, c->intermediate_size); - if (launch_check() != 0) return -1; - if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, - total, c->intermediate_size, c->hidden_size, - 2 * c->intermediate_size, 1.0f) != 0) - return -1; - } - } - - rms_norm_kernel<<stream>>>( - ctx->x, ctx->x, ctx->norm, total, c->hidden_size, c->rms_norm_eps); - if (launch_check() != 0) return -1; - - return 0; -} - -int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; - const pplx_config_t *c = &ctx->config; - if (cuda_forward_batch(ctx, inputs, batch) != 0) - return -1; - if (ensure_pooled_rows(ctx, batch) != 0) - return -1; - mean_pool_kernel<<stream>>>( - ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); - if (launch_check() != 0) return -1; - cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, - (size_t)batch * c->hidden_size * sizeof(float), - cudaMemcpyDeviceToHost, ctx->stream); - if (ce == cudaSuccess) - ce = cudaStreamSynchronize(ctx->stream); - if (ce != cudaSuccess) { - fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce)); - return -1; - } - return 0; -} - -int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx, - const pplx_context_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; - const pplx_config_t *c = &ctx->config; - int total_spans = 0; - int total_tokens = 0; - for (int b = 0; b < batch; b++) { - const pplx_context_input_t *input = &inputs[b]; - int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans || - total_tokens > INT_MAX - n_tokens) - return -1; - for (int s = 0; s < input->n_spans; s++) { - int start = input->spans[s].start; - int len = input->spans[s].n_tokens; - if (start < 0 || len <= 0 || start > n_tokens || - len > n_tokens - start) - return -1; - } - total_spans += input->n_spans; - total_tokens += n_tokens; - } - - pplx_input_t *packed = (pplx_input_t *)malloc((size_t)batch * sizeof(*packed)); - int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int)); - int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int)); - if (!packed || !h_starts || !h_lens) { - free(packed); - free(h_starts); - free(h_lens); - return -1; - } - - int token_offset = 0; - int span_offset = 0; - for (int b = 0; b < batch; b++) { - const pplx_context_input_t *input = &inputs[b]; - packed[b] = input->input; - for (int s = 0; s < input->n_spans; s++) { - h_starts[span_offset] = token_offset + input->spans[s].start; - h_lens[span_offset] = input->spans[s].n_tokens; - span_offset++; - } - token_offset += input->input.n_tokens; - } - - int rc = -1; - cudaError_t ce; - if (cuda_forward_batch(ctx, packed, batch) != 0) - goto cleanup; - if (ensure_pooled_rows(ctx, total_spans) != 0 || - ensure_span_buffers(ctx, total_spans) != 0) - goto cleanup; - ce = cudaMemcpyAsync(ctx->span_starts, h_starts, - (size_t)total_spans * sizeof(int), - cudaMemcpyHostToDevice, ctx->stream); - if (ce == cudaSuccess) - ce = cudaMemcpyAsync(ctx->span_lens, h_lens, - (size_t)total_spans * sizeof(int), - cudaMemcpyHostToDevice, ctx->stream); - if (ce != cudaSuccess) { - fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce)); - goto cleanup; - } - span_pool_kernel<<stream>>>( - ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, - total_spans, c->hidden_size); - if (launch_check() != 0) - goto cleanup; - ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, - (size_t)total_spans * c->hidden_size * sizeof(float), - cudaMemcpyDeviceToHost, ctx->stream); - if (ce == cudaSuccess) - ce = cudaStreamSynchronize(ctx->stream); - if (ce != cudaSuccess) { - fprintf(stderr, "cuda span output copy failed: %s\n", - cudaGetErrorString(ce)); - goto cleanup; - } - rc = 0; - -cleanup: - free(h_lens); - free(h_starts); - free(packed); - return rc; -} - -int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding) -{ - pplx_input_t input = { token_ids, n_tokens }; - return pplx_cuda_embed_batch(ctx, &input, 1, out_embedding); -} - -float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens) -{ - if (!ctx) return NULL; - int hidden = ctx->config.hidden_size; - float *out = (float *)malloc((size_t)hidden * sizeof(float)); - if (!out) return NULL; - if (pplx_cuda_embed_into(ctx, token_ids, n_tokens, out) != 0) { - free(out); - return NULL; - } - return out; -} diff --git a/pplx_embed_cuda.h b/pplx_embed_cuda.h deleted file mode 100644 index badff9c..0000000 --- a/pplx_embed_cuda.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * pplx_embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference. - */ - -#ifndef PPLX_EMBED_CUDA_H -#define PPLX_EMBED_CUDA_H - -#include "pplx_embed.h" - -typedef struct pplx_cuda_ctx pplx_cuda_ctx_t; - -int pplx_cuda_set_fast_gemm(const char *mode); -int pplx_cuda_set_weights_bf16(int on); - -pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir); -void pplx_cuda_free(pplx_cuda_ctx_t *ctx); - -const pplx_config_t *pplx_cuda_config(const pplx_cuda_ctx_t *ctx); - -int pplx_cuda_embed_into(pplx_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding); -float *pplx_cuda_embed(pplx_cuda_ctx_t *ctx, const int *token_ids, - int n_tokens); -int pplx_cuda_embed_batch(pplx_cuda_ctx_t *ctx, const pplx_input_t *inputs, - int batch, float *out_embeddings); -int pplx_cuda_embed_spans_batch(pplx_cuda_ctx_t *ctx, - const pplx_context_input_t *inputs, - int batch, float *out_embeddings); - -#endif /* PPLX_EMBED_CUDA_H */ diff --git a/pplx_embed_internal.h b/pplx_embed_internal.h deleted file mode 100644 index 1e7e4d0..0000000 --- a/pplx_embed_internal.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef PPLX_EMBED_INTERNAL_H -#define PPLX_EMBED_INTERNAL_H - -#include "pplx_embed.h" -#include "qwen_safetensors.h" - -typedef struct { - safetensor_dtype_t dtype; - const void *data; -} pplx_weight_ref_t; - -typedef struct { - pplx_weight_ref_t wq; /* [q_dim, hidden] */ - pplx_weight_ref_t wk; /* [kv_dim, hidden] */ - pplx_weight_ref_t wv; /* [kv_dim, hidden] */ - pplx_weight_ref_t wo; /* [hidden, q_dim] */ - const float *q_norm; /* [head_dim] */ - const float *k_norm; /* [head_dim] */ - const float *input_norm; /* [hidden] */ - const float *post_attn_norm; /* [hidden] */ - pplx_weight_ref_t gate_proj; /* [intermediate, hidden] */ - pplx_weight_ref_t up_proj; /* [intermediate, hidden] */ - pplx_weight_ref_t down_proj; /* [hidden, intermediate] */ -} pplx_layer_t; - -typedef struct { - pplx_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ - pplx_layer_t *layers; /* [n_layers] heap-allocated */ - const float *norm; /* [hidden] */ -} pplx_weights_t; - -struct pplx_model { - pplx_config_t config; - pplx_weights_t weights; - void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ - int layer_start; /* loaded transformer range, inclusive */ - int layer_end; /* loaded transformer range, exclusive */ -}; - -struct pplx_workspace { - const pplx_model_t *model; /* immutable model this workspace belongs to */ - int buf_seq_cap; - float *x; /* [seq, hidden] */ - float *x_norm; /* [seq, hidden] */ - float *q; /* [seq, q_dim] */ - float *k; /* [seq, kv_dim] */ - float *v; /* [seq, kv_dim] */ - float *attn_out; /* [seq, q_dim] */ - float *proj_out; /* [seq, hidden] */ - float *ffn_gate; /* [seq, intermediate] */ - float *ffn_up; /* [seq, intermediate] */ - float *attn_scores; /* reusable BLAS attention score tiles */ - size_t attn_scores_bytes; - - int *offsets; /* [batch + 1] packed sequence offsets */ - int offsets_cap; - - /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */ - float *rope_cos; - float *rope_sin; - int rope_cache_cap; -}; - -#endif /* PPLX_EMBED_INTERNAL_H */ diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c deleted file mode 100644 index 602571d..0000000 --- a/pplx_embed_mlx.c +++ /dev/null @@ -1,2296 +0,0 @@ -#include "pplx_embed_mlx.h" -#include "pplx_embed.h" -#include "qwen_safetensors.h" - -#include - -#include -#include -#include -#include -#include -#include - -/* ======================================================================== - * Per-layer MLX weight arrays - * ======================================================================== */ - -typedef struct { - mlx_array w; - mlx_array wt; - mlx_array scales; - mlx_array biases; - int quantized; - int bits; - int group_size; -} mlx_linear_t; - -typedef struct { - mlx_linear_t wq, wk, wv, wo; - mlx_array q_norm, k_norm; - mlx_array input_norm, post_attn_norm; - mlx_linear_t gate_proj, up_proj, down_proj; -} mlx_layer_t; - -struct pplx_mlx_ctx { - pplx_config_t config; - mlx_array embed_tokens; - mlx_layer_t *layers; /* heap-allocated [n_layers] */ - mlx_array norm; - mlx_stream stream; - multi_safetensors_t *ms; - int layer_start; /* loaded transformer range, inclusive */ - int layer_end; /* loaded transformer range, exclusive */ - int quantize_bits; - int quantize_group_size; - mlx_dtype weight_dtype; -}; - -struct pplx_mlx_late_ctx { - pplx_mlx_ctx_t *base; - multi_safetensors_t *dense_ms; - mlx_linear_t projection; - int token_dim; -}; - -struct pplx_mlx_late_vectors { - pplx_mlx_late_ctx_t *owner; - mlx_array vectors; /* [tokens, dim] */ - int tokens; - int dim; -}; - -/* ======================================================================== - * Helpers - * ======================================================================== */ - -static int arr_ok(mlx_array a) { return a.ctx != NULL; } - -static int mlx_mul_size(size_t a, size_t b, size_t *out) -{ - if (a != 0 && b > SIZE_MAX / a) return -1; - *out = a * b; - return 0; -} - -static mlx_dtype mlx_weight_dtype(const pplx_mlx_ctx_t *ctx) -{ - return ctx->weight_dtype; -} - -static size_t safetensor_dtype_size(safetensor_dtype_t dtype) -{ - switch (dtype) { - case DTYPE_F32: return sizeof(float); - case DTYPE_BF16: return sizeof(uint16_t); - default: return 0; - } -} - -static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, const int *shape, - int ndim) -{ - size_t elem_size = safetensor_dtype_size(t->dtype); - if (elem_size == 0) { - fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", - name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (shape[i] <= 0 || t->shape[i] != shape[i]) { - fprintf(stderr, "mlx: bad shape for %s at dim %d: " - "got %lld, expected %d\n", - name, i, (long long)t->shape[i], shape[i]); - return 0; - } - if (mlx_mul_size(numel, (size_t)shape[i], &numel) != 0) { - fprintf(stderr, "mlx: shape too large for %s\n", name); - return 0; - } - } - - size_t bytes; - if (mlx_mul_size(numel, elem_size, &bytes) != 0) { - fprintf(stderr, "mlx: tensor data too large for %s\n", name); - return 0; - } - if (t->data_size != bytes) { - fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", - name, t->data_size, bytes); - return 0; - } - - if (sf->header_size > SIZE_MAX - 8) return 0; - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || - t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, - const int *shape, int ndim) -{ - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "mlx: tensor not found: %s\n", name); - return (mlx_array){0}; - } - if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim)) - return (mlx_array){0}; - mlx_dtype dtype; - if (t->dtype == DTYPE_F32) { - dtype = MLX_FLOAT32; - } else if (t->dtype == DTYPE_BF16) { - dtype = MLX_BFLOAT16; - } else { - fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); - return (mlx_array){0}; - } - return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); -} - -static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms, - const char *name, - const int *shape, int ndim) -{ - mlx_linear_t l = {0}; - l.w = load_tensor(ms, name, shape, ndim); - return l; -} - -static int linear_ok(mlx_linear_t l) -{ - return arr_ok(l.w) || arr_ok(l.wt); -} - -static void free_linear(mlx_linear_t *l) -{ - mlx_array_free(l->w); - mlx_array_free(l->wt); - mlx_array_free(l->scales); - mlx_array_free(l->biases); - memset(l, 0, sizeof(*l)); -} - -static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) -{ - if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt)) - return 0; - - mlx_array transposed = mlx_array_new(); - mlx_transpose(&transposed, l->w, s); - if (!arr_ok(transposed)) { - mlx_array_free(transposed); - fprintf(stderr, "mlx: failed to prepare transposed linear weight\n"); - return -1; - } - - l->wt = mlx_array_new(); - mlx_contiguous(&l->wt, transposed, false, s); - mlx_array_free(transposed); - if (!arr_ok(l->wt)) { - mlx_array_free(l->wt); - l->wt = (mlx_array){0}; - fprintf(stderr, "mlx: failed to materialize transposed linear weight\n"); - return -1; - } - mlx_array_eval(l->wt); - mlx_synchronize(s); - mlx_array_free(l->w); - l->w = (mlx_array){0}; - return 0; -} - -static int quantize_linear(mlx_linear_t *l, int bits, int group_size, - mlx_stream s) -{ - if (!l || !arr_ok(l->w) || bits == 0) return 0; - if (bits != 8) { - fprintf(stderr, "mlx: quantized weights support only 8 bits\n"); - return -1; - } - if (group_size <= 0) group_size = 64; - - mlx_optional_int gs = {.value = group_size, .has_value = true}; - mlx_optional_int qb = {.value = bits, .has_value = true}; - mlx_array null_arr = (mlx_array){0}; - mlx_vector_array q = mlx_vector_array_new(); - mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s); - if (!q.ctx || mlx_vector_array_size(q) < 2) { - mlx_vector_array_free(q); - fprintf(stderr, "mlx: quantize failed\n"); - return -1; - } - - mlx_array qw = (mlx_array){0}; - mlx_array scales = (mlx_array){0}; - mlx_array biases = (mlx_array){0}; - mlx_vector_array_get(&qw, q, 0); - mlx_vector_array_get(&scales, q, 1); - if (mlx_vector_array_size(q) >= 3) - mlx_vector_array_get(&biases, q, 2); - mlx_vector_array_free(q); - - if (!arr_ok(qw) || !arr_ok(scales)) { - mlx_array_free(qw); - mlx_array_free(scales); - mlx_array_free(biases); - fprintf(stderr, "mlx: quantize returned invalid arrays\n"); - return -1; - } - mlx_array_eval(qw); - mlx_array_eval(scales); - if (arr_ok(biases)) mlx_array_eval(biases); - mlx_synchronize(s); - - mlx_array_free(l->w); - mlx_array_free(l->wt); - l->w = qw; - l->wt = (mlx_array){0}; - l->scales = scales; - l->biases = biases; - l->quantized = 1; - l->bits = bits; - l->group_size = group_size; - return 0; -} - -/* y = x @ W^T */ -static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) -{ - if (W->quantized) { - if (!arr_ok(W->w)) return (mlx_array){0}; - mlx_array res = mlx_array_new(); - mlx_optional_int gs = {.value = W->group_size, .has_value = true}; - mlx_optional_int bits = {.value = W->bits, .has_value = true}; - mlx_array null_arr = (mlx_array){0}; - mlx_quantized_matmul(&res, x, W->w, W->scales, - arr_ok(W->biases) ? W->biases : null_arr, - true, gs, bits, "affine", s); - return res; - } - mlx_array Wt; - int free_wt = 0; - if (arr_ok(W->wt)) { - Wt = W->wt; - } else if (arr_ok(W->w)) { - Wt = mlx_array_new(); - mlx_transpose(&Wt, W->w, s); - free_wt = 1; - } else { - return (mlx_array){0}; - } - mlx_array res = mlx_array_new(); - if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && - mlx_array_dtype(x) == MLX_FLOAT32) { - /* F32 activations with BF16 weights: upcast the weight operand and run - * the matmul in F32 (so accumulation is F32). Only the weights are - * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers), - * matching the CPU/CUDA accuracy while keeping BF16 weight storage. */ - mlx_array Wf = mlx_array_new(); - mlx_astype(&Wf, Wt, MLX_FLOAT32, s); - mlx_matmul(&res, x, Wf, s); - mlx_array_free(Wf); - } else { - mlx_matmul(&res, x, Wt, s); - } - if (free_wt) mlx_array_free(Wt); - return res; -} - -const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx) -{ - return ctx ? &ctx->config : NULL; -} - -/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the - * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the - * residual stream in F32 while keeping the large matmul weights in BF16. */ -static void upcast_f32_inplace(mlx_array *a, mlx_stream s) -{ - if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return; - mlx_array f = mlx_array_new(); - mlx_astype(&f, *a, MLX_FLOAT32, s); - mlx_array_free(*a); - *a = f; -} - -/* ======================================================================== - * Load - * ======================================================================== */ - -/* We duplicate a minimal config parser here so MLX can load standalone. - * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */ - -static const char *skip_ws_m(const char *p) -{ - while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; - return p; -} - -static const char *json_find_m(const char *json, const char *key) -{ - size_t klen = strlen(key); - const char *p = json; - while ((p = strstr(p, key)) != NULL) { - if (p > json && *(p - 1) == '"') { - const char *a = p + klen; - if (*a == '"') { - a = skip_ws_m(a + 1); - if (*a == ':') return skip_ws_m(a + 1); - } - } - p += klen; - } - return NULL; -} - -static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir) -{ - char path[1024]; - int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir); - if (path_len < 0 || (size_t)path_len >= sizeof(path)) { - fprintf(stderr, "mlx: model path too long\n"); - return -1; - } - FILE *f = fopen(path, "rb"); - if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; } - if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - char *buf = (char *)malloc((size_t)sz + 1); - if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); free(buf); return -1; - } - fclose(f); - buf[sz] = '\0'; - -#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0) -#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0) - GI("hidden_size", hidden_size, 0); - GI("num_hidden_layers", n_layers, 0); - GI("num_attention_heads", n_heads, 0); - GI("num_key_value_heads", n_kv_heads, 0); - GI("head_dim", head_dim, PPLX_HEAD_DIM); - GI("intermediate_size", intermediate_size, 0); - GI("vocab_size", vocab_size, PPLX_VOCAB_SIZE); - GF("rms_norm_eps", rms_norm_eps, 1e-6f); - GF("rope_theta", rope_theta, 1000000.0f); -#undef GI -#undef GF - - cfg->q_dim = 0; - cfg->kv_dim = 0; - if (cfg->n_heads > 0 && cfg->head_dim > 0 && - cfg->n_heads <= INT_MAX / cfg->head_dim) - cfg->q_dim = cfg->n_heads * cfg->head_dim; - if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && - cfg->n_kv_heads <= INT_MAX / cfg->head_dim) - cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; - free(buf); - - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || - cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || - cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || - cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || - (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || - !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { - fprintf(stderr, "mlx: invalid config in %s " - "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", - path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, - cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); - return -1; - } - if (cfg->n_layers > PPLX_MAX_LAYERS) { - fprintf(stderr, "mlx: too many layers (%d > %d)\n", - cfg->n_layers, PPLX_MAX_LAYERS); - return -1; - } - return 0; -} - -static int mlx_model_dir_has_file(const char *model_dir, const char *rel) -{ - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) return 0; - FILE *f = fopen(path, "rb"); - if (!f) return 0; - fclose(f); - return 1; -} - -static int mlx_model_dir_has_late_projection(const char *model_dir) -{ - return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") && - mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors"); -} - -static pplx_mlx_options_t normalize_mlx_options(const pplx_mlx_options_t *opts) -{ - pplx_mlx_options_t out = {0}; - if (opts) out = *opts; - if (out.quantize_bits != 0 && - out.quantize_bits != 8) { - fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); - out.quantize_bits = -1; - } - if (out.quantize_bits && out.quantize_group_size <= 0) - out.quantize_group_size = 64; - return out; -} - -static int quantize_layer(mlx_layer_t *l, int bits, int group_size, - mlx_stream s) -{ - if (bits == 0) return 0; - return quantize_linear(&l->wq, bits, group_size, s) || - quantize_linear(&l->wk, bits, group_size, s) || - quantize_linear(&l->wv, bits, group_size, s) || - quantize_linear(&l->wo, bits, group_size, s) || - quantize_linear(&l->gate_proj, bits, group_size, s) || - quantize_linear(&l->up_proj, bits, group_size, s) || - quantize_linear(&l->down_proj, bits, group_size, s); -} - -static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) -{ - return prepare_linear_transpose(&l->wq, s) || - prepare_linear_transpose(&l->wk, s) || - prepare_linear_transpose(&l->wv, s) || - prepare_linear_transpose(&l->wo, s) || - prepare_linear_transpose(&l->gate_proj, s) || - prepare_linear_transpose(&l->up_proj, s) || - prepare_linear_transpose(&l->down_proj, s); -} - -static pplx_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, - int layer_start, int layer_end, - const pplx_mlx_options_t *opts, - int allow_late) -{ - pplx_mlx_options_t options = normalize_mlx_options(opts); - if (options.quantize_bits < 0) return NULL; - if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "mlx: late-interaction models require token-level " - "MaxSim support and are not valid pooled embedding models " - "yet\n"); - return NULL; - } - - pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); - if (!ctx) return NULL; - ctx->quantize_bits = options.quantize_bits; - ctx->quantize_group_size = options.quantize_group_size; - - if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } - - const pplx_config_t *c = &ctx->config; - if (layer_end == -1) - layer_end = c->n_layers; - if (layer_start < 0 || layer_start >= layer_end || - layer_end > c->n_layers) { - fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", - layer_start, layer_end, c->n_layers); - free(ctx); - return NULL; - } - ctx->layer_start = layer_start; - ctx->layer_end = layer_end; - - ctx->ms = multi_safetensors_open(model_dir); - if (!ctx->ms) { - fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); - free(ctx); return NULL; - } - - int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; - int inter = c->intermediate_size, hd = c->head_dim; - - /* Only the first stage needs token embeddings. */ - if (layer_start == 0) { - int emb_shape[] = {c->vocab_size, h}; - ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", - emb_shape, 2); - if (!arr_ok(ctx->embed_tokens)) goto fail; - } - - /* Layers */ - ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); - if (!ctx->layers) goto fail; - - char name[256]; - for (int i = layer_start; i < layer_end; i++) { - mlx_layer_t *l = &ctx->layers[i]; - -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) - - LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); - LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); - LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); - LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); -#undef LD - -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!arr_ok(l->fld)) goto fail; \ -} while (0) - - LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); - LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); - LD(input_norm, "layers.%d.input_layernorm.weight", h); - LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); -#undef LD - -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) - - LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); - LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); - LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); -#undef LD - - if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); - } - - if (layer_end == c->n_layers) { - int ns[] = {h}; - ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); - if (!arr_ok(ctx->norm)) goto fail; - } - - ctx->stream = mlx_default_gpu_stream_new(); - ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w); - /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the - * tiny norm vectors so they match the F32 activations in mlx_forward_layers; - * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */ - if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) { - for (int i = layer_start; i < layer_end; i++) { - mlx_layer_t *l = &ctx->layers[i]; - upcast_f32_inplace(&l->q_norm, ctx->stream); - upcast_f32_inplace(&l->k_norm, ctx->stream); - upcast_f32_inplace(&l->input_norm, ctx->stream); - upcast_f32_inplace(&l->post_attn_norm, ctx->stream); - } - upcast_f32_inplace(&ctx->norm, ctx->stream); - } - if (ctx->quantize_bits) { - for (int i = layer_start; i < layer_end; i++) { - if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, - ctx->quantize_group_size, ctx->stream) != 0) - goto fail; - } - mlx_clear_cache(); - if (pplx_verbose >= 1) - fprintf(stderr, "mlx: quantized linear weights to %d-bit " - "(group_size=%d)\n", - ctx->quantize_bits, ctx->quantize_group_size); - } else { - for (int i = layer_start; i < layer_end; i++) { - if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0) - goto fail; - } - } - - if (pplx_verbose >= 2) { - size_t active = 0, cache = 0, peak = 0; - mlx_get_active_memory(&active); - mlx_get_cache_memory(&cache); - mlx_get_peak_memory(&peak); - fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB " - "peak=%.2f GiB\n", - (double)active / (1024.0 * 1024.0 * 1024.0), - (double)cache / (1024.0 * 1024.0 * 1024.0), - (double)peak / (1024.0 * 1024.0 * 1024.0)); - } - - if (pplx_verbose >= 1) - fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", - layer_start, layer_end, h); - - return ctx; - -fail: - pplx_mlx_free(ctx); - return NULL; -} - -pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) -{ - return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); -} - -pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, - const pplx_mlx_options_t *opts) -{ - return mlx_load_range_ex(model_dir, 0, -1, opts, 0); -} - -pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, - int layer_start, int layer_end) -{ - return mlx_load_range_ex(model_dir, layer_start, layer_end, NULL, 0); -} - -pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( - const char *model_dir, int layer_start, int layer_end, - const pplx_mlx_options_t *opts) -{ - return mlx_load_range_ex(model_dir, layer_start, layer_end, opts, 0); -} - -/* ======================================================================== - * Free - * ======================================================================== */ - -static void free_mlx_layer(mlx_layer_t *l) -{ - free_linear(&l->wq); free_linear(&l->wk); - free_linear(&l->wv); free_linear(&l->wo); - mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); - mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); - free_linear(&l->gate_proj); free_linear(&l->up_proj); - free_linear(&l->down_proj); -} - -void pplx_mlx_free(pplx_mlx_ctx_t *ctx) -{ - if (!ctx) return; - mlx_array_free(ctx->embed_tokens); - if (ctx->layers) { - for (int i = 0; i < ctx->config.n_layers; i++) - free_mlx_layer(&ctx->layers[i]); - free(ctx->layers); - } - mlx_array_free(ctx->norm); - if (ctx->stream.ctx) mlx_stream_free(ctx->stream); - if (ctx->ms) multi_safetensors_close(ctx->ms); - free(ctx); -} - -pplx_mlx_late_ctx_t *pplx_mlx_late_load_with_options( - const char *model_dir, const pplx_mlx_options_t *opts) -{ - if (!mlx_model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "mlx late: missing 1_Dense projection\n"); - return NULL; - } - - pplx_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); - if (!late) return NULL; - - late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1); - if (!late->base) goto fail; - - char dense_dir[1024]; - int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir); - if (n < 0 || (size_t)n >= sizeof(dense_dir)) { - fprintf(stderr, "mlx late: model path too long\n"); - goto fail; - } - - late->dense_ms = multi_safetensors_open(dense_dir); - if (!late->dense_ms) { - fprintf(stderr, "mlx late: failed to open safetensors in %s\n", - dense_dir); - goto fail; - } - - const safetensor_t *t = multi_safetensors_find(late->dense_ms, - "linear.weight", NULL); - if (!t) { - fprintf(stderr, "mlx late: missing linear.weight\n"); - goto fail; - } - if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX || - t->shape[1] != late->base->config.hidden_size) { - fprintf(stderr, "mlx late: bad projection shape\n"); - goto fail; - } - - int shape[] = { (int)t->shape[0], late->base->config.hidden_size }; - late->projection = load_linear_tensor(late->dense_ms, "linear.weight", - shape, 2); - if (!linear_ok(late->projection)) goto fail; - if (prepare_linear_transpose(&late->projection, late->base->stream) != 0) - goto fail; - late->token_dim = shape[0]; - - if (pplx_verbose >= 1) - fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", - late->token_dim); - return late; - -fail: - pplx_mlx_late_free(late); - return NULL; -} - -pplx_mlx_late_ctx_t *pplx_mlx_late_load(const char *model_dir) -{ - return pplx_mlx_late_load_with_options(model_dir, NULL); -} - -void pplx_mlx_late_free(pplx_mlx_late_ctx_t *ctx) -{ - if (!ctx) return; - free_linear(&ctx->projection); - if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms); - pplx_mlx_free(ctx->base); - free(ctx); -} - -const pplx_config_t *pplx_mlx_late_config(const pplx_mlx_late_ctx_t *ctx) -{ - return ctx && ctx->base ? &ctx->base->config : NULL; -} - -int pplx_mlx_late_token_dim(const pplx_mlx_late_ctx_t *ctx) -{ - return ctx ? ctx->token_dim : 0; -} - -/* ======================================================================== - * Forward pass - * ======================================================================== */ - -static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, - int batch, int max_seq, int has_padding, - mlx_array attn_mask, - int layer_start, int layer_end) -{ - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - int n_heads = c->n_heads; - int n_kv_heads = c->n_kv_heads; - int head_dim = c->head_dim; - int q_dim = c->q_dim; - float scale = 1.0f / sqrtf((float)head_dim); - mlx_array null_arr = (mlx_array){0}; - - /* For non-quantized BF16 weights, run the residual stream, norms, and - * attention in F32; matmuls downcast their operands to BF16 internally - * (see linear). This keeps weight storage at BF16 while avoiding the - * cross-layer accumulation error of a fully BF16 forward. F32 and quantized - * models are unaffected. */ - mlx_array local_mask = attn_mask; - int free_mask = 0; - int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 && - !ctx->layers[layer_start].wq.quantized; - if (mixed_bf16) { - if (mlx_array_dtype(x) != MLX_FLOAT32) { - mlx_array xf = mlx_array_new(); - mlx_astype(&xf, x, MLX_FLOAT32, S); - mlx_array_free(x); - x = xf; - } - if (has_padding && arr_ok(attn_mask) && - mlx_array_dtype(attn_mask) != MLX_FLOAT32) { - local_mask = mlx_array_new(); - mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S); - free_mask = 1; - } - } - - for (int layer = layer_start; layer < layer_end; layer++) { - mlx_layer_t *l = &ctx->layers[layer]; - - mlx_array xn = mlx_array_new(); - mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); - - mlx_array q_flat = linear(xn, &l->wq, S); - mlx_array k_flat = linear(xn, &l->wk, S); - mlx_array v_flat = linear(xn, &l->wv, S); - mlx_array_free(xn); - - int q_shape[] = {batch, max_seq, n_heads, head_dim}; - int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; - mlx_array q = mlx_array_new(); - mlx_array k = mlx_array_new(); - mlx_array v = mlx_array_new(); - mlx_reshape(&q, q_flat, q_shape, 4, S); - mlx_reshape(&k, k_flat, k_shape, 4, S); - mlx_reshape(&v, v_flat, k_shape, 4, S); - mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); - - mlx_array qn = mlx_array_new(); - mlx_array kn = mlx_array_new(); - mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); - mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); - mlx_array_free(q); mlx_array_free(k); - - /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ - int perm[] = {0, 2, 1, 3}; - mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); - mlx_transpose_axes(&qt, qn, perm, 4, S); - mlx_transpose_axes(&kt, kn, perm, 4, S); - mlx_transpose_axes(&vt, v, perm, 4, S); - mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v); - - mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; - mlx_array qr = mlx_array_new(), kr = mlx_array_new(); - mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); - mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); - mlx_array_free(qt); mlx_array_free(kt); - - mlx_array attn = mlx_array_new(); - mlx_fast_scaled_dot_product_attention( - &attn, qr, kr, vt, scale, "", - has_padding ? local_mask : null_arr, null_arr, S); - mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); - - mlx_array attn_t = mlx_array_new(); - mlx_transpose_axes(&attn_t, attn, perm, 4, S); - mlx_array_free(attn); - - int flat_shape[] = {batch, max_seq, q_dim}; - mlx_array attn_flat = mlx_array_new(); - mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S); - mlx_array_free(attn_t); - - mlx_array proj = linear(attn_flat, &l->wo, S); - mlx_array_free(attn_flat); - mlx_array x2 = mlx_array_new(); - mlx_add(&x2, x, proj, S); - mlx_array_free(x); mlx_array_free(proj); - x = x2; - - mlx_array xn2 = mlx_array_new(); - mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); - - mlx_array gate = linear(xn2, &l->gate_proj, S); - mlx_array up = linear(xn2, &l->up_proj, S); - mlx_array_free(xn2); - - mlx_array gate_sig = mlx_array_new(); - mlx_sigmoid(&gate_sig, gate, S); - mlx_array silu_gate = mlx_array_new(); - mlx_multiply(&silu_gate, gate, gate_sig, S); - mlx_array_free(gate); mlx_array_free(gate_sig); - - mlx_array mid = mlx_array_new(); - mlx_multiply(&mid, silu_gate, up, S); - mlx_array_free(silu_gate); mlx_array_free(up); - - mlx_array ffn = linear(mid, &l->down_proj, S); - mlx_array_free(mid); - - mlx_array x3 = mlx_array_new(); - mlx_add(&x3, x, ffn, S); - mlx_array_free(x); mlx_array_free(ffn); - x = x3; - } - if (free_mask) mlx_array_free(local_mask); - return x; -} - -pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device( - pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, - int normalize) -{ - if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || - ctx->token_dim <= 0 || !linear_ok(ctx->projection)) - return NULL; - - pplx_mlx_ctx_t *base = ctx->base; - if (base->layer_start != 0 || base->layer_end != base->config.n_layers || - !arr_ok(base->embed_tokens) || !arr_ok(base->norm)) - return NULL; - - const pplx_config_t *c = &base->config; - mlx_stream S = base->stream; - for (int i = 0; i < n_tokens; i++) { - if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { - fprintf(stderr, "mlx late: invalid token id %d at position %d\n", - token_ids[i], i); - return NULL; - } - } - - size_t id_bytes; - if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0) - return NULL; - - int *ids_data = (int *)malloc(id_bytes); - if (!ids_data) return NULL; - memcpy(ids_data, token_ids, id_bytes); - - int ids_shape[] = {1, n_tokens}; - mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32); - free(ids_data); - if (!arr_ok(ids)) return NULL; - - pplx_mlx_late_vectors_t *result = NULL; - mlx_array x = mlx_array_new(); - mlx_array x_normed = (mlx_array){0}; - mlx_array projected = (mlx_array){0}; - mlx_array reshaped = (mlx_array){0}; - - mlx_take_axis(&x, base->embed_tokens, ids, 0, S); - mlx_array_free(ids); - if (!arr_ok(x)) goto cleanup; - - x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, - 0, c->n_layers); - if (!arr_ok(x)) goto cleanup; - - x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S); - mlx_array_free(x); - x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; - - projected = linear(x_normed, &ctx->projection, S); - mlx_array_free(x_normed); - x_normed = (mlx_array){0}; - if (!arr_ok(projected)) goto cleanup; - - if (normalize) { - mlx_array sq = mlx_array_new(); - mlx_square(&sq, projected, S); - if (!arr_ok(sq)) { - mlx_array_free(sq); - goto cleanup; - } - - mlx_array norm2 = mlx_array_new(); - mlx_sum_axis(&norm2, sq, 2, true, S); - mlx_array_free(sq); - if (!arr_ok(norm2)) { - mlx_array_free(norm2); - goto cleanup; - } - - mlx_array inv_norm = mlx_array_new(); - mlx_rsqrt(&inv_norm, norm2, S); - mlx_array_free(norm2); - if (!arr_ok(inv_norm)) { - mlx_array_free(inv_norm); - goto cleanup; - } - - mlx_array normalized = mlx_array_new(); - mlx_multiply(&normalized, projected, inv_norm, S); - mlx_array_free(inv_norm); - mlx_array_free(projected); - projected = normalized; - if (!arr_ok(projected)) goto cleanup; - } - - int vec_shape[] = {n_tokens, ctx->token_dim}; - reshaped = mlx_array_new(); - mlx_reshape(&reshaped, projected, vec_shape, 2, S); - if (!arr_ok(reshaped)) goto cleanup; - - result = (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*result)); - if (!result) goto cleanup; - result->owner = ctx; - result->vectors = reshaped; - result->tokens = n_tokens; - result->dim = ctx->token_dim; - reshaped = (mlx_array){0}; - -cleanup: - mlx_array_free(reshaped); - mlx_array_free(projected); - mlx_array_free(x_normed); - mlx_array_free(x); - return result; -} - -void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs) -{ - if (!vecs) return; - mlx_array_free(vecs->vectors); - free(vecs); -} - -int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs) -{ - return vecs ? vecs->tokens : 0; -} - -int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs) -{ - return vecs ? vecs->dim : 0; -} - -int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs, - float *out_vectors) -{ - if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || - vecs->tokens <= 0 || vecs->dim <= 0) - return -1; - - size_t values, bytes; - if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 || - mlx_mul_size(values, sizeof(float), &bytes) != 0) - return -1; - - mlx_stream S = vecs->owner->base->stream; - mlx_array out_f32 = mlx_array_new(); - mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S); - if (!arr_ok(out_f32)) { - mlx_array_free(out_f32); - return -1; - } - - int rc = -1; - mlx_array_eval(out_f32); - const float *data = mlx_array_data_float32(out_f32); - if (data) { - memcpy(out_vectors, data, bytes); - rc = 0; - } - mlx_array_free(out_f32); - return rc; -} - -pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat( - pplx_mlx_late_ctx_t *ctx, - const pplx_mlx_late_vectors_t *const *items, int count) -{ - if (!ctx || !ctx->base || !items || count <= 0) return NULL; - int dim = 0; - int total = 0; - for (int i = 0; i < count; i++) { - const pplx_mlx_late_vectors_t *v = items[i]; - if (!v || v->owner != ctx || !arr_ok(v->vectors) || - v->tokens <= 0 || v->dim <= 0) - return NULL; - if (i == 0) dim = v->dim; - if (v->dim != dim || total > INT_MAX - v->tokens) - return NULL; - total += v->tokens; - } - - mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts)); - if (!parts) return NULL; - for (int i = 0; i < count; i++) - parts[i] = items[i]->vectors; - - mlx_stream S = ctx->base->stream; - mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count); - free(parts); - if (!vec.ctx) return NULL; - - mlx_array joined = mlx_array_new(); - mlx_concatenate_axis(&joined, vec, 0, S); - mlx_vector_array_free(vec); - if (!arr_ok(joined)) { - mlx_array_free(joined); - return NULL; - } - - pplx_mlx_late_vectors_t *out = - (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out)); - if (!out) { - mlx_array_free(joined); - return NULL; - } - out->owner = ctx; - out->vectors = joined; - out->tokens = total; - out->dim = dim; - return out; -} - -pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select( - pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs, - const int *token_indices, int count) -{ - if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || - !arr_ok(vecs->vectors) || !token_indices || count <= 0 || - vecs->tokens <= 0 || vecs->dim <= 0) - return NULL; - - for (int i = 0; i < count; i++) { - if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens) - return NULL; - } - - size_t idx_bytes; - if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0) - return NULL; - int *idx_data = (int *)malloc(idx_bytes); - if (!idx_data) return NULL; - memcpy(idx_data, token_indices, idx_bytes); - - int idx_shape[] = {count}; - mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32); - free(idx_data); - if (!arr_ok(idx)) return NULL; - - mlx_stream S = ctx->base->stream; - mlx_array selected = mlx_array_new(); - mlx_take_axis(&selected, vecs->vectors, idx, 0, S); - mlx_array_free(idx); - if (!arr_ok(selected)) { - mlx_array_free(selected); - return NULL; - } - - pplx_mlx_late_vectors_t *out = - (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out)); - if (!out) { - mlx_array_free(selected); - return NULL; - } - out->owner = ctx; - out->vectors = selected; - out->tokens = count; - out->dim = vecs->dim; - return out; -} - -int pplx_mlx_late_maxsim_batch_device( - pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query, - const pplx_mlx_late_vectors_t *docs, const int *doc_offsets, - int docs_count, float *scores) -{ - if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || - !scores || query->owner != ctx || docs->owner != ctx || - query->dim <= 0 || query->dim != docs->dim || - query->tokens <= 0 || docs->tokens <= 0 || - !arr_ok(query->vectors) || !arr_ok(docs->vectors) || - doc_offsets[0] != 0) - return -1; - - int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths)); - if (!lengths) return -1; - int max_doc_tokens = 0; - for (int i = 0; i < docs_count; i++) { - if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] || - doc_offsets[i + 1] > docs->tokens) { - free(lengths); - return -1; - } - int n = doc_offsets[i + 1] - doc_offsets[i]; - lengths[i] = n; - if (n > max_doc_tokens) max_doc_tokens = n; - } - if (max_doc_tokens <= 0) { - free(lengths); - return -1; - } - - mlx_stream S = ctx->base->stream; - int dim = query->dim; - int rc = -1; - mlx_array *doc_rows = - (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); - if (!doc_rows) { - free(lengths); - return -1; - } - - mlx_vector_array doc_vec = (mlx_vector_array){0}; - mlx_array pad_value = mlx_array_new_float32(0.0f); - mlx_array doc_batch = (mlx_array){0}; - mlx_array query_3d = (mlx_array){0}; - mlx_array query_batched = (mlx_array){0}; - mlx_array docs_t = (mlx_array){0}; - mlx_array sims = (mlx_array){0}; - mlx_array lengths_arr = (mlx_array){0}; - mlx_array positions = (mlx_array){0}; - mlx_array lengths_2d = (mlx_array){0}; - mlx_array positions_2d = (mlx_array){0}; - mlx_array valid = (mlx_array){0}; - mlx_array valid_3d = (mlx_array){0}; - mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX); - mlx_array masked_sims = (mlx_array){0}; - mlx_array maxes = (mlx_array){0}; - mlx_array sums = (mlx_array){0}; - mlx_array sums_f32 = (mlx_array){0}; - - for (int i = 0; i < docs_count; i++) { - int n = lengths[i]; - int start[] = {doc_offsets[i], 0}; - int stop[] = {doc_offsets[i + 1], dim}; - int strides[] = {1, 1}; - - mlx_array doc = mlx_array_new(); - mlx_array padded = (mlx_array){0}; - - mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S); - if (!arr_ok(doc)) goto cleanup; - - if (n < max_doc_tokens) { - int axes[] = {0}; - int low[] = {0}; - int high[] = {max_doc_tokens - n}; - padded = mlx_array_new(); - mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, - "constant", S); - mlx_array_free(doc); - doc = padded; - padded = (mlx_array){0}; - if (!arr_ok(doc)) goto cleanup; - } - - doc_rows[i] = mlx_array_new(); - mlx_expand_dims(&doc_rows[i], doc, 0, S); - mlx_array_free(doc); - if (!arr_ok(doc_rows[i])) goto cleanup; - } - - doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count); - if (!doc_vec.ctx) goto cleanup; - - doc_batch = mlx_array_new(); - mlx_concatenate_axis(&doc_batch, doc_vec, 0, S); - if (!arr_ok(doc_batch)) goto cleanup; - - query_3d = mlx_array_new(); - mlx_expand_dims(&query_3d, query->vectors, 0, S); - if (!arr_ok(query_3d)) goto cleanup; - - int q_shape[] = {docs_count, query->tokens, dim}; - query_batched = mlx_array_new(); - mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S); - if (!arr_ok(query_batched)) goto cleanup; - - int perm[] = {0, 2, 1}; - docs_t = mlx_array_new(); - mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S); - if (!arr_ok(docs_t)) goto cleanup; - - sims = mlx_array_new(); - mlx_matmul(&sims, query_batched, docs_t, S); - if (!arr_ok(sims)) goto cleanup; - - int len_shape[] = {docs_count}; - lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32); - if (!arr_ok(lengths_arr)) goto cleanup; - - positions = mlx_array_new(); - mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S); - if (!arr_ok(positions)) goto cleanup; - - lengths_2d = mlx_array_new(); - mlx_expand_dims(&lengths_2d, lengths_arr, 1, S); - if (!arr_ok(lengths_2d)) goto cleanup; - - positions_2d = mlx_array_new(); - mlx_expand_dims(&positions_2d, positions, 0, S); - if (!arr_ok(positions_2d)) goto cleanup; - - valid = mlx_array_new(); - mlx_less(&valid, positions_2d, lengths_2d, S); - if (!arr_ok(valid)) goto cleanup; - - valid_3d = mlx_array_new(); - mlx_expand_dims(&valid_3d, valid, 1, S); - if (!arr_ok(valid_3d)) goto cleanup; - - masked_sims = mlx_array_new(); - mlx_where(&masked_sims, valid_3d, sims, invalid_value, S); - if (!arr_ok(masked_sims)) goto cleanup; - - maxes = mlx_array_new(); - mlx_max_axis(&maxes, masked_sims, 2, false, S); - if (!arr_ok(maxes)) goto cleanup; - - sums = mlx_array_new(); - mlx_sum_axis(&sums, maxes, 1, false, S); - if (!arr_ok(sums)) goto cleanup; - - sums_f32 = mlx_array_new(); - mlx_astype(&sums_f32, sums, MLX_FLOAT32, S); - if (!arr_ok(sums_f32)) goto cleanup; - - mlx_array_eval(sums_f32); - const float *data = mlx_array_data_float32(sums_f32); - if (!data) goto cleanup; - memcpy(scores, data, (size_t)docs_count * sizeof(float)); - rc = 0; - -cleanup: - mlx_array_free(sums_f32); - mlx_array_free(sums); - mlx_array_free(maxes); - mlx_array_free(masked_sims); - mlx_array_free(invalid_value); - mlx_array_free(valid_3d); - mlx_array_free(valid); - mlx_array_free(positions_2d); - mlx_array_free(lengths_2d); - mlx_array_free(positions); - mlx_array_free(lengths_arr); - mlx_array_free(sims); - mlx_array_free(docs_t); - mlx_array_free(query_batched); - mlx_array_free(query_3d); - mlx_array_free(doc_batch); - if (doc_vec.ctx) mlx_vector_array_free(doc_vec); - mlx_array_free(pad_value); - for (int i = 0; i < docs_count; i++) - mlx_array_free(doc_rows[i]); - free(doc_rows); - free(lengths); - return rc; -} - -int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors) -{ - pplx_mlx_late_vectors_t *vecs = - pplx_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, - normalize); - if (!vecs) return -1; - int rc = pplx_mlx_late_vectors_copy(vecs, out_vectors); - pplx_mlx_late_vectors_free(vecs); - return rc; -} - -static int pplx_mlx_embed_batch_dense(pplx_mlx_ctx_t *ctx, - const pplx_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) - return -1; - - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - - int hidden = c->hidden_size; - int max_seq = 0; - int has_padding = 0; - - for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - } - if (max_seq <= 0) return -1; - - for (int b = 0; b < batch; b++) { - if (inputs[b].n_tokens != max_seq) has_padding = 1; - } - - size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 || - mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 || - mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 || - mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || - mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || - mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) - return -1; - int *padded_ids = (int *)malloc(ids_bytes); - float *pool_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; - float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; - float *len_data = has_padding ? (float *)malloc(len_bytes) : NULL; - if (!padded_ids || (has_padding && (!pool_mask_data || !attn_mask_data || !len_data))) { - free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); - return -1; - } - - for (int b = 0; b < batch; b++) { - if (has_padding) len_data[b] = (float)inputs[b].n_tokens; - for (int t = 0; t < max_seq; t++) { - size_t idx = (size_t)b * max_seq + t; - if (t < inputs[b].n_tokens) { - int id = inputs[b].ids[t]; - if (id < 0 || id >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - id, b, t); - free(padded_ids); free(pool_mask_data); - free(attn_mask_data); free(len_data); - return -1; - } - padded_ids[idx] = id; - if (has_padding) { - pool_mask_data[idx] = 1.0f; - attn_mask_data[idx] = 0.0f; - } - } else { - padded_ids[idx] = 0; - pool_mask_data[idx] = 0.0f; - attn_mask_data[idx] = -1.0e9f; - } - } - } - - int ids_shape[] = {batch, max_seq}; - int pool_mask_shape[] = {batch, max_seq, 1}; - int attn_mask_shape[] = {batch, 1, 1, max_seq}; - int len_shape[] = {batch, 1}; - - mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - mlx_array pool_mask = has_padding - ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array lengths = has_padding - ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) - : (mlx_array){0}; - free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); - - if (!arr_ok(ids) || (has_padding && (!arr_ok(pool_mask) || - !arr_ok(attn_mask) || !arr_ok(lengths)))) { - mlx_array_free(ids); mlx_array_free(pool_mask); - if (has_padding) mlx_array_free(attn_mask); - mlx_array_free(lengths); - return -1; - } - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - mlx_array attn_mask_bf16 = mlx_array_new(); - mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); - mlx_array_free(attn_mask); - attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) { - mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths); - return -1; - } - } - - /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */ - mlx_array x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); - mlx_array_free(ids); - - /* 2. Transformer layers. */ - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - 0, c->n_layers); - - /* 3. Final RMSNorm. */ - mlx_array x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); - mlx_array_free(x); - - mlx_array emb = mlx_array_new(); - if (has_padding) { - /* 4. Masked mean pool over T. */ - mlx_array masked = mlx_array_new(); - mlx_multiply(&masked, x_normed, pool_mask, S); - mlx_array_free(x_normed); - mlx_array_free(pool_mask); - - mlx_array emb_sum = mlx_array_new(); - mlx_sum_axis(&emb_sum, masked, 1, false, S); - mlx_array_free(masked); - - mlx_divide(&emb, emb_sum, lengths, S); - mlx_array_free(emb_sum); mlx_array_free(lengths); - mlx_array_free(attn_mask); - } else { - /* Equal-length batches do not need masks or a separate divide. */ - mlx_mean_axis(&emb, x_normed, 1, false, S); - mlx_array_free(x_normed); - } - - /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are - * intentionally unnormalized; callers should use cosine similarity. */ - mlx_array emb_f32 = mlx_array_new(); - mlx_astype(&emb_f32, emb, MLX_FLOAT32, S); - mlx_array_free(emb); - - mlx_array_eval(emb_f32); - const float *data = mlx_array_data_float32(emb_f32); - int rc = -1; - if (data) { - memcpy(out_embeddings, data, out_bytes); - rc = 0; - } - mlx_array_free(emb_f32); - return rc; -} - -typedef struct { - int index; - const pplx_input_t *input; -} mlx_batch_item_t; - -static int mlx_batch_item_cmp(const void *a, const void *b) -{ - const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a; - const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b; - int da = ia->input->n_tokens; - int db = ib->input->n_tokens; - return (da > db) - (da < db); -} - -static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) -{ - if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5) - return 1; - return padded * 5 > tokens * 11; -} - -static int mlx_batch_should_split(const pplx_input_t *inputs, int batch, - int *out_total_tokens, int *out_max_seq) -{ - int max_seq = 0; - int total_tokens = 0; - for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0; - total_tokens += inputs[b].n_tokens; - } - if (out_total_tokens) *out_total_tokens = total_tokens; - if (out_max_seq) *out_max_seq = max_seq; - if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0; - - size_t padded; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) - return 0; - - return mlx_padding_exceeds_limit(padded, (size_t)total_tokens); -} - -static int mlx_should_add_to_group(const pplx_mlx_ctx_t *ctx, - int group_count, int group_max, - int next_tokens) -{ - if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0) - return 1; - - int next_count = group_count + 1; - int next_max = next_tokens > group_max ? next_tokens : group_max; - size_t merged, split_left, split_total; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, &merged) != 0 || - mlx_mul_size((size_t)group_count, (size_t)group_max, &split_left) != 0) - return 0; - /* Treat one extra MLX launch as roughly 256 padded token rows for 4B. - * This preserves small mixed groups while splitting larger padded tails. */ - if (split_left > SIZE_MAX - (size_t)next_tokens || - split_left + (size_t)next_tokens > SIZE_MAX - 256) - return 0; - split_total = split_left + (size_t)next_tokens + 256; - return merged <= split_total; -} - -int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) - return -1; - - if (!mlx_batch_should_split(inputs, batch, NULL, NULL)) - return pplx_mlx_embed_batch_dense(ctx, inputs, batch, out_embeddings); - - const int hidden = ctx->config.hidden_size; - mlx_batch_item_t *items = (mlx_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - pplx_input_t *group_inputs = (pplx_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); - size_t group_values, group_bytes; - if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 || - mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); - return -1; - } - float *group_out = (float *)malloc(group_bytes); - if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); - return -1; - } - for (int i = 0; i < batch; i++) { - items[i].index = i; - items[i].input = &inputs[i]; - } - qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp); - - int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { - int group_max = 0; - int group_tokens = 0; - int end = start; - while (end < batch) { - int next_tokens = items[end].input->n_tokens; - int next_count = end - start + 1; - int next_max = next_tokens > group_max ? next_tokens : group_max; - int next_total = group_tokens + next_tokens; - if (end > start) { - size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) - break; - } - group_max = next_max; - group_tokens = next_total; - end++; - } - - int group_count = end - start; - for (int i = 0; i < group_count; i++) - group_inputs[i] = *items[start + i].input; - rc = pplx_mlx_embed_batch_dense(ctx, group_inputs, group_count, - group_out); - if (rc != 0) break; - for (int i = 0; i < group_count; i++) { - int dst = items[start + i].index; - memcpy(out_embeddings + (size_t)dst * hidden, - group_out + (size_t)i * hidden, - (size_t)hidden * sizeof(float)); - } - start = end; - } - - free(items); free(group_inputs); free(group_out); - return rc; -} - -int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding) -{ - pplx_input_t input = { token_ids, n_tokens }; - return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding); -} - -int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states) -{ - if (!ctx || !inputs || batch <= 0 || !out_states || - layer_start < 0 || layer_start > layer_end || - layer_end > ctx->config.n_layers || - layer_start < ctx->layer_start || layer_end > ctx->layer_end || - (layer_start == 0 && input_states) || - (layer_start != 0 && !input_states) || - (layer_start == 0 && !arr_ok(ctx->embed_tokens)) || - (apply_final_norm && - (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm)))) - return -1; - - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int max_seq = 0; - int total_seq = 0; - int has_padding = 0; - for (int b = 0; b < batch; b++) { - if ((layer_start == 0 && !inputs[b].ids) || inputs[b].n_tokens <= 0 || - total_seq > INT_MAX - inputs[b].n_tokens) - return -1; - total_seq += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - } - for (int b = 0; b < batch; b++) { - if (inputs[b].n_tokens != max_seq) has_padding = 1; - } - - size_t rows, ids_bytes, mask_bytes, state_bytes, row_bytes; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || - mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || - mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || - mlx_mul_size((size_t)hidden, sizeof(float), &row_bytes) != 0) - return -1; - if (rows == 0 || row_bytes == 0 || (has_padding && mask_bytes == 0)) - return -1; - if (layer_start != 0 && - mlx_mul_size(rows, (size_t)hidden, &state_bytes) != 0) - return -1; - if (layer_start == 0) - state_bytes = 0; - - int *padded_ids = layer_start == 0 - ? (int *)malloc(ids_bytes) : NULL; - float *padded_states = layer_start != 0 - ? (float *)calloc(1, state_bytes) : NULL; - float *attn_mask_data = has_padding - ? (float *)malloc(mask_bytes) : NULL; - if ((layer_start == 0 && !padded_ids) || - (layer_start != 0 && !padded_states) || - (has_padding && !attn_mask_data)) { - free(padded_ids); free(padded_states); free(attn_mask_data); - return -1; - } - - int packed_offset = 0; - for (int b = 0; b < batch; b++) { - for (int t = 0; t < max_seq; t++) { - size_t dense_row = (size_t)b * max_seq + t; - if (t < inputs[b].n_tokens) { - if (layer_start == 0) { - int id = inputs[b].ids[t]; - if (id < 0 || id >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - id, b, t); - free(padded_ids); free(padded_states); - free(attn_mask_data); - return -1; - } - padded_ids[dense_row] = id; - } else { - memcpy(padded_states + dense_row * hidden, - input_states + (size_t)(packed_offset + t) * hidden, - row_bytes); - } - if (has_padding) attn_mask_data[dense_row] = 0.0f; - } else { - if (layer_start == 0) padded_ids[dense_row] = 0; - if (has_padding) attn_mask_data[dense_row] = -1.0e9f; - } - } - packed_offset += inputs[b].n_tokens; - } - - int attn_mask_shape[] = {batch, 1, 1, max_seq}; - mlx_array attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - free(attn_mask_data); - if (has_padding && !arr_ok(attn_mask)) { - free(padded_ids); free(padded_states); - return -1; - } - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - mlx_array attn_mask_bf16 = mlx_array_new(); - mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); - mlx_array_free(attn_mask); - attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) { - free(padded_ids); free(padded_states); - return -1; - } - } - - mlx_array x; - if (layer_start == 0) { - int ids_shape[] = {batch, max_seq}; - mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - free(padded_ids); - if (!arr_ok(ids)) { - if (has_padding) mlx_array_free(attn_mask); - return -1; - } - x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); - mlx_array_free(ids); - } else { - int states_shape[] = {batch, max_seq, hidden}; - mlx_array states = mlx_array_new_data(padded_states, states_shape, 3, - MLX_FLOAT32); - free(padded_states); - if (!arr_ok(states)) { - if (has_padding) mlx_array_free(attn_mask); - return -1; - } - if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - x = mlx_array_new(); - mlx_astype(&x, states, MLX_BFLOAT16, S); - mlx_array_free(states); - } else { - x = states; - } - } - - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - layer_start, layer_end); - if (has_padding) mlx_array_free(attn_mask); - - if (apply_final_norm) { - mlx_array x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); - mlx_array_free(x); - x = x_normed; - } - - mlx_array x_f32 = mlx_array_new(); - mlx_astype(&x_f32, x, MLX_FLOAT32, S); - mlx_array_free(x); - mlx_array_eval(x_f32); - - const float *data = mlx_array_data_float32(x_f32); - if (!data) { - mlx_array_free(x_f32); - return -1; - } - packed_offset = 0; - for (int b = 0; b < batch; b++) { - memcpy(out_states + (size_t)packed_offset * hidden, - data + (size_t)b * max_seq * hidden, - (size_t)inputs[b].n_tokens * row_bytes); - packed_offset += inputs[b].n_tokens; - } - mlx_array_free(x_f32); - return 0; -} - -typedef struct { - int index; - int span_offset; - const pplx_context_input_t *input; -} mlx_context_batch_item_t; - -static int mlx_context_batch_item_cmp(const void *a, const void *b) -{ - const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a; - const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b; - int da = ia->input->input.n_tokens; - int db = ib->input->input.n_tokens; - return (da > db) - (da < db); -} - -static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) -{ - if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4) - return 1; - return padded * 4 > tokens * 5; -} - -static int mlx_context_batch_should_split(const pplx_context_input_t *inputs, - int batch, int *out_total_spans) -{ - int max_seq = 0; - int total_tokens = 0; - int total_spans = 0; - for (int b = 0; b < batch; b++) { - int n_tokens = inputs[b].input.n_tokens; - if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || - inputs[b].n_spans <= 0) - return 0; - if (n_tokens > max_seq) max_seq = n_tokens; - if (total_tokens > INT_MAX - n_tokens || - total_spans > INT_MAX - inputs[b].n_spans) - return 0; - total_tokens += n_tokens; - total_spans += inputs[b].n_spans; - } - if (out_total_spans) *out_total_spans = total_spans; - if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0; - - size_t padded; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) - return 0; - return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens); -} - -static int pplx_mlx_embed_spans_batch_dense(pplx_mlx_ctx_t *ctx, - const pplx_context_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || - ctx->layer_end != ctx->config.n_layers || - !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) - return -1; - - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int max_seq = 0; - int total_spans = 0; - int has_padding = 0; - for (int b = 0; b < batch; b++) { - const pplx_context_input_t *input = &inputs[b]; - int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans) - return -1; - total_spans += input->n_spans; - if (n_tokens > max_seq) max_seq = n_tokens; - for (int t = 0; t < n_tokens; t++) { - if (input->input.ids[t] < 0 || - input->input.ids[t] >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - input->input.ids[t], b, t); - return -1; - } - } - for (int s = 0; s < input->n_spans; s++) { - if (input->spans[s].start < 0 || - input->spans[s].start > n_tokens || - input->spans[s].n_tokens <= 0 || - input->spans[s].n_tokens > n_tokens - input->spans[s].start) - return -1; - } - } - for (int b = 0; b < batch; b++) - if (inputs[b].input.n_tokens != max_seq) has_padding = 1; - - size_t rows, ids_bytes, mask_bytes, index_bytes; - size_t span_float_bytes, out_values, out_bytes; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || - mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || - mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || - mlx_mul_size((size_t)total_spans, sizeof(int), &index_bytes) != 0 || - mlx_mul_size((size_t)total_spans, sizeof(float), &span_float_bytes) != 0 || - mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 || - mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) - return -1; - if (rows > INT_MAX) return -1; - int *padded_ids = (int *)malloc(ids_bytes); - float *attn_mask_data = has_padding - ? (float *)malloc(mask_bytes) : NULL; - if (!padded_ids || (has_padding && !attn_mask_data)) { - free(padded_ids); - free(attn_mask_data); - return -1; - } - for (int b = 0; b < batch; b++) { - const pplx_input_t *input = &inputs[b].input; - for (int t = 0; t < max_seq; t++) { - size_t row = (size_t)b * max_seq + t; - if (t < input->n_tokens) { - padded_ids[row] = input->ids[t]; - if (has_padding) attn_mask_data[row] = 0.0f; - } else { - padded_ids[row] = 0; - if (has_padding) attn_mask_data[row] = -1.0e9f; - } - } - } - - mlx_array ids = (mlx_array){0}; - mlx_array attn_mask = (mlx_array){0}; - mlx_array x = (mlx_array){0}; - mlx_array x_normed = (mlx_array){0}; - mlx_array prefix = (mlx_array){0}; - mlx_array flat_prefix = (mlx_array){0}; - mlx_array end_idx = (mlx_array){0}; - mlx_array start_idx = (mlx_array){0}; - mlx_array lengths = (mlx_array){0}; - mlx_array start_mask = (mlx_array){0}; - mlx_array end_values = (mlx_array){0}; - mlx_array start_values = (mlx_array){0}; - mlx_array masked_start = (mlx_array){0}; - mlx_array span_sums = (mlx_array){0}; - mlx_array embeddings = (mlx_array){0}; - mlx_array embeddings_f32 = (mlx_array){0}; - mlx_array *pooled = NULL; - mlx_vector_array pooled_vec = (mlx_vector_array){0}; - int *end_idx_data = NULL; - int *start_idx_data = NULL; - float *lengths_data = NULL; - float *start_mask_data = NULL; - int rc = -1; - - int ids_shape[] = {batch, max_seq}; - int attn_mask_shape[] = {batch, 1, 1, max_seq}; - ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - free(padded_ids); - free(attn_mask_data); - if (!arr_ok(ids)) goto cleanup; - if (has_padding && !arr_ok(attn_mask)) goto cleanup; - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - mlx_array attn_mask_bf16 = mlx_array_new(); - mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); - mlx_array_free(attn_mask); - attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) goto cleanup; - } - - x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); - mlx_array_free(ids); - ids = (mlx_array){0}; - if (!arr_ok(x)) goto cleanup; - - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - 0, c->n_layers); - if (has_padding) { - mlx_array_free(attn_mask); - attn_mask = (mlx_array){0}; - } - if (!arr_ok(x)) goto cleanup; - - x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); - mlx_array_free(x); - x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; - - if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled)); - if (!pooled) goto cleanup; - - int pooled_count = 0; - for (int b = 0; b < batch; b++) { - for (int s = 0; s < inputs[b].n_spans; s++) { - const pplx_span_t *span = &inputs[b].spans[s]; - int start[] = {b, span->start, 0}; - int stop[] = {b + 1, span->start + span->n_tokens, hidden}; - int strides[] = {1, 1, 1}; - mlx_array slice = mlx_array_new(); - mlx_slice(&slice, x_normed, start, 3, stop, 3, strides, 3, S); - if (!arr_ok(slice)) { - mlx_array_free(slice); - goto cleanup; - } - - pooled[pooled_count] = mlx_array_new(); - mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S); - mlx_array_free(slice); - if (!arr_ok(pooled[pooled_count])) goto cleanup; - pooled_count++; - } - } - - pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans); - if (!pooled_vec.ctx) goto cleanup; - embeddings = mlx_array_new(); - mlx_concatenate_axis(&embeddings, pooled_vec, 0, S); - if (!arr_ok(embeddings)) goto cleanup; - } else { - end_idx_data = (int *)malloc(index_bytes); - start_idx_data = (int *)malloc(index_bytes); - lengths_data = (float *)malloc(span_float_bytes); - start_mask_data = (float *)malloc(span_float_bytes); - if (!end_idx_data || !start_idx_data || !lengths_data || - !start_mask_data) - goto cleanup; - - int span_count = 0; - for (int b = 0; b < batch; b++) { - for (int s = 0; s < inputs[b].n_spans; s++) { - const pplx_span_t *span = &inputs[b].spans[s]; - int base = b * max_seq; - int start = span->start; - int stop = span->start + span->n_tokens; - end_idx_data[span_count] = base + stop - 1; - start_idx_data[span_count] = start > 0 ? base + start - 1 : base; - lengths_data[span_count] = (float)span->n_tokens; - start_mask_data[span_count] = start > 0 ? 1.0f : 0.0f; - span_count++; - } - } - - int index_shape[] = {total_spans}; - int scale_shape[] = {total_spans, 1}; - end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32); - start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, - MLX_INT32); - lengths = mlx_array_new_data(lengths_data, scale_shape, 2, - MLX_FLOAT32); - start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, - MLX_FLOAT32); - free(end_idx_data); end_idx_data = NULL; - free(start_idx_data); start_idx_data = NULL; - free(lengths_data); lengths_data = NULL; - free(start_mask_data); start_mask_data = NULL; - if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || - !arr_ok(start_mask)) - goto cleanup; - - prefix = mlx_array_new(); - mlx_cumsum(&prefix, x_normed, 1, false, true, S); - mlx_array_free(x_normed); - x_normed = (mlx_array){0}; - if (!arr_ok(prefix)) goto cleanup; - - int flat_shape[] = {(int)rows, hidden}; - flat_prefix = mlx_array_new(); - mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S); - mlx_array_free(prefix); - prefix = (mlx_array){0}; - if (!arr_ok(flat_prefix)) goto cleanup; - - end_values = mlx_array_new(); - mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S); - if (!arr_ok(end_values)) goto cleanup; - - start_values = mlx_array_new(); - mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S); - if (!arr_ok(start_values)) goto cleanup; - - masked_start = mlx_array_new(); - mlx_multiply(&masked_start, start_values, start_mask, S); - if (!arr_ok(masked_start)) goto cleanup; - - span_sums = mlx_array_new(); - mlx_subtract(&span_sums, end_values, masked_start, S); - if (!arr_ok(span_sums)) goto cleanup; - - embeddings = mlx_array_new(); - mlx_divide(&embeddings, span_sums, lengths, S); - if (!arr_ok(embeddings)) goto cleanup; - } - - embeddings_f32 = mlx_array_new(); - mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S); - if (!arr_ok(embeddings_f32)) goto cleanup; - - mlx_array_eval(embeddings_f32); - const float *data = mlx_array_data_float32(embeddings_f32); - if (data) { - memcpy(out_embeddings, data, out_bytes); - rc = 0; - } - -cleanup: - free(start_mask_data); - free(lengths_data); - free(start_idx_data); - free(end_idx_data); - mlx_array_free(embeddings_f32); - mlx_array_free(embeddings); - if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec); - if (pooled) { - for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]); - free(pooled); - } - mlx_array_free(span_sums); - mlx_array_free(masked_start); - mlx_array_free(start_values); - mlx_array_free(end_values); - mlx_array_free(start_mask); - mlx_array_free(lengths); - mlx_array_free(start_idx); - mlx_array_free(end_idx); - mlx_array_free(flat_prefix); - mlx_array_free(prefix); - mlx_array_free(x_normed); - mlx_array_free(x); - mlx_array_free(attn_mask); - mlx_array_free(ids); - return rc; -} - -int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, - const pplx_context_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) - return -1; - - int total_spans = 0; - if (!mlx_context_batch_should_split(inputs, batch, &total_spans)) - return pplx_mlx_embed_spans_batch_dense(ctx, inputs, batch, - out_embeddings); - - const int hidden = ctx->config.hidden_size; - mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - pplx_context_input_t *group_inputs = (pplx_context_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); - size_t group_values, group_bytes; - if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 || - mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); - return -1; - } - float *group_out = (float *)malloc(group_bytes); - if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); - return -1; - } - - int span_offset = 0; - for (int i = 0; i < batch; i++) { - items[i].index = i; - items[i].span_offset = span_offset; - items[i].input = &inputs[i]; - span_offset += inputs[i].n_spans; - } - qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp); - - int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { - int group_max = 0; - int group_tokens = 0; - int end = start; - while (end < batch) { - int next_tokens = items[end].input->input.n_tokens; - int next_count = end - start + 1; - int next_max = next_tokens > group_max ? next_tokens : group_max; - int next_total = group_tokens + next_tokens; - if (end > start) { - size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_context_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) - break; - } - group_max = next_max; - group_tokens = next_total; - end++; - } - - int group_count = end - start; - for (int i = 0; i < group_count; i++) - group_inputs[i] = *items[start + i].input; - rc = pplx_mlx_embed_spans_batch_dense(ctx, group_inputs, - group_count, group_out); - if (rc != 0) break; - int src_span = 0; - for (int i = 0; i < group_count; i++) { - int n_spans = group_inputs[i].n_spans; - int dst_span = items[start + i].span_offset; - memcpy(out_embeddings + (size_t)dst_span * hidden, - group_out + (size_t)src_span * hidden, - (size_t)n_spans * hidden * sizeof(float)); - src_span += n_spans; - } - start = end; - } - - free(items); free(group_inputs); free(group_out); - return rc; -} - -int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, const pplx_span_t *spans, - int n_spans, float *out_embeddings) -{ - pplx_context_input_t input = { - .input = { token_ids, n_tokens }, - .spans = spans, - .n_spans = n_spans, - }; - return pplx_mlx_embed_spans_batch(ctx, &input, 1, out_embeddings); -} - -float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) -{ - if (!ctx || !token_ids || n_tokens <= 0) return NULL; - - int hidden = ctx->config.hidden_size; - size_t out_bytes; - if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0) - return NULL; - float *out = (float *)malloc(out_bytes); - if (!out) return NULL; - - if (pplx_mlx_embed_into(ctx, token_ids, n_tokens, out) != 0) { - free(out); - return NULL; - } - return out; -} diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h deleted file mode 100644 index f2ad136..0000000 --- a/pplx_embed_mlx.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - * pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference - * - * Uses Apple's mlx-c to run the transformer on Metal. - */ - -#ifndef PPLX_EMBED_MLX_H -#define PPLX_EMBED_MLX_H - -#include -#include -#include "pplx_embed.h" - -/* Opaque MLX model context */ -typedef struct pplx_mlx_ctx pplx_mlx_ctx_t; -typedef struct pplx_mlx_late_ctx pplx_mlx_late_ctx_t; -typedef struct pplx_mlx_late_vectors pplx_mlx_late_vectors_t; - -typedef struct { - int quantize_bits; /* 0 disables; currently 8 */ - int quantize_group_size; /* default 64 when quantize_bits is set */ -} pplx_mlx_options_t; - -/* - * Load model into MLX arrays from safetensors + config.json. - * Returns NULL on error. - */ -pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); -pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, - const pplx_mlx_options_t *opts); - -/* - * Load one contiguous transformer layer range for distributed execution. - * Uses the same half-open range and endpoint ownership rules as - * pplx_model_load_slice(). - */ -pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, - int layer_start, int layer_end); -pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( - const char *model_dir, int layer_start, int layer_end, - const pplx_mlx_options_t *opts); - -void pplx_mlx_free(pplx_mlx_ctx_t *ctx); - -/* - * Late-interaction MLX path. This is separate from pooled embeddings because - * late snapshots produce one projected token vector per input token for MaxSim - * scoring, not one pooled document vector. - */ -pplx_mlx_late_ctx_t *pplx_mlx_late_load(const char *model_dir); -pplx_mlx_late_ctx_t *pplx_mlx_late_load_with_options( - const char *model_dir, const pplx_mlx_options_t *opts); -void pplx_mlx_late_free(pplx_mlx_late_ctx_t *ctx); - -const pplx_config_t *pplx_mlx_late_config(const pplx_mlx_late_ctx_t *ctx); -int pplx_mlx_late_token_dim(const pplx_mlx_late_ctx_t *ctx); - -/* - * Encode token_ids into out_vectors[n_tokens, token_dim]. - * - * If normalize is non-zero, every token vector is L2-normalized on device - * before copying back to CPU memory. - */ -int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors); - -/* - * Device-resident late-interaction token vectors. - * - * The model context owns the MLX stream; the returned vector handle must not - * outlive ctx. Copying is optional: MaxSim can consume these handles directly - * and copy back only final candidate scores. - */ -pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device( - pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, - int normalize); -void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs); -int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs); -int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs); -int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs, - float *out_vectors); -pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat( - pplx_mlx_late_ctx_t *ctx, - const pplx_mlx_late_vectors_t *const *items, int count); -pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select( - pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs, - const int *token_indices, int count); -int pplx_mlx_late_maxsim_batch_device( - pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query, - const pplx_mlx_late_vectors_t *docs, const int *doc_offsets, - int docs_count, float *scores); - -/* - * Compute embedding for token_ids[0..n_tokens-1]. - * Returns malloc'd float[hidden_size] (caller frees). NULL on error. - */ -float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens); - -/* - * Compute one embedding into caller-provided out[hidden_size]. - * Returns 0 on success, -1 on error. - */ -int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding); - -/* - * Compute a true padded dense batch on MLX. - * - * out_embeddings is caller-provided [batch, hidden_size]. - * Padded tokens are masked out of attention keys and mean pooling. - * Returns 0 on success, -1 on error. - */ -int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, - int batch, float *out_embeddings); - -/* - * MLX equivalent of pplx_model_forward_slice_batch(). - * - * Hidden states cross the API boundary as packed float32 rows even though MLX - * executes a padded dense batch internally. - */ -int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states); - -/* - * Run one contextual sequence and pool selected token spans. - * Returns 0 on success, -1 on error. - */ -int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, const pplx_span_t *spans, - int n_spans, float *out_embeddings); - -/* - * Run a padded dense contextual document batch and pool every selected span. - * out_embeddings contains spans in document order. - */ -int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, - const pplx_context_input_t *inputs, int batch, - float *out_embeddings); - -/* Get the config (so main.c can read hidden_size etc.) */ -const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx); - -#endif /* PPLX_EMBED_MLX_H */ diff --git a/pplx_server.c b/pplx_server.c deleted file mode 100644 index d831062..0000000 --- a/pplx_server.c +++ /dev/null @@ -1,2431 +0,0 @@ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include "pplx_server.h" -#include "pplx_embed.h" -#include "qwen_safetensors.h" -#include "qwen_tokenizer.h" - -#ifdef USE_MLX -#include "pplx_embed_mlx.h" -#endif -#ifdef USE_CUDA -#include "pplx_embed_cuda.h" -#endif - -#include "deps/ae/ae.h" -#include "deps/ae/anet.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -#include - -#define PPLX_HTTP_IO_BUF 8192 -#define PPLX_HTTP_MAX_HEADER (64u * 1024u) -#define PPLX_HTTP_MAX_BODY (64u * 1024u * 1024u) -#define PPLX_HTTP_MAX_PATH 255u -#define PPLX_HTTP_BACKLOG 128 -#define PPLX_HTTP_SETSIZE 10240 -#define PPLX_HTTP_CLIENT_TIMEOUT_MS 30000 -#define PPLX_HTTP_SWEEP_MS 1000 - -#define PPLX_API_MAX_STANDARD_INPUTS 512 -#define PPLX_API_MAX_CONTEXT_DOCS 512 -#define PPLX_API_MAX_CONTEXT_CHUNKS 16000 -#define PPLX_API_MAX_ITEM_TOKENS 32768 -#define PPLX_API_MAX_TOTAL_TOKENS 120000 -#define PPLX_SERVER_DEFAULT_BATCH_SIZE 8 -#define PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 -#define PPLX_SERVER_BATCH_WAIT_US 1000 -#define PPLX_SERVER_MICROBATCH_MAX_JOBS 128 -#define PPLX_MLX_MEMORY_BUDGET_PERCENT 90 -#define PPLX_MLX_RESIDENT_MULTIPLIER 2 - -typedef struct { - char *ptr; - size_t len; - size_t cap; -} sbuf; - -static void die_oom(void) { - fprintf(stderr, "pplx-serve: out of memory\n"); - exit(1); -} - -static void *xmalloc(size_t n) { - void *p = malloc(n ? n : 1); - if (!p) die_oom(); - return p; -} - -static void *xcalloc(size_t n, size_t sz) { - void *p = calloc(n ? n : 1, sz ? sz : 1); - if (!p) die_oom(); - return p; -} - -static void *xrealloc(void *p, size_t n) { - p = realloc(p, n ? n : 1); - if (!p) die_oom(); - return p; -} - -static char *xstrdup(const char *s) { - size_t n = strlen(s); - char *p = xmalloc(n + 1); - memcpy(p, s, n + 1); - return p; -} - -static char *xstrndup(const char *s, size_t n) { - char *p = xmalloc(n + 1); - memcpy(p, s, n); - p[n] = '\0'; - return p; -} - -static uint64_t mstime(void) { - struct timespec ts; - if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) - return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; - struct timeval tv; - gettimeofday(&tv, NULL); - return (uint64_t)tv.tv_sec * 1000u + (uint64_t)tv.tv_usec / 1000u; -} - -static uint64_t nstime(void) { - struct timespec ts; - if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) - return (uint64_t)ts.tv_sec * 1000000000u + (uint64_t)ts.tv_nsec; - struct timeval tv; - gettimeofday(&tv, NULL); - return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u; -} - -static double ns_to_ms(uint64_t ns) { - return (double)ns / 1000000.0; -} - -static void server_log(const char *fmt, ...) { - char ts[32]; - time_t now = time(NULL); - struct tm tmv; - localtime_r(&now, &tmv); - strftime(ts, sizeof(ts), "%m%d %H:%M:%S", &tmv); - fprintf(stderr, "%s ", ts); - va_list ap; - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - fputc('\n', stderr); -} - -static void sbuf_reserve(sbuf *b, size_t add) { - if (add > SIZE_MAX - b->len - 1) die_oom(); - size_t need = b->len + add + 1; - if (need <= b->cap) return; - size_t cap = b->cap ? b->cap * 2 : 256; - while (cap < need) { - if (cap > SIZE_MAX / 2) die_oom(); - cap *= 2; - } - b->ptr = xrealloc(b->ptr, cap); - b->cap = cap; -} - -static void sbuf_append(sbuf *b, const void *p, size_t n) { - sbuf_reserve(b, n); - memcpy(b->ptr + b->len, p, n); - b->len += n; - b->ptr[b->len] = '\0'; -} - -static void sbuf_putc(sbuf *b, char c) { sbuf_append(b, &c, 1); } -static void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); } - -static void sbuf_clear(sbuf *b) { - b->len = 0; - if (b->ptr) b->ptr[0] = '\0'; -} - -static void sbuf_printf(sbuf *b, const char *fmt, ...) { - va_list ap, ap2; - va_start(ap, fmt); - va_copy(ap2, ap); - int n = vsnprintf(NULL, 0, fmt, ap); - va_end(ap); - if (n < 0) die_oom(); - sbuf_reserve(b, (size_t)n); - int n2 = vsnprintf(b->ptr + b->len, b->cap - b->len, fmt, ap2); - va_end(ap2); - if (n2 < 0 || n2 != n) die_oom(); - b->len += (size_t)n; -} - -static void sbuf_free(sbuf *b) { - free(b->ptr); - memset(b, 0, sizeof(*b)); -} - -typedef enum { - MODEL_STD_06, - MODEL_STD_4, - MODEL_CTX_06, - MODEL_CTX_4, - MODEL_UNKNOWN -} model_slot; - -typedef struct { - const char *id; - int is_contextual; - int dim; - double price_per_mtok; -} model_info; - -static const model_info k_models[] = { - {"pplx-embed-v1-0.6b", 0, 1024, 0.004}, - {"pplx-embed-v1-4b", 0, 2560, 0.030}, - {"pplx-embed-context-v1-0.6b", 1, 1024, 0.008}, - {"pplx-embed-context-v1-4b", 1, 2560, 0.050}, -}; - -static model_slot model_slot_for_id(const char *id) { - if (!id) return MODEL_UNKNOWN; - for (int i = 0; i < 4; i++) { - if (!strcmp(id, k_models[i].id)) return (model_slot)i; - } - return MODEL_UNKNOWN; -} - -typedef struct { - const model_info *info; - char *path; - qwen_tokenizer_t *tok; - qwen_tokenizer_workspace_t *tok_ws; - int context_separator_id; - pplx_model_t *cpu_model; - pplx_workspace_t *cpu_ws; -#ifdef USE_MLX - pplx_mlx_ctx_t *mlx_ctx; -#endif -#ifdef USE_CUDA - pplx_cuda_ctx_t *cuda_ctx; -#endif -} loaded_model; - -typedef struct client client; -typedef struct job job; - -typedef struct { - aeEventLoop *loop; - int listen_fd; - int completion_pipe[2]; - int signal_pipe[2]; - const char *host; - int port; - int batch_size; - int max_batch_tokens; - int batch_wait_us; - int enable_cors; - char *api_key; - loaded_model models[4]; - int use_mlx; - int use_cuda; - int mlx_quantize_bits; - int mlx_quantize_group_size; - - pthread_t worker; - pthread_mutex_t mu; - pthread_cond_t cv; - int stopping; - int worker_ready; - int worker_init_rc; - job *job_head; - job *job_tail; - job *done_head; - job *done_tail; - - client *clients; - int n_clients; -} http_server; - -typedef struct { - char method[8]; - char path[PPLX_HTTP_MAX_PATH + 1]; - char version[16]; - char *body; - size_t body_len; - char *auth; -} http_request; - -struct client { - http_server *srv; - int fd; - int refcount; - int cancelled; - int linked; - uint64_t last_active_ms; - sbuf in; - sbuf out; - size_t sent; - int header_done; - size_t header_len; - size_t content_length; - int close_after_write; - http_request req; - client *prev; - client *next; -}; - -struct job { - http_server *srv; - client *c; - char method[8]; - char path[PPLX_HTTP_MAX_PATH + 1]; - char *body; - size_t body_len; - char *auth; - int status; - char *content_type; - char *extra_headers; - char *response; - size_t response_len; - uint64_t created_ns; - uint64_t started_ns; - uint64_t parse_ns; - uint64_t tokenize_ns; - uint64_t infer_ns; - uint64_t encode_ns; - job *next; -}; - -static int queue_write(client *c); -static void dispatch_request(client *c); -static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask); - -static volatile sig_atomic_t g_stop_requested = 0; -static volatile sig_atomic_t g_signal_wfd = -1; - -static void stop_signal_handler(int sig) { - (void)sig; - if (g_stop_requested) _exit(130); - int saved = errno; - g_stop_requested = 1; - int fd = (int)g_signal_wfd; - if (fd >= 0) { - const unsigned char byte = 1; - (void)write(fd, &byte, 1); - } - errno = saved; -} - -static int set_nonblock(int fd) { - int flags = fcntl(fd, F_GETFL, 0); - if (flags < 0) return -1; - return fcntl(fd, F_SETFL, flags | O_NONBLOCK); -} - -static void http_request_free(http_request *r) { - free(r->body); - free(r->auth); - memset(r, 0, sizeof(*r)); -} - -static void client_free(client *c) { - if (!c) return; - http_request_free(&c->req); - sbuf_free(&c->in); - sbuf_free(&c->out); - free(c); -} - -static void client_incref(client *c) { - pthread_mutex_lock(&c->srv->mu); - c->refcount++; - pthread_mutex_unlock(&c->srv->mu); -} - -static void client_decref(client *c) { - int free_now = 0; - pthread_mutex_lock(&c->srv->mu); - if (--c->refcount == 0) free_now = 1; - pthread_mutex_unlock(&c->srv->mu); - if (free_now) client_free(c); -} - -static void client_decref_n(client *c, int n) { - int free_now = 0; - if (n <= 0) return; - pthread_mutex_lock(&c->srv->mu); - c->refcount -= n; - if (c->refcount <= 0) free_now = 1; - pthread_mutex_unlock(&c->srv->mu); - if (free_now) client_free(c); -} - -static void client_unlink(client *c) { - http_server *s = c->srv; - if (!c->linked) return; - if (c->prev) c->prev->next = c->next; - else s->clients = c->next; - if (c->next) c->next->prev = c->prev; - c->prev = c->next = NULL; - c->linked = 0; - if (s->n_clients > 0) s->n_clients--; -} - -static int close_client_unlink(client *c) { - if (!c || c->cancelled) return 0; - c->cancelled = 1; - if (c->fd >= 0) { - aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); - aeDeleteFileEvent(c->srv->loop, c->fd, AE_WRITABLE); - close(c->fd); - c->fd = -1; - } - client_unlink(c); - return 1; -} - -static void close_client(client *c) { - if (!close_client_unlink(c)) return; - client_decref(c); -} - -static void append_http_response_ex(client *c, int status, const char *ctype, - const char *extra_headers, - const char *body, size_t body_len) { - const char *reason = "OK"; - if (status == 204) reason = "No Content"; - else if (status == 400) reason = "Bad Request"; - else if (status == 401) reason = "Unauthorized"; - else if (status == 404) reason = "Not Found"; - else if (status == 405) reason = "Method Not Allowed"; - else if (status == 422) reason = "Unprocessable Entity"; - else if (status == 503) reason = "Service Unavailable"; - else if (status >= 500) reason = "Internal Server Error"; - - sbuf_printf(&c->out, - "HTTP/1.1 %d %s\r\n" - "Content-Length: %zu\r\n" - "Connection: %s\r\n", - status, reason, body_len, - c->close_after_write ? "close" : "keep-alive"); - if (ctype) - sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype); - if (c->srv->enable_cors) { - sbuf_puts(&c->out, "Access-Control-Allow-Origin: *\r\n"); - sbuf_puts(&c->out, "Access-Control-Allow-Headers: authorization, content-type\r\n"); - sbuf_puts(&c->out, "Access-Control-Allow-Methods: POST, OPTIONS\r\n"); - } - if (extra_headers) - sbuf_puts(&c->out, extra_headers); - sbuf_puts(&c->out, "\r\n"); - if (body_len) sbuf_append(&c->out, body, body_len); -} - -static void append_http_response(client *c, int status, const char *ctype, - const char *body, size_t body_len) { - append_http_response_ex(c, status, ctype, NULL, body, body_len); -} - -static void append_json_error(sbuf *b, const char *message, const char *type) { - sbuf_puts(b, "{\"error\":{\"message\":\""); - for (const char *p = message ? message : ""; *p; p++) { - if (*p == '"' || *p == '\\') { - sbuf_putc(b, '\\'); - sbuf_putc(b, *p); - } else { - sbuf_putc(b, *p); - } - } - sbuf_puts(b, "\",\"type\":\""); - sbuf_puts(b, type ? type : "server_error"); - sbuf_puts(b, "\"}}"); -} - -static char *json_error_body(const char *message, const char *type, size_t *len) { - sbuf b = {0}; - append_json_error(&b, message, type); - *len = b.len; - return b.ptr; -} - -static void job_set_error(job *j, int status, const char *message, const char *type) { - free(j->response); - j->status = status; - j->content_type = "application/json"; - j->response = json_error_body(message, type, &j->response_len); -} - -static bool auth_ok(http_server *s, const char *auth) { - if (!s->api_key || !s->api_key[0]) return true; - if (!auth) return false; - const char prefix[] = "Bearer "; - return !strncmp(auth, prefix, sizeof(prefix) - 1) && - !strcmp(auth + sizeof(prefix) - 1, s->api_key); -} - -static ssize_t find_header_end(const char *p, size_t n) { - for (size_t i = 3; i < n; i++) { - if (p[i - 3] == '\r' && p[i - 2] == '\n' && - p[i - 1] == '\r' && p[i] == '\n') - return (ssize_t)(i + 1); - } - for (size_t i = 1; i < n; i++) { - if (p[i - 1] == '\n' && p[i] == '\n') return (ssize_t)(i + 1); - } - return -1; -} - -static char *header_value_dup(const char *h, size_t n, const char *name) { - size_t name_len = strlen(name); - const char *p = h, *end = h + n; - while (p < end) { - const char *line = p; - while (p < end && *p != '\n') p++; - size_t len = (size_t)(p - line); - if (len && line[len - 1] == '\r') len--; - if (len > name_len && !strncasecmp(line, name, name_len) && - line[name_len] == ':') { - const char *v = line + name_len + 1; - while (v < line + len && isspace((unsigned char)*v)) v++; - const char *e = line + len; - while (e > v && isspace((unsigned char)e[-1])) e--; - return xstrndup(v, (size_t)(e - v)); - } - if (p < end) p++; - } - return NULL; -} - -static int header_has_token(const char *value, const char *token) { - size_t token_len = strlen(token); - const char *p = value; - while (p && *p) { - while (*p == ',' || isspace((unsigned char)*p)) p++; - const char *start = p; - while (*p && *p != ',') p++; - const char *end = p; - while (end > start && isspace((unsigned char)end[-1])) end--; - if ((size_t)(end - start) == token_len && - !strncasecmp(start, token, token_len)) - return 1; - } - return 0; -} - -static int parse_request_headers(client *c) { - char line[512]; - size_t i = 0; - while (i < c->header_len && c->in.ptr[i] != '\n' && i + 1 < sizeof(line)) { - line[i] = c->in.ptr[i]; - i++; - } - line[i] = '\0'; - int fields = sscanf(line, "%7s %255s %15s", - c->req.method, c->req.path, c->req.version); - if (fields < 2) - return -1; - if (fields < 3) - snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0"); - char *q = strchr(c->req.path, '?'); - if (q) *q = '\0'; - - c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0; - char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection"); - if (conn) { - if (header_has_token(conn, "close")) - c->close_after_write = 1; - else if (header_has_token(conn, "keep-alive")) - c->close_after_write = 0; - free(conn); - } - - char *cl = header_value_dup(c->in.ptr, c->header_len, "Content-Length"); - if (cl) { - char *end = NULL; - errno = 0; - unsigned long long v = strtoull(cl, &end, 10); - if (errno || !end || *end || v > PPLX_HTTP_MAX_BODY) { - free(cl); - return -1; - } - c->content_length = (size_t)v; - free(cl); - } - c->req.auth = header_value_dup(c->in.ptr, c->header_len, "Authorization"); - return 0; -} - -static int complete_request(client *c) { - if (!c->header_done) { - ssize_t hend = find_header_end(c->in.ptr, c->in.len); - if (hend < 0) { - if (c->in.len > PPLX_HTTP_MAX_HEADER) return -1; - return 0; - } - c->header_done = 1; - c->header_len = (size_t)hend; - if (parse_request_headers(c) != 0) return -1; - } - if (c->content_length > PPLX_HTTP_MAX_BODY) return -1; - if (c->in.len < c->header_len + c->content_length) return 0; - c->req.body_len = c->content_length; - c->req.body = xmalloc(c->content_length + 1); - memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length); - c->req.body[c->content_length] = '\0'; - return 1; -} - -static void send_bad_http_request(client *c) { - size_t len; - char *body = json_error_body("bad HTTP request", "invalid_request_error", - &len); - c->close_after_write = 1; - append_http_response(c, 400, "application/json", body, len); - free(body); - aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); - if (queue_write(c) != AE_OK) close_client(c); -} - -static void reset_client_for_next_request(client *c) { - size_t consumed = c->header_len + c->content_length; - if (consumed > c->in.len) consumed = c->in.len; - size_t remain = c->in.len - consumed; - if (remain) - memmove(c->in.ptr, c->in.ptr + consumed, remain); - c->in.len = remain; - if (c->in.ptr) c->in.ptr[remain] = '\0'; - - http_request_free(&c->req); - sbuf_clear(&c->out); - c->sent = 0; - c->header_done = 0; - c->header_len = 0; - c->content_length = 0; - c->close_after_write = 0; -} - -static void arm_next_request(client *c) { - reset_client_for_next_request(c); - if (c->cancelled || c->fd < 0) return; - - if (c->in.len) { - int done = complete_request(c); - if (done < 0) { - send_bad_http_request(c); - return; - } - if (done > 0) { - dispatch_request(c); - return; - } - } - - if (aeCreateFileEvent(c->srv->loop, c->fd, AE_READABLE, read_cb, c) != AE_OK) - close_client(c); -} - -static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; - client *c = clientData; - while (c->sent < c->out.len) { - ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent); - if (n > 0) { - c->sent += (size_t)n; - c->last_active_ms = mstime(); - continue; - } - if (n < 0 && errno == EINTR) continue; - if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) return; - close_client(c); - return; - } - aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE); - if (c->close_after_write) close_client(c); - else arm_next_request(c); -} - -static int queue_write(client *c) { - if (!c || c->cancelled || c->fd < 0) return -1; - return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c); -} - -static void enqueue_done(job *j) { - http_server *s = j->srv; - pthread_mutex_lock(&s->mu); - if (s->done_tail) s->done_tail->next = j; - else s->done_head = j; - s->done_tail = j; - pthread_mutex_unlock(&s->mu); - const unsigned char byte = 1; - (void)write(s->completion_pipe[1], &byte, 1); -} - -static job *dequeue_job(http_server *s) { - pthread_mutex_lock(&s->mu); - while (!s->job_head && !s->stopping) - pthread_cond_wait(&s->cv, &s->mu); - job *j = s->job_head; - if (j) { - s->job_head = j->next; - if (!s->job_head) s->job_tail = NULL; - j->next = NULL; - } - pthread_mutex_unlock(&s->mu); - return j; -} - -static int drain_ready_jobs(http_server *s, job **jobs, int max_jobs) { - int n = 0; - pthread_mutex_lock(&s->mu); - while (n < max_jobs && s->job_head) { - job *j = s->job_head; - s->job_head = j->next; - if (!s->job_head) s->job_tail = NULL; - j->next = NULL; - jobs[n++] = j; - } - pthread_mutex_unlock(&s->mu); - return n; -} - -static void enqueue_job(job *j) { - http_server *s = j->srv; - pthread_mutex_lock(&s->mu); - if (s->job_tail) s->job_tail->next = j; - else s->job_head = j; - s->job_tail = j; - pthread_cond_signal(&s->cv); - pthread_mutex_unlock(&s->mu); -} - -static job *pop_done(http_server *s) { - pthread_mutex_lock(&s->mu); - job *j = s->done_head; - if (j) { - s->done_head = j->next; - if (!s->done_head) s->done_tail = NULL; - j->next = NULL; - } - pthread_mutex_unlock(&s->mu); - return j; -} - -static void job_free(job *j) { - if (!j) return; - free(j->body); - free(j->auth); - free(j->extra_headers); - free(j->response); - free(j); -} - -static void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; - http_server *s = clientData; - unsigned char tmp[128]; - while (read(fd, tmp, sizeof(tmp)) > 0) {} - - job *j; - while ((j = pop_done(s)) != NULL) { - client *c = j->c; - int refs_to_drop = 1; /* job reference */ - if (!c->cancelled) { - append_http_response_ex(c, j->status, j->content_type, - j->extra_headers, - j->response ? j->response : "", - j->response ? j->response_len : 0); - if (queue_write(c) != AE_OK) - refs_to_drop += close_client_unlink(c); - } - client_decref_n(c, refs_to_drop); - job_free(j); - } -} - -static bool route_is_embedding(const char *method, const char *path) { - return !strcmp(method, "POST") && - (!strcmp(path, "/v1/embeddings") || - !strcmp(path, "/v1/contextualizedembeddings")); -} - -static void dispatch_request(client *c) { - aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); - - if (!strcmp(c->req.method, "OPTIONS")) { - append_http_response(c, 204, NULL, "", 0); - if (queue_write(c) != AE_OK) close_client(c); - return; - } - - if (!route_is_embedding(c->req.method, c->req.path)) { - size_t len; - char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); - append_http_response(c, 404, "application/json", body, len); - free(body); - if (queue_write(c) != AE_OK) close_client(c); - return; - } - - if (!auth_ok(c->srv, c->req.auth)) { - size_t len; - char *body = json_error_body("missing or invalid bearer token", - "authentication_error", &len); - append_http_response(c, 401, "application/json", body, len); - free(body); - if (queue_write(c) != AE_OK) close_client(c); - return; - } - - job *j = xcalloc(1, sizeof(*j)); - j->srv = c->srv; - j->c = c; - snprintf(j->method, sizeof(j->method), "%s", c->req.method); - snprintf(j->path, sizeof(j->path), "%s", c->req.path); - j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); - j->body_len = c->req.body_len; - j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; - j->created_ns = nstime(); - client_incref(c); - enqueue_job(j); -} - -static void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; - client *c = clientData; - char tmp[PPLX_HTTP_IO_BUF]; - for (;;) { - ssize_t n = read(fd, tmp, sizeof(tmp)); - if (n > 0) { - sbuf_append(&c->in, tmp, (size_t)n); - c->last_active_ms = mstime(); - int done = complete_request(c); - if (done < 0) { - send_bad_http_request(c); - return; - } - if (done > 0) { - dispatch_request(c); - return; - } - continue; - } - if (n == 0) { - close_client(c); - return; - } - if (errno == EINTR) continue; - if (errno == EAGAIN || errno == EWOULDBLOCK) return; - close_client(c); - return; - } -} - -static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; (void)mask; - http_server *s = clientData; - char cip[64]; - int cport = 0; - for (;;) { - int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); - if (cfd == ANET_ERR) { - if (errno == EAGAIN || errno == EWOULDBLOCK) return; - if (errno == EINTR) continue; - return; - } - set_nonblock(cfd); - client *c = xcalloc(1, sizeof(*c)); - c->srv = s; - c->fd = cfd; - c->refcount = 1; - c->last_active_ms = mstime(); - c->linked = 1; - c->next = s->clients; - if (s->clients) s->clients->prev = c; - s->clients = c; - s->n_clients++; - if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) - close_client(c); - } -} - -static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { - (void)loop; (void)id; - http_server *s = clientData; - uint64_t now = mstime(); - client *c = s->clients; - while (c) { - client *next = c->next; - if (!c->cancelled && - now - c->last_active_ms > PPLX_HTTP_CLIENT_TIMEOUT_MS) - close_client(c); - c = next; - } - return PPLX_HTTP_SWEEP_MS; -} - -static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)fd; (void)mask; - http_server *s = clientData; - unsigned char tmp[128]; - while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {} - aeStop(loop); -} - -static bool ve_add(cJSON *detail, const char *loc_json, - const char *msg, const char *type) { - cJSON *obj = cJSON_CreateObject(); - cJSON *loc = cJSON_Parse(loc_json); - if (!obj || !loc) { - cJSON_Delete(obj); - cJSON_Delete(loc); - return false; - } - cJSON_AddItemToObject(obj, "loc", loc); - cJSON_AddStringToObject(obj, "msg", msg ? msg : ""); - cJSON_AddStringToObject(obj, "type", type ? type : ""); - cJSON_AddItemToArray(detail, obj); - return true; -} - -static void job_set_422(job *j, cJSON *detail) { - cJSON *root = cJSON_CreateObject(); - cJSON_AddItemToObject(root, "detail", cJSON_Duplicate(detail, 1)); - char *s = cJSON_PrintUnformatted(root); - cJSON_Delete(root); - if (!s) { - job_set_error(j, 500, "failed to render validation error", "server_error"); - return; - } - j->status = 422; - j->content_type = "application/json"; - j->response = xstrdup(s); - j->response_len = strlen(j->response); - cJSON_free(s); -} - -static void job_set_timing_header(job *j) { - if (!j || j->extra_headers || !j->created_ns) return; - uint64_t done_ns = nstime(); - uint64_t queue_ns = j->started_ns > j->created_ns - ? j->started_ns - j->created_ns : 0; - uint64_t worker_ns = j->started_ns && done_ns > j->started_ns - ? done_ns - j->started_ns : 0; - - char buf[384]; - int n = snprintf(buf, sizeof(buf), - "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, " - "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, " - "worker;dur=%.3f\r\n", - ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), - ns_to_ms(j->tokenize_ns), ns_to_ms(j->infer_ns), - ns_to_ms(j->encode_ns), ns_to_ms(worker_ns)); - if (n > 0 && (size_t)n < sizeof(buf)) - j->extra_headers = xstrdup(buf); -} - -static cJSON *parse_json_body(job *j, cJSON *detail) { - if (!j->body || j->body_len == 0) { - ve_add(detail, "[\"body\"]", "body required", "missing"); - return NULL; - } - const char *end = NULL; - cJSON *root = cJSON_ParseWithLengthOpts(j->body, j->body_len, &end, false); - if (!root) { - ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); - return NULL; - } - const char *body_end = j->body + j->body_len; - while (end && end < body_end && isspace((unsigned char)*end)) end++; - if (!end || end != body_end) { - ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid"); - cJSON_Delete(root); - return NULL; - } - if (!cJSON_IsObject(root)) { - ve_add(detail, "[\"body\"]", "body must be a JSON object", "type_error.object"); - cJSON_Delete(root); - return NULL; - } - return root; -} - -static bool cjson_is_integer(cJSON *item) { - if (!cJSON_IsNumber(item)) return false; - double d = item->valuedouble; - int i = item->valueint; - return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i; -} - -static const char *encoding_from_root(cJSON *root, cJSON *detail) { - cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); - if (!encoding) return "base64_int8"; - if (!cJSON_IsString(encoding) || !encoding->valuestring) { - ve_add(detail, "[\"body\",\"encoding_format\"]", - "encoding_format must be a string", "type_error.string"); - return "base64_int8"; - } - if (strcmp(encoding->valuestring, "base64_int8") && - strcmp(encoding->valuestring, "base64_binary") && - strcmp(encoding->valuestring, "float")) { - ve_add(detail, "[\"body\",\"encoding_format\"]", - "value is not a valid enum member; permitted: 'base64_int8', 'base64_binary', 'float'", - "enum"); - return "base64_int8"; - } - return encoding->valuestring; -} - -static int dimensions_from_root(cJSON *root, cJSON *detail, int max_dim, - const char *encoding) { - int dims = max_dim; - cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); - if (dimensions) { - if (!cjson_is_integer(dimensions)) { - ve_add(detail, "[\"body\",\"dimensions\"]", "dimensions must be an integer", - "type_error.integer"); - } else { - dims = dimensions->valueint; - if (dims < 128 || dims > max_dim) { - char msg[96]; - snprintf(msg, sizeof(msg), "dimensions must be between 128 and %d", max_dim); - ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); - } - } - } - if (!strcmp(encoding, "base64_binary") && dims % 8 != 0) - ve_add(detail, "[\"body\",\"dimensions\"]", - "dimensions must be divisible by 8 for base64_binary", - "value_error.divisible"); - return dims; -} - -static char *base64_encode(const unsigned char *src, size_t n) { - static const char tbl[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - size_t out_len = ((n + 2) / 3) * 4; - char *out = xmalloc(out_len + 1); - size_t o = 0; - for (size_t i = 0; i < n; i += 3) { - unsigned v = (unsigned)src[i] << 16; - int remain = (int)(n - i); - if (remain > 1) v |= (unsigned)src[i + 1] << 8; - if (remain > 2) v |= (unsigned)src[i + 2]; - out[o++] = tbl[(v >> 18) & 63]; - out[o++] = tbl[(v >> 12) & 63]; - out[o++] = remain > 1 ? tbl[(v >> 6) & 63] : '='; - out[o++] = remain > 2 ? tbl[v & 63] : '='; - } - out[o] = '\0'; - return out; -} - -static signed char quantize_int8_tanh(float x) { - int v = (int)lrintf(tanhf(x) * 127.0f); - if (v > 127) v = 127; - if (v < -128) v = -128; - return (signed char)v; -} - -static char *encode_embedding(const float *emb, int dims, const char *encoding) { - if (!strcmp(encoding, "base64_binary")) { - size_t bytes = (size_t)dims / 8; - unsigned char *bits = xcalloc(bytes, 1); - for (int i = 0; i < dims; i++) { - if (emb[i] >= 0.0f) - bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7)); - } - char *out = base64_encode(bits, bytes); - free(bits); - return out; - } - - signed char *q = xmalloc((size_t)dims); - for (int i = 0; i < dims; i++) - q[i] = quantize_int8_tanh(emb[i]); - char *out = base64_encode((const unsigned char *)q, (size_t)dims); - free(q); - return out; -} - -typedef struct { - int *ids; - int n_tokens; -} token_buf; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - int dims; - const char *encoding; - int n_inputs; - token_buf *tokens; - pplx_input_t *inputs; - int total_tokens; - int ready; -} embedding_request; - -typedef struct { - int *ids; - int n_tokens; - pplx_span_t *spans; - int n_spans; -} contextual_doc; - -typedef struct { - job *j; - cJSON *root; - loaded_model *model; - int dims; - const char *encoding; - contextual_doc *docs; - int n_docs; - int total_chunks; - int total_tokens; - int ready; -} contextual_request; - -static void free_token_bufs(token_buf *t, int n) { - if (!t) return; - for (int i = 0; i < n; i++) free(t[i].ids); -} - -static void embedding_request_free(embedding_request *r) { - if (!r) return; - free_token_bufs(r->tokens, r->n_inputs); - free(r->tokens); - free(r->inputs); - if (r->root) cJSON_Delete(r->root); - memset(r, 0, sizeof(*r)); -} - -static void contextual_request_free(contextual_request *r) { - if (!r) return; - for (int i = 0; i < r->n_docs; i++) { - free(r->docs[i].ids); - free(r->docs[i].spans); - } - free(r->docs); - if (r->root) cJSON_Delete(r->root); - memset(r, 0, sizeof(*r)); -} - -static int tokenize_one(loaded_model *m, job *j, const char *text, - token_buf *out) { - memset(out, 0, sizeof(*out)); - uint64_t t0 = nstime(); - out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, - text, &out->n_tokens); - if (j) j->tokenize_ns += nstime() - t0; - if (!out->ids || out->n_tokens <= 0) { - free(out->ids); - memset(out, 0, sizeof(*out)); - return -1; - } - return 0; -} - -static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs, - int batch, float *out) { -#ifdef USE_MLX - if (m->mlx_ctx) - return pplx_mlx_embed_batch(m->mlx_ctx, inputs, batch, out); -#endif -#ifdef USE_CUDA - if (m->cuda_ctx) - return pplx_cuda_embed_batch(m->cuda_ctx, inputs, batch, out); -#endif - return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); -} - -static int model_embed_spans_batch(loaded_model *m, - const pplx_context_input_t *inputs, - int batch, float *out) { -#ifdef USE_MLX - if (m->mlx_ctx) - return pplx_mlx_embed_spans_batch(m->mlx_ctx, inputs, batch, out); -#endif -#ifdef USE_CUDA - if (m->cuda_ctx) - return pplx_cuda_embed_spans_batch(m->cuda_ctx, inputs, batch, out); -#endif - return pplx_model_embed_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, - out); -} - -static int model_uses_dense_batches(const loaded_model *m) { -#ifdef USE_MLX - return m->mlx_ctx != NULL; -#else - (void)m; - return 0; -#endif -} - -/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX - * pads each dense row to the longest input in the chunk. */ -static int inference_batch_accepts_input(const loaded_model *m, int batch, - int packed_tokens, int next_tokens, - int max_batch_tokens) { - if (batch == 0) return 1; - if (model_uses_dense_batches(m)) - return next_tokens <= max_batch_tokens / (batch + 1); - return next_tokens <= max_batch_tokens - packed_tokens; -} - -static void append_embedding_object(sbuf *b, int index, const char *embedding) { - sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index); - sbuf_puts(b, embedding); - sbuf_puts(b, "\"}"); -} - -static void append_float_embedding_object(sbuf *b, int index, - const float *emb, int dims) { - sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); - for (int i = 0; i < dims; i++) { - if (i) sbuf_putc(b, ','); - float v = (float)quantize_int8_tanh(emb[i]) / 128.0f; - sbuf_printf(b, "%.9g", (double)v); - } - sbuf_puts(b, "]}"); -} - -static void set_response_from_buf(job *j, sbuf *b) { - j->status = 200; - j->content_type = "application/json"; - j->response = b->ptr; - j->response_len = b->len; - memset(b, 0, sizeof(*b)); -} - -static void render_embedding_response(embedding_request *r, const float *embs) { - char **encoded = NULL; - if (strcmp(r->encoding, "float")) { - encoded = xcalloc((size_t)r->n_inputs, sizeof(*encoded)); - for (int i = 0; i < r->n_inputs; i++) - encoded[i] = encode_embedding(embs + (size_t)i * r->model->info->dim, - r->dims, r->encoding); - } - - sbuf b = {0}; - sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); - for (int i = 0; i < r->n_inputs; i++) { - if (i) sbuf_putc(&b, ','); - if (encoded) - append_embedding_object(&b, i, encoded[i]); - else - append_float_embedding_object(&b, i, - embs + (size_t)i * r->model->info->dim, r->dims); - } - double cost = ((double)r->total_tokens / 1000000.0) * - r->model->info->price_per_mtok; - sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d," - "\"total_tokens\":%d,\"cost\":{\"input_cost\":%.10g," - "\"total_cost\":%.10g,\"currency\":\"USD\"}}}", - r->model->info->id, r->total_tokens, r->total_tokens, cost, cost); - set_response_from_buf(r->j, &b); - - for (int i = 0; encoded && i < r->n_inputs; i++) free(encoded[i]); - free(encoded); -} - -static int embedding_request_compatible(const embedding_request *a, - const embedding_request *b) { - return a->ready && b->ready && - a->model == b->model && - a->dims == b->dims && - !strcmp(a->encoding, b->encoding); -} - -typedef struct { - pplx_input_t input; - int output_index; -} embedding_batch_item; - -static int embedding_batch_item_cmp(const void *a, const void *b) { - const embedding_batch_item *ia = a; - const embedding_batch_item *ib = b; - if (ia->input.n_tokens < ib->input.n_tokens) return -1; - if (ia->input.n_tokens > ib->input.n_tokens) return 1; - return ia->output_index - ib->output_index; -} - -static int embedding_request_fits_group(const embedding_request *r, - int group_inputs, int group_tokens, - int max_batch, int max_batch_tokens) { - if (group_inputs == 0) return 1; - return r->n_inputs <= max_batch - group_inputs && - r->total_tokens <= max_batch_tokens - group_tokens; -} - -static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { - if (n_reqs <= 0) return; - loaded_model *m = reqs[0]->model; - int dim = m->info->dim; - int total_inputs = 0; - for (int i = 0; i < n_reqs; i++) - total_inputs += reqs[i]->n_inputs; - - embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items)); - float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float)); - - int pos = 0; - for (int i = 0; i < n_reqs; i++) { - for (int k = 0; k < reqs[i]->n_inputs; k++) { - items[pos].input = reqs[i]->inputs[k]; - items[pos].output_index = pos; - pos++; - } - } - qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); - - int max_batch = reqs[0]->j->srv->batch_size > 0 - ? reqs[0]->j->srv->batch_size : total_inputs; - if (max_batch > total_inputs) max_batch = total_inputs; - int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - pplx_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); - float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); - - int failed = 0; - uint64_t infer_ns = 0; - for (int start = 0; start < total_inputs;) { - int cur = 0; - int tokens = 0; - while (start + cur < total_inputs && cur < max_batch) { - int n_tokens = items[start + cur].input.n_tokens; - if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, - max_batch_tokens)) - break; - inputs[cur] = items[start + cur].input; - tokens += n_tokens; - cur++; - } - uint64_t t0 = nstime(); - int embed_rc = model_embed_batch(m, inputs, cur, batch_embs); - infer_ns += nstime() - t0; - if (embed_rc != 0) { - failed = 1; - break; - } - for (int i = 0; i < cur; i++) - memcpy(embs + (size_t)items[start + i].output_index * dim, - batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float)); - start += cur; - } - - pos = 0; - for (int i = 0; i < n_reqs; i++) { - reqs[i]->j->infer_ns += infer_ns; - if (failed) { - job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); - } else { - uint64_t t0 = nstime(); - render_embedding_response(reqs[i], embs + (size_t)pos * dim); - reqs[i]->j->encode_ns += nstime() - t0; - } - pos += reqs[i]->n_inputs; - } - - free(batch_embs); - free(inputs); - free(embs); - free(items); -} - -static void render_contextual_response(contextual_request *r, - const float *embs) { - sbuf b = {0}; - sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); - int span_offset = 0; - for (int di = 0; di < r->n_docs; di++) { - if (di) sbuf_putc(&b, ','); - sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); - for (int ci = 0; ci < r->docs[di].n_spans; ci++) { - if (ci) sbuf_putc(&b, ','); - const float *emb = embs + (size_t)span_offset * r->model->info->dim; - if (!strcmp(r->encoding, "float")) { - append_float_embedding_object(&b, ci, - emb, r->dims); - } else { - char *encoded = encode_embedding(emb, r->dims, r->encoding); - append_embedding_object(&b, ci, encoded); - free(encoded); - } - span_offset++; - } - sbuf_puts(&b, "]}"); - } - sbuf_printf(&b, - "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", - r->model->info->id, r->total_tokens, r->total_tokens); - set_response_from_buf(r->j, &b); -} - -static int contextual_request_compatible(const contextual_request *a, - const contextual_request *b) { - return a->ready && b->ready && - a->model == b->model && - a->dims == b->dims && - !strcmp(a->encoding, b->encoding); -} - -static int contextual_request_fits_group(const contextual_request *r, - int group_docs, int group_tokens, - int max_batch, int max_batch_tokens) { - if (group_docs == 0) return 1; - return r->n_docs <= max_batch - group_docs && - r->total_tokens <= max_batch_tokens - group_tokens; -} - -typedef struct { - pplx_context_input_t input; - int output_span_index; - int order; -} contextual_batch_item; - -static int contextual_batch_item_cmp(const void *a, const void *b) { - const contextual_batch_item *ia = a; - const contextual_batch_item *ib = b; - if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1; - if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1; - return ia->order - ib->order; -} - -static void execute_contextual_request_list(contextual_request **reqs, - int n_reqs) { - if (n_reqs <= 0) return; - loaded_model *m = reqs[0]->model; - int dim = m->info->dim; - int total_docs = 0; - int total_chunks = 0; - for (int i = 0; i < n_reqs; i++) { - if (total_docs > INT_MAX - reqs[i]->n_docs || - total_chunks > INT_MAX - reqs[i]->total_chunks) { - for (int k = 0; k < n_reqs; k++) - job_set_error(reqs[k]->j, 500, "contextual batch is too large", - "server_error"); - return; - } - total_docs += reqs[i]->n_docs; - total_chunks += reqs[i]->total_chunks; - } - if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { - for (int i = 0; i < n_reqs; i++) - job_set_error(reqs[i]->j, 500, "contextual batch is too large", - "server_error"); - return; - } - - contextual_batch_item *items = - xmalloc((size_t)total_docs * sizeof(*items)); - float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); - int pos = 0; - int span_offset = 0; - for (int i = 0; i < n_reqs; i++) { - for (int d = 0; d < reqs[i]->n_docs; d++) { - contextual_doc *doc = &reqs[i]->docs[d]; - items[pos].input.input.ids = doc->ids; - items[pos].input.input.n_tokens = doc->n_tokens; - items[pos].input.spans = doc->spans; - items[pos].input.n_spans = doc->n_spans; - items[pos].output_span_index = span_offset; - items[pos].order = pos; - span_offset += doc->n_spans; - pos++; - } - } - qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); - - int max_batch = reqs[0]->j->srv->batch_size > 0 - ? reqs[0]->j->srv->batch_size : total_docs; - if (max_batch > total_docs) max_batch = total_docs; - int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - pplx_context_input_t *inputs = - xmalloc((size_t)max_batch * sizeof(*inputs)); - float *batch_embs = NULL; - size_t batch_emb_cap = 0; - - int failed = 0; - uint64_t infer_ns = 0; - for (int start = 0; start < total_docs;) { - int cur = 0; - int tokens = 0; - int chunks = 0; - while (start + cur < total_docs && cur < max_batch) { - const pplx_context_input_t *input = &items[start + cur].input; - if (!inference_batch_accepts_input(m, cur, tokens, - input->input.n_tokens, - max_batch_tokens)) - break; - inputs[cur] = *input; - tokens += input->input.n_tokens; - chunks += input->n_spans; - cur++; - } - size_t needed = (size_t)chunks * dim; - if (needed > batch_emb_cap) { - batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs)); - batch_emb_cap = needed; - } - uint64_t t0 = nstime(); - int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs); - infer_ns += nstime() - t0; - if (embed_rc != 0) { - failed = 1; - break; - } - int batch_span = 0; - for (int i = 0; i < cur; i++) { - int n_spans = inputs[i].n_spans; - memcpy(embs + (size_t)items[start + i].output_span_index * dim, - batch_embs + (size_t)batch_span * dim, - (size_t)n_spans * dim * sizeof(*embs)); - batch_span += n_spans; - } - start += cur; - } - - span_offset = 0; - for (int i = 0; i < n_reqs; i++) { - reqs[i]->j->infer_ns += infer_ns; - if (failed) { - job_set_error(reqs[i]->j, 500, "contextual embedding failed", - "server_error"); - } else { - uint64_t t0 = nstime(); - render_contextual_response(reqs[i], - embs + (size_t)span_offset * dim); - reqs[i]->j->encode_ns += nstime() - t0; - } - span_offset += reqs[i]->total_chunks; - } - - free(batch_embs); - free(inputs); - free(embs); - free(items); -} - -static int validate_common(cJSON *root, cJSON *detail, int expect_context, - loaded_model **out_model, int *out_dims, - const char **out_encoding, http_server *s) { - cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); - const char *model_id = NULL; - if (!model_item) { - ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); - } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { - ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); - } else { - model_id = model_item->valuestring; - model_slot slot = model_slot_for_id(model_id); - if (slot == MODEL_UNKNOWN || k_models[slot].is_contextual != expect_context) { - ve_add(detail, "[\"body\",\"model\"]", - "value is not a valid enum member for this endpoint", "enum"); - } else { - loaded_model *m = &s->models[slot]; - if (!m->info) { - *out_model = NULL; - } else { - *out_model = m; - } - } - } - const char *enc = encoding_from_root(root, detail); - int max_dim = model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN - ? k_models[model_slot_for_id(model_id)].dim : 2560; - int dims = dimensions_from_root(root, detail, max_dim, enc); - *out_encoding = enc; - *out_dims = dims; - return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && - !s->models[model_slot_for_id(model_id)].info ? 503 : 0; -} - -static void prepare_embedding_request(job *j, cJSON *root, http_server *s, - embedding_request *out) { - memset(out, 0, sizeof(*out)); - out->j = j; - out->root = root; - - cJSON *detail = cJSON_CreateArray(); - loaded_model *m = NULL; - int dims = 0; - const char *encoding = NULL; - int unloaded = validate_common(root, detail, 0, &m, &dims, &encoding, s); - out->model = m; - out->dims = dims; - out->encoding = encoding; - - cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); - int n_inputs = 0; - if (!input) { - ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); - } else if (cJSON_IsString(input)) { - if (!input->valuestring || input->valuestring[0] == '\0') - ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", - "value_error.empty"); - n_inputs = 1; - } else if (cJSON_IsArray(input)) { - n_inputs = cJSON_GetArraySize(input); - if (n_inputs < 1) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", - "value_error.list.min_items"); - else if (n_inputs > PPLX_API_MAX_STANDARD_INPUTS) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", - "value_error.list.max_items"); - cJSON *item; - int i = 0; - cJSON_ArrayForEach(item, input) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i); - if (!cJSON_IsString(item)) { - ve_add(detail, loc, "input item must be a string", "type_error.string"); - } else if (!item->valuestring || item->valuestring[0] == '\0') { - ve_add(detail, loc, "input item must not be empty", "value_error.empty"); - } - i++; - } - } else { - ve_add(detail, "[\"body\",\"input\"]", - "input must be a string or an array of strings", "type_error"); - } - - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); - cJSON_Delete(detail); - return; - } - - if (cJSON_GetArraySize(detail) == 0 && m && input) { - out->n_inputs = n_inputs; - out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens)); - out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs)); - - int idx = 0; - if (input && cJSON_IsString(input) && input->valuestring) { - if (tokenize_one(m, j, input->valuestring, &out->tokens[0]) != 0) { - ve_add(detail, "[\"body\",\"input\"]", "tokenization failed", - "value_error.tokenization"); - } else { - out->inputs[0].ids = out->tokens[0].ids; - out->inputs[0].n_tokens = out->tokens[0].n_tokens; - out->total_tokens = out->tokens[0].n_tokens; - if (out->tokens[0].n_tokens > PPLX_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", - "input exceeds 32768 token limit", - "value_error.context_length"); - } - } else { - cJSON *item; - cJSON_ArrayForEach(item, input) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); - if (!cJSON_IsString(item) || !item->valuestring) { - ve_add(detail, loc, "input item must be a string", - "type_error.string"); - idx++; - continue; - } - if (tokenize_one(m, j, item->valuestring, - &out->tokens[idx]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); - } else { - out->inputs[idx].ids = out->tokens[idx].ids; - out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; - out->total_tokens += out->tokens[idx].n_tokens; - if (out->tokens[idx].n_tokens > PPLX_API_MAX_ITEM_TOKENS) - ve_add(detail, loc, "input exceeds 32768 token limit", - "value_error.context_length"); - } - idx++; - } - } - - if (out->total_tokens > PPLX_API_MAX_TOTAL_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", - "value_error.context_length"); - } - - if (cJSON_GetArraySize(detail) > 0) { - job_set_422(j, detail); - embedding_request_free(out); - out->j = j; - } else { - out->ready = 1; - } - cJSON_Delete(detail); -} - -static void prepare_contextual_request(job *j, cJSON *root, http_server *s, - contextual_request *out) { - memset(out, 0, sizeof(*out)); - out->j = j; - out->root = root; - - cJSON *detail = cJSON_CreateArray(); - loaded_model *m = NULL; - int dims = 0; - const char *encoding = NULL; - int unloaded = validate_common(root, detail, 1, &m, &dims, &encoding, s); - out->model = m; - out->dims = dims; - out->encoding = encoding; - - cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); - int n_docs = 0; - int total_chunks = 0; - if (!input) { - ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); - } else if (!cJSON_IsArray(input)) { - ve_add(detail, "[\"body\",\"input\"]", - "input must be an array of document chunk arrays", "type_error.array"); - } else { - n_docs = cJSON_GetArraySize(input); - if (n_docs < 1) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", - "value_error.list.min_items"); - else if (n_docs > PPLX_API_MAX_CONTEXT_DOCS) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", - "value_error.list.max_items"); - cJSON *doc_arr; - int di = 0; - cJSON_ArrayForEach(doc_arr, input) { - if (!cJSON_IsArray(doc_arr)) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document must be an array of strings", - "type_error.array"); - di++; - continue; - } - int doc_chunks = cJSON_GetArraySize(doc_arr); - if (doc_chunks > PPLX_API_MAX_CONTEXT_CHUNKS - total_chunks) - total_chunks = PPLX_API_MAX_CONTEXT_CHUNKS + 1; - else - total_chunks += doc_chunks; - if (doc_chunks < 1) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document must contain at least 1 chunk", - "value_error.list.min_items"); - } - cJSON *chunk; - int ci = 0; - cJSON_ArrayForEach(chunk, doc_arr) { - char loc[80]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); - if (!cJSON_IsString(chunk)) { - ve_add(detail, loc, "chunk must be a string", "type_error.string"); - } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { - ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); - } - ci++; - } - di++; - } - if (total_chunks > PPLX_API_MAX_CONTEXT_CHUNKS) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", - "value_error.list.max_items"); - } - - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); - cJSON_Delete(detail); - return; - } - - if (cJSON_GetArraySize(detail) == 0 && m) { - out->n_docs = n_docs; - out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); - int64_t request_tokens = 0; - cJSON *doc_arr; - int di = 0; - cJSON_ArrayForEach(doc_arr, input) { - contextual_doc *doc = &out->docs[di]; - int n_chunks = cJSON_GetArraySize(doc_arr); - token_buf *chunk_tokens = - xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); - doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); - doc->n_spans = n_chunks; - int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; - int doc_valid = 1; - - cJSON *chunk; - int ci = 0; - cJSON_ArrayForEach(chunk, doc_arr) { - char loc[80]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); - if (tokenize_one(m, j, chunk->valuestring, - &chunk_tokens[ci]) != 0) { - ve_add(detail, loc, "tokenization failed", - "value_error.tokenization"); - doc_valid = 0; - } else { - doc_tokens += chunk_tokens[ci].n_tokens; - } - ci++; - } - - if (doc_tokens > PPLX_API_MAX_ITEM_TOKENS) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document exceeds 32768 token limit", - "value_error.context_length"); - doc_valid = 0; - } - request_tokens += doc_tokens; - - if (doc_valid) { - doc->n_tokens = (int)doc_tokens; - doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); - int pos = 0; - for (ci = 0; ci < n_chunks; ci++) { - doc->spans[ci].start = pos; - doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens; - memcpy(doc->ids + pos, chunk_tokens[ci].ids, - (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); - pos += chunk_tokens[ci].n_tokens; - if (ci + 1 < n_chunks) { - doc->ids[pos++] = m->context_separator_id; - } - } - } - free_token_bufs(chunk_tokens, n_chunks); - free(chunk_tokens); - di++; - } - if (request_tokens > PPLX_API_MAX_TOTAL_TOKENS) { - ve_add(detail, "[\"body\",\"input\"]", - "request exceeds 120000 token limit", - "value_error.context_length"); - } else { - out->total_tokens = (int)request_tokens; - out->total_chunks = total_chunks; - } - } - - if (cJSON_GetArraySize(detail) > 0) { - job_set_422(j, detail); - contextual_request_free(out); - out->j = j; - } else { - out->ready = 1; - } - cJSON_Delete(detail); -} - -static int parse_job_root(job *j, cJSON **out_root) { - cJSON *detail = cJSON_CreateArray(); - cJSON *root = parse_json_body(j, detail); - if (!root) { - job_set_422(j, detail); - cJSON_Delete(detail); - *out_root = NULL; - return -1; - } - cJSON_Delete(detail); - *out_root = root; - return 0; -} - -static void process_unknown_job(job *j) { - cJSON *root = NULL; - if (!j->started_ns) j->started_ns = nstime(); - uint64_t t0 = nstime(); - int rc = parse_job_root(j, &root); - j->parse_ns += nstime() - t0; - if (rc != 0) - return; - job_set_error(j, 404, "unknown endpoint", "invalid_request_error"); - cJSON_Delete(root); -} - -static void process_job_group(http_server *s, job **jobs, int n_jobs) { - embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); - contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); - int *kind = xcalloc((size_t)n_jobs, sizeof(*kind)); - int *done = xcalloc((size_t)n_jobs, sizeof(*done)); - embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); - contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group)); - - for (int i = 0; i < n_jobs; i++) { - cJSON *root = NULL; - if (!jobs[i]->started_ns) jobs[i]->started_ns = nstime(); - if (!strcmp(jobs[i]->path, "/v1/embeddings")) { - kind[i] = 1; - } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { - kind[i] = 2; - } else { - continue; - } - uint64_t t0 = nstime(); - int parse_rc = parse_job_root(jobs[i], &root); - jobs[i]->parse_ns += nstime() - t0; - if (parse_rc != 0) - continue; - if (kind[i] == 1) - prepare_embedding_request(jobs[i], root, s, &std_reqs[i]); - else - prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]); - } - - for (int i = 0; i < n_jobs; i++) { - if (done[i]) - continue; - - if (!kind[i]) { - process_unknown_job(jobs[i]); - job_set_timing_header(jobs[i]); - enqueue_done(jobs[i]); - done[i] = 1; - continue; - } - - if ((kind[i] == 1 && !std_reqs[i].ready) || - (kind[i] == 2 && !ctx_reqs[i].ready)) { - job_set_timing_header(jobs[i]); - enqueue_done(jobs[i]); - done[i] = 1; - continue; - } - - if (kind[i] == 1) { - int group_n = 1; - int group_inputs = std_reqs[i].n_inputs; - int group_tokens = std_reqs[i].total_tokens; - std_group[0] = &std_reqs[i]; - for (int k = i + 1; k < n_jobs; k++) { - if (!done[k] && kind[k] == 1 && std_reqs[k].ready && - embedding_request_compatible(&std_reqs[i], &std_reqs[k]) && - embedding_request_fits_group(&std_reqs[k], group_inputs, - group_tokens, s->batch_size, s->max_batch_tokens)) { - std_group[group_n++] = &std_reqs[k]; - group_inputs += std_reqs[k].n_inputs; - group_tokens += std_reqs[k].total_tokens; - } - } - execute_embedding_request_list(std_group, group_n); - for (int k = 0; k < group_n; k++) { - int idx = (int)(std_group[k] - std_reqs); - job_set_timing_header(jobs[idx]); - enqueue_done(jobs[idx]); - done[idx] = 1; - } - } else { - int group_n = 1; - int group_docs = ctx_reqs[i].n_docs; - int group_tokens = ctx_reqs[i].total_tokens; - ctx_group[0] = &ctx_reqs[i]; - for (int k = i + 1; k < n_jobs; k++) { - if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && - contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) && - contextual_request_fits_group(&ctx_reqs[k], group_docs, - group_tokens, s->batch_size, s->max_batch_tokens)) { - ctx_group[group_n++] = &ctx_reqs[k]; - group_docs += ctx_reqs[k].n_docs; - group_tokens += ctx_reqs[k].total_tokens; - } - } - execute_contextual_request_list(ctx_group, group_n); - for (int k = 0; k < group_n; k++) { - int idx = (int)(ctx_group[k] - ctx_reqs); - job_set_timing_header(jobs[idx]); - enqueue_done(jobs[idx]); - done[idx] = 1; - } - } - } - - for (int i = 0; i < n_jobs; i++) { - embedding_request_free(&std_reqs[i]); - contextual_request_free(&ctx_reqs[i]); - } - free(ctx_group); - free(std_group); - free(done); - free(kind); - free(ctx_reqs); - free(std_reqs); -} - -static void *worker_main(void *arg) { - http_server *s = arg; -#ifdef USE_MLX - if (s->use_mlx) { - int rc = 0; - for (int i = 0; i < 4; i++) { - loaded_model *m = &s->models[i]; - if (!m->info) continue; - pplx_mlx_options_t mlx_opts = { - .quantize_bits = s->mlx_quantize_bits, - .quantize_group_size = s->mlx_quantize_group_size, - }; - m->mlx_ctx = pplx_mlx_load_with_options(m->path, &mlx_opts); - if (!m->mlx_ctx) { - server_log("pplx-serve: failed to load MLX model on worker: %s", - m->path); - rc = 1; - break; - } - if (pplx_mlx_config(m->mlx_ctx)->hidden_size != m->info->dim) { - server_log("pplx-serve: model %s has unexpected dimension %d", - m->info->id, - pplx_mlx_config(m->mlx_ctx)->hidden_size); - rc = 1; - break; - } - } - pthread_mutex_lock(&s->mu); - s->worker_init_rc = rc; - s->worker_ready = 1; - if (rc) s->stopping = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - if (rc) { - for (int i = 0; i < 4; i++) { - loaded_model *m = &s->models[i]; - if (m->mlx_ctx) { - pplx_mlx_free(m->mlx_ctx); - m->mlx_ctx = NULL; - } - } - return NULL; - } - } else -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - int rc = 0; - for (int i = 0; i < 4; i++) { - loaded_model *m = &s->models[i]; - if (!m->info) continue; - m->cuda_ctx = pplx_cuda_load(m->path); - if (!m->cuda_ctx) { - server_log("pplx-serve: failed to load CUDA model on worker: %s", - m->path); - rc = 1; - break; - } - if (pplx_cuda_config(m->cuda_ctx)->hidden_size != m->info->dim) { - server_log("pplx-serve: model %s has unexpected dimension %d", - m->info->id, - pplx_cuda_config(m->cuda_ctx)->hidden_size); - rc = 1; - break; - } - } - pthread_mutex_lock(&s->mu); - s->worker_init_rc = rc; - s->worker_ready = 1; - if (rc) s->stopping = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - if (rc) { - for (int i = 0; i < 4; i++) { - loaded_model *m = &s->models[i]; - if (m->cuda_ctx) { - pplx_cuda_free(m->cuda_ctx); - m->cuda_ctx = NULL; - } - } - return NULL; - } - } else -#endif - { - pthread_mutex_lock(&s->mu); - s->worker_ready = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - } - - for (;;) { - job *j = dequeue_job(s); - if (!j) break; - job *jobs[PPLX_SERVER_MICROBATCH_MAX_JOBS]; - int n_jobs = 1; - jobs[0] = j; - - if ((!strcmp(j->path, "/v1/embeddings") || - !strcmp(j->path, "/v1/contextualizedembeddings")) && - s->batch_size > 1) { - if (s->batch_wait_us > 0) - usleep((useconds_t)s->batch_wait_us); - n_jobs += drain_ready_jobs(s, jobs + 1, - PPLX_SERVER_MICROBATCH_MAX_JOBS - 1); - } - - process_job_group(s, jobs, n_jobs); - } -#ifdef USE_MLX - if (s->use_mlx) { - for (int i = 0; i < 4; i++) { - loaded_model *m = &s->models[i]; - if (m->mlx_ctx) { - pplx_mlx_free(m->mlx_ctx); - m->mlx_ctx = NULL; - } - } - } -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - for (int i = 0; i < 4; i++) { - loaded_model *m = &s->models[i]; - if (m->cuda_ctx) { - pplx_cuda_free(m->cuda_ctx); - m->cuda_ctx = NULL; - } - } - } -#endif - return NULL; -} - -static int listen_on(const char *host, int port) { - char portbuf[32]; - snprintf(portbuf, sizeof(portbuf), "%d", port); - char err[ANET_ERR_LEN]; - int fd = anetTcpServer(err, port, (char *)host, PPLX_HTTP_BACKLOG); - (void)portbuf; - if (fd == ANET_ERR) { - server_log("pplx-serve: listen failed on %s:%d: %s", host, port, err); - return -1; - } - set_nonblock(fd); - return fd; -} - -static int load_one_model(http_server *s, model_slot slot, const char *path) { - loaded_model *m = &s->models[slot]; - m->info = &k_models[slot]; - m->path = xstrdup(path); - char vocab_path[1024]; - snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); - m->tok = qwen_tokenizer_load(vocab_path); - if (!m->tok) { - server_log("pplx-serve: failed to load tokenizer: %s", vocab_path); - return -1; - } - m->tok_ws = qwen_tokenizer_workspace_new(); - if (!m->tok_ws) { - server_log("pplx-serve: failed to allocate tokenizer workspace: %s", - path); - return -1; - } - m->context_separator_id = PPLX_CONTEXT_SEPARATOR_TOKEN_ID; -#ifdef USE_MLX - if (s->use_mlx) { - /* MLX streams are thread-local. The inference worker loads the MLX - * model so all MLX arrays/streams are created, used, and freed by the - * same thread. */ - return 0; - } -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - return 0; - } -#endif - m->cpu_model = pplx_model_load(path); - if (!m->cpu_model) { - server_log("pplx-serve: failed to load model: %s", path); - return -1; - } - m->cpu_ws = pplx_workspace_new(m->cpu_model); - if (!m->cpu_ws) { - server_log("pplx-serve: failed to allocate workspace: %s", path); - return -1; - } - if (pplx_model_config(m->cpu_model)->hidden_size != m->info->dim) { - server_log("pplx-serve: model %s has unexpected dimension %d", - m->info->id, pplx_model_config(m->cpu_model)->hidden_size); - return -1; - } - return 0; -} - -static void free_models(http_server *s) { - for (int i = 0; i < 4; i++) { - loaded_model *m = &s->models[i]; - free(m->path); - if (m->tok_ws) qwen_tokenizer_workspace_free(m->tok_ws); - if (m->tok) qwen_tokenizer_free(m->tok); - if (m->cpu_ws) pplx_workspace_free(m->cpu_ws); - if (m->cpu_model) pplx_model_free(m->cpu_model); -#ifdef USE_MLX - if (m->mlx_ctx) pplx_mlx_free(m->mlx_ctx); -#endif -#ifdef USE_CUDA - if (m->cuda_ctx) pplx_cuda_free(m->cuda_ctx); -#endif - } -} - -static uint64_t physical_memory_nbytes(void) { -#ifdef __APPLE__ - uint64_t bytes = 0; - size_t len = sizeof(bytes); - if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && - len == sizeof(bytes) && bytes > 0) - return bytes; -#endif - long pages = sysconf(_SC_PHYS_PAGES); - long page_size = sysconf(_SC_PAGESIZE); - if (pages <= 0 || page_size <= 0 || - (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) - return 0; - return (uint64_t)pages * (uint64_t)page_size; -} - -static int validate_model_specs(const pplx_server_config_t *cfg) { - int seen[4] = {0}; - for (int i = 0; i < cfg->n_models; i++) { - const pplx_server_model_spec_t *spec = &cfg->models[i]; - model_slot slot = model_slot_for_id(spec->id); - if (slot == MODEL_UNKNOWN) { - fprintf(stderr, "pplx-serve: unknown model id: %s\n", - spec->id ? spec->id : ""); - return -1; - } - if (!spec->path || !spec->path[0]) { - fprintf(stderr, "pplx-serve: model %s has an empty path\n", - spec->id); - return -1; - } - if (seen[slot]) { - fprintf(stderr, "pplx-serve: duplicate model id: %s\n", spec->id); - return -1; - } - seen[slot] = 1; - } - return 0; -} - -static int mlx_memory_preflight(const pplx_server_config_t *cfg) { - if (!cfg->use_mlx) return 0; - - uint64_t payload = 0; - for (int i = 0; i < cfg->n_models; i++) { - multi_safetensors_t *ms = - multi_safetensors_open(cfg->models[i].path); - size_t model_payload = 0; - if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { - multi_safetensors_close(ms); - fprintf(stderr, "pplx-serve: failed to inspect MLX model: %s\n", - cfg->models[i].path); - return -1; - } - multi_safetensors_close(ms); - if ((uint64_t)model_payload > UINT64_MAX - payload) { - fprintf(stderr, "pplx-serve: MLX model payload size overflow\n"); - return -1; - } - payload += (uint64_t)model_payload; - } - - if (payload > UINT64_MAX / PPLX_MLX_RESIDENT_MULTIPLIER) { - fprintf(stderr, "pplx-serve: MLX resident-memory estimate overflow\n"); - return -1; - } - uint64_t estimated = payload * PPLX_MLX_RESIDENT_MULTIPLIER; - uint64_t physical = physical_memory_nbytes(); - if (physical == 0) { - server_log("pplx-serve: warning: could not determine physical memory; " - "skipping MLX memory preflight"); - return 0; - } - double utilization = cfg->memory_utilization > 0.0 - ? cfg->memory_utilization - : PPLX_MLX_MEMORY_BUDGET_PERCENT / 100.0; - uint64_t budget = (uint64_t)((double)physical * utilization); - const double gib = 1024.0 * 1024.0 * 1024.0; - server_log("pplx-serve: MLX memory preflight: %.1f GiB tensors, " - "%.1f GiB estimated resident, %.1f GiB budget " - "(%.2f of physical memory)", - (double)payload / gib, (double)estimated / gib, - (double)budget / gib, utilization); - if (cfg->mlx_quantize_bits) { - server_log("pplx-serve: MLX %d-bit quantization enabled; preflight " - "uses source payload as a conservative peak estimate", - cfg->mlx_quantize_bits); - } - if (estimated <= budget) return 0; - server_log("pplx-serve: refusing MLX model set above the host-memory " - "budget"); - server_log("pplx-serve: use BF16 artifacts, load fewer models, or raise " - "--memory-utilization (values above 1.0 overcommit)"); - return -1; -} - -int pplx_run_server(const pplx_server_config_t *cfg) { - if (!cfg || !cfg->models || cfg->n_models <= 0) { - fprintf(stderr, "pplx-serve: at least one --model MODEL_ID=PATH is required\n"); - return 1; - } - if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0) - return 1; - - http_server s; - memset(&s, 0, sizeof(s)); - s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; - s.port = cfg->port > 0 ? cfg->port : 8000; - s.batch_size = cfg->batch_size > 0 - ? cfg->batch_size : PPLX_SERVER_DEFAULT_BATCH_SIZE; - s.max_batch_tokens = cfg->max_batch_tokens > 0 - ? cfg->max_batch_tokens : PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS; - s.batch_wait_us = cfg->batch_wait_us >= 0 - ? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US; - s.enable_cors = cfg->enable_cors; - s.use_mlx = cfg->use_mlx; - s.use_cuda = cfg->use_cuda; - s.mlx_quantize_bits = cfg->mlx_quantize_bits; - s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 - ? cfg->mlx_quantize_group_size : 64; - if (cfg->api_key && cfg->api_key[0]) - s.api_key = xstrdup(cfg->api_key); - else { - const char *env_key = getenv("PPLX_API_KEY"); - if (env_key && env_key[0]) s.api_key = xstrdup(env_key); - } - - pthread_mutex_init(&s.mu, NULL); - pthread_cond_init(&s.cv, NULL); - - for (int i = 0; i < cfg->n_models; i++) { - model_slot slot = model_slot_for_id(cfg->models[i].id); - if (slot == MODEL_UNKNOWN) { - fprintf(stderr, "pplx-serve: unknown model id: %s\n", cfg->models[i].id); - free_models(&s); - return 1; - } - if (load_one_model(&s, slot, cfg->models[i].path) != 0) { - free_models(&s); - return 1; - } - } - - if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { - perror("pipe"); - free_models(&s); - return 1; - } - set_nonblock(s.completion_pipe[0]); - set_nonblock(s.completion_pipe[1]); - set_nonblock(s.signal_pipe[0]); - set_nonblock(s.signal_pipe[1]); - - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = stop_signal_handler; - sigemptyset(&sa.sa_mask); - sigaction(SIGINT, &sa, NULL); - sigaction(SIGTERM, &sa, NULL); - signal(SIGPIPE, SIG_IGN); - g_signal_wfd = s.signal_pipe[1]; - - s.loop = aeCreateEventLoop(PPLX_HTTP_SETSIZE); - if (!s.loop) { - fprintf(stderr, "pplx-serve: failed to create event loop\n"); - free_models(&s); - return 1; - } - - if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { - fprintf(stderr, "pplx-serve: failed to start worker\n"); - aeDeleteEventLoop(s.loop); - free_models(&s); - return 1; - } - pthread_mutex_lock(&s.mu); - while (!s.worker_ready) - pthread_cond_wait(&s.cv, &s.mu); - int worker_init_rc = s.worker_init_rc; - pthread_mutex_unlock(&s.mu); - if (worker_init_rc) { - pthread_join(s.worker, NULL); - aeDeleteEventLoop(s.loop); - free_models(&s); - free(s.api_key); - pthread_cond_destroy(&s.cv); - pthread_mutex_destroy(&s.mu); - return 1; - } - - s.listen_fd = listen_on(s.host, s.port); - if (s.listen_fd < 0) { - pthread_mutex_lock(&s.mu); - s.stopping = 1; - pthread_cond_signal(&s.cv); - pthread_mutex_unlock(&s.mu); - pthread_join(s.worker, NULL); - aeDeleteEventLoop(s.loop); - free_models(&s); - return 1; - } - - aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); - aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); - aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); - aeCreateTimeEvent(s.loop, PPLX_HTTP_SWEEP_MS, timeout_cb, &s, NULL); - - server_log("pplx-serve: listening on http://%s:%d", s.host, s.port); - aeMain(s.loop); - - server_log("pplx-serve: shutdown requested"); - if (s.listen_fd >= 0) close(s.listen_fd); - client *c = s.clients; - while (c) { - client *next = c->next; - close_client(c); - c = next; - } - - pthread_mutex_lock(&s.mu); - s.stopping = 1; - pthread_cond_signal(&s.cv); - pthread_mutex_unlock(&s.mu); - pthread_join(s.worker, NULL); - - aeDeleteEventLoop(s.loop); - close(s.completion_pipe[0]); - close(s.completion_pipe[1]); - close(s.signal_pipe[0]); - close(s.signal_pipe[1]); - free_models(&s); - free(s.api_key); - pthread_cond_destroy(&s.cv); - pthread_mutex_destroy(&s.mu); - return 0; -} diff --git a/pplx_server.h b/pplx_server.h deleted file mode 100644 index c38f0ca..0000000 --- a/pplx_server.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef PPLX_SERVER_H -#define PPLX_SERVER_H - -#include - -typedef struct { - const char *id; - const char *path; -} pplx_server_model_spec_t; - -typedef struct { - const pplx_server_model_spec_t *models; - int n_models; - const char *host; - int port; - int batch_size; - int max_batch_tokens; - int batch_wait_us; - int use_mlx; - int use_cuda; - int mlx_quantize_bits; - int mlx_quantize_group_size; - double memory_utilization; /* fraction of physical memory the MLX - preflight may plan for; 0 = default 0.90, - values above 1.0 allow overcommit */ - int enable_cors; - const char *api_key; -} pplx_server_config_t; - -int pplx_run_server(const pplx_server_config_t *cfg); - -#endif diff --git a/server_main.c b/server_main.c deleted file mode 100644 index ace0100..0000000 --- a/server_main.c +++ /dev/null @@ -1,340 +0,0 @@ -/* server_main.c - pplx-embed-server: Perplexity-compatible embeddings API */ - -#include "pplx_embed.h" -#include "pplx_server.h" -#include "qwen_kernels.h" - -#ifdef USE_CUDA -#include "pplx_embed_cuda.h" -#endif - -#include -#include -#include - -static void print_usage(const char *prog) -{ - fprintf(stderr, - "Usage: %s --model ID=DIR [options]\n" - "\n" - "Serves POST /v1/embeddings and /v1/contextualizedembeddings.\n" - "\n" - "Options:\n" - " --model ID=DIR Model to serve (repeatable)\n" - " --backend cpu|mlx|cuda Execution backend (default: cpu); --mlx and\n" - " --cuda are shorthands\n" - " --host HOST Bind host (default: 127.0.0.1)\n" - " --port N Bind port (default: 8000)\n" - " --api-key K Require Authorization: Bearer K\n" - " --cors Enable CORS headers\n" - " -b, --batch-size N Max texts or documents per inference batch\n" - " (default: 8)\n" - " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" - " --batch-wait-us N Micro-batch wait in microseconds\n" - " (default: 1000; 0 drains only queued work)\n" - " --memory-utilization F Fraction of physical memory the MLX model-set\n" - " preflight may plan for (default: 0.90; values\n" - " above 1.0 overcommit)\n" - " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" - " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" - " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" - " (default: f32, exact)\n" - " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" - " f32); bf16 halves weight memory\n" - " -t, --threads N CPU threads (default: all cores)\n" - " -v, --verbose Verbose (-vv for debug)\n" - " -h, --help Show this help\n" - "\n" - "Examples:\n" - " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" - " %s --backend mlx --mlx-quant-bits 8 \\\n" - " --model pplx-embed-v1-4b=./model-4b-bf16\n", - prog, prog, prog); -} - -typedef struct { - pplx_server_model_spec_t *v; - int n; - int cap; -} model_specs_t; - -static int append_model_spec(model_specs_t *specs, const char *arg) -{ - const char *eq = strchr(arg, '='); - if (!eq || eq == arg || !eq[1]) { - fprintf(stderr, "--model expects MODEL_ID=DIR\n"); - return -1; - } - if (specs->n == specs->cap) { - int new_cap = specs->cap ? specs->cap * 2 : 4; - void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); - if (!p) return -1; - specs->v = p; - specs->cap = new_cap; - } - size_t id_len = (size_t)(eq - arg); - char *id = (char *)malloc(id_len + 1); - char *path = strdup(eq + 1); - if (id) { - memcpy(id, arg, id_len); - id[id_len] = '\0'; - } - specs->v[specs->n].id = id; - specs->v[specs->n].path = path; - if (!specs->v[specs->n].id || !specs->v[specs->n].path) - return -1; - specs->n++; - return 0; -} - -static void free_model_specs(model_specs_t *specs) -{ - if (!specs) return; - for (int i = 0; i < specs->n; i++) { - free((char *)specs->v[i].id); - free((char *)specs->v[i].path); - } - free(specs->v); -} - -int main(int argc, char *argv[]) -{ - int n_threads = 0; - int verbose = 0; - int use_mlx = 0; - int use_cuda = 0; - int batch_size = 0; - int max_batch_tokens = 0; - int batch_wait_us = -1; - int mlx_quantize_bits = 0; - int mlx_quantize_group_size = 64; - const char *cuda_fast_gemm = NULL; - const char *cuda_weights = NULL; - double memory_utilization = 0.0; - const char *host = "127.0.0.1"; - const char *api_key = NULL; - int port = 8000; - int cors = 0; - model_specs_t model_specs = {0}; - - int arg = 1; - while (arg < argc && argv[arg][0] == '-') { - const char *f = argv[arg]; - if (!strcmp(f, "--model")) { - if (append_model_spec(&model_specs, argv[++arg]) != 0) { - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--backend")) { - const char *backend = argv[++arg]; - if (!strcmp(backend, "mlx")) { -#ifdef USE_MLX - use_mlx = 1; - use_cuda = 0; -#else - fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); - free_model_specs(&model_specs); - return 1; -#endif - } else if (!strcmp(backend, "cuda")) { -#ifdef USE_CUDA - use_cuda = 1; - use_mlx = 0; -#else - fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); - free_model_specs(&model_specs); - return 1; -#endif - } else if (!strcmp(backend, "cpu")) { - use_mlx = 0; - use_cuda = 0; - } else { - fprintf(stderr, "invalid --backend: %s\n", backend); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--mlx")) { -#ifdef USE_MLX - use_mlx = 1; - use_cuda = 0; -#else - fprintf(stderr, "--mlx not available (build with: make mlx)\n"); - free_model_specs(&model_specs); - return 1; -#endif - } - else if (!strcmp(f, "--cuda")) { -#ifdef USE_CUDA - use_cuda = 1; - use_mlx = 0; -#else - fprintf(stderr, "--cuda not available (build with: make cuda)\n"); - free_model_specs(&model_specs); - return 1; -#endif - } - else if (!strcmp(f, "--host")) { host = argv[++arg]; } - else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); } - else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; } - else if (!strcmp(f, "--cors")) { cors = 1; } - else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { - batch_size = atoi(argv[++arg]); - } - else if (!strcmp(f, "--max-batch-tokens")) { - max_batch_tokens = atoi(argv[++arg]); - if (max_batch_tokens <= 0) { - fprintf(stderr, "--max-batch-tokens must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--batch-wait-us")) { - batch_wait_us = atoi(argv[++arg]); - if (batch_wait_us < 0) { - fprintf(stderr, "--batch-wait-us must be >= 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--memory-utilization")) { - memory_utilization = atof(argv[++arg]); - if (memory_utilization <= 0.0) { - fprintf(stderr, "--memory-utilization must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--mlx-quant-bits")) { - mlx_quantize_bits = atoi(argv[++arg]); - if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--mlx-quant-group-size")) { - mlx_quantize_group_size = atoi(argv[++arg]); - if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--cuda-gemm-mode")) { - cuda_fast_gemm = argv[++arg]; - } - else if (!strcmp(f, "--cuda-weight-dtype")) { - cuda_weights = argv[++arg]; - } - else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { - n_threads = atoi(argv[++arg]); - } - else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } - else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { - print_usage(argv[0]); - free_model_specs(&model_specs); - return 0; - } - else { - fprintf(stderr, "unknown option: %s\n", f); - print_usage(argv[0]); - free_model_specs(&model_specs); - return 1; - } - arg++; - } - - pplx_verbose = verbose; - qwen_verbose = verbose; - - if (arg < argc) { - fprintf(stderr, "unexpected argument: %s\n", argv[arg]); - print_usage(argv[0]); - free_model_specs(&model_specs); - return 1; - } - if (model_specs.n == 0) { - fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n"); - print_usage(argv[0]); - free_model_specs(&model_specs); - return 1; - } - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quant-bits requires --backend mlx\n"); - free_model_specs(&model_specs); - return 1; - } - - if (cuda_fast_gemm) { -#ifdef USE_CUDA - if (!use_cuda) { - fprintf(stderr, "--cuda-gemm-mode requires --backend cuda\n"); - free_model_specs(&model_specs); - return 1; - } - if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { - fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); - free_model_specs(&model_specs); - return 1; - } -#else - fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); - free_model_specs(&model_specs); - return 1; -#endif - } - - if (cuda_weights) { -#ifdef USE_CUDA - if (!use_cuda) { - fprintf(stderr, "--cuda-weight-dtype requires --backend cuda\n"); - free_model_specs(&model_specs); - return 1; - } - if (!strcmp(cuda_weights, "bf16")) { - pplx_cuda_set_weights_bf16(1); - } else if (!strcmp(cuda_weights, "f32")) { - pplx_cuda_set_weights_bf16(0); - } else { - fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); - free_model_specs(&model_specs); - return 1; - } -#else - fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); - free_model_specs(&model_specs); - return 1; -#endif - } - - if (!use_mlx && !use_cuda) { - if (n_threads <= 0) n_threads = qwen_get_num_cpus(); - qwen_set_threads(n_threads); - if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } else if (verbose >= 1) { - fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); - } - - pplx_server_config_t scfg = { - .models = model_specs.v, - .n_models = model_specs.n, - .host = host, - .port = port, - .batch_size = batch_size > 0 ? batch_size : 8, - .max_batch_tokens = max_batch_tokens, - .batch_wait_us = batch_wait_us, - .use_mlx = use_mlx, - .use_cuda = use_cuda, - .mlx_quantize_bits = mlx_quantize_bits, - .mlx_quantize_group_size = mlx_quantize_group_size, - .memory_utilization = memory_utilization, - .enable_cors = cors, - .api_key = api_key, - }; - int rc = pplx_run_server(&scfg); - free_model_specs(&model_specs); - return rc; -} -- cgit v1.3.1