diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 08:24:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 2c37727e653d2340d8977dd5db4addc767dd81ea (patch) | |
| tree | 578ad6affac511f847a346a0efc6d40ebf6a279f /pplx_embed_mlx.c | |
| parent | ec417bcb52f2aa8e24aeec312e8ddbf3a0798056 (diff) | |
| download | ffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.tar.gz ffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.zip | |
gitignore: track new binary and library names
Diffstat (limited to 'pplx_embed_mlx.c')
| -rw-r--r-- | pplx_embed_mlx.c | 2296 |
1 files changed, 0 insertions, 2296 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c deleted file mode 100644 index 602571d..0000000 --- a/pplx_embed_mlx.c +++ /dev/null @@ -1,2296 +0,0 @@ -#include "pplx_embed_mlx.h" -#include "pplx_embed.h" -#include "qwen_safetensors.h" - -#include <mlx/c/mlx.h> - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <math.h> -#include <limits.h> -#include <float.h> - -/* ======================================================================== - * Per-layer MLX weight arrays - * ======================================================================== */ - -typedef struct { - mlx_array w; - mlx_array wt; - mlx_array scales; - mlx_array biases; - int quantized; - int bits; - int group_size; -} mlx_linear_t; - -typedef struct { - mlx_linear_t wq, wk, wv, wo; - mlx_array q_norm, k_norm; - mlx_array input_norm, post_attn_norm; - mlx_linear_t gate_proj, up_proj, down_proj; -} mlx_layer_t; - -struct pplx_mlx_ctx { - pplx_config_t config; - mlx_array embed_tokens; - mlx_layer_t *layers; /* heap-allocated [n_layers] */ - mlx_array norm; - mlx_stream stream; - multi_safetensors_t *ms; - int layer_start; /* loaded transformer range, inclusive */ - int layer_end; /* loaded transformer range, exclusive */ - int quantize_bits; - int quantize_group_size; - mlx_dtype weight_dtype; -}; - -struct pplx_mlx_late_ctx { - pplx_mlx_ctx_t *base; - multi_safetensors_t *dense_ms; - mlx_linear_t projection; - int token_dim; -}; - -struct pplx_mlx_late_vectors { - pplx_mlx_late_ctx_t *owner; - mlx_array vectors; /* [tokens, dim] */ - int tokens; - int dim; -}; - -/* ======================================================================== - * Helpers - * ======================================================================== */ - -static int arr_ok(mlx_array a) { return a.ctx != NULL; } - -static int mlx_mul_size(size_t a, size_t b, size_t *out) -{ - if (a != 0 && b > SIZE_MAX / a) return -1; - *out = a * b; - return 0; -} - -static mlx_dtype mlx_weight_dtype(const pplx_mlx_ctx_t *ctx) -{ - return ctx->weight_dtype; -} - -static size_t safetensor_dtype_size(safetensor_dtype_t dtype) -{ - switch (dtype) { - case DTYPE_F32: return sizeof(float); - case DTYPE_BF16: return sizeof(uint16_t); - default: return 0; - } -} - -static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, - const safetensor_t *t, - const char *name, const int *shape, - int ndim) -{ - size_t elem_size = safetensor_dtype_size(t->dtype); - if (elem_size == 0) { - fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); - return 0; - } - if (t->ndim != ndim) { - fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", - name, t->ndim, ndim); - return 0; - } - - size_t numel = 1; - for (int i = 0; i < ndim; i++) { - if (shape[i] <= 0 || t->shape[i] != shape[i]) { - fprintf(stderr, "mlx: bad shape for %s at dim %d: " - "got %lld, expected %d\n", - name, i, (long long)t->shape[i], shape[i]); - return 0; - } - if (mlx_mul_size(numel, (size_t)shape[i], &numel) != 0) { - fprintf(stderr, "mlx: shape too large for %s\n", name); - return 0; - } - } - - size_t bytes; - if (mlx_mul_size(numel, elem_size, &bytes) != 0) { - fprintf(stderr, "mlx: tensor data too large for %s\n", name); - return 0; - } - if (t->data_size != bytes) { - fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", - name, t->data_size, bytes); - return 0; - } - - if (sf->header_size > SIZE_MAX - 8) return 0; - size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || - t->data_offset > sf->file_size - data_start || - t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name); - return 0; - } - return 1; -} - -static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, - const int *shape, int ndim) -{ - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); - if (!t) { - fprintf(stderr, "mlx: tensor not found: %s\n", name); - return (mlx_array){0}; - } - if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim)) - return (mlx_array){0}; - mlx_dtype dtype; - if (t->dtype == DTYPE_F32) { - dtype = MLX_FLOAT32; - } else if (t->dtype == DTYPE_BF16) { - dtype = MLX_BFLOAT16; - } else { - fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); - return (mlx_array){0}; - } - return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); -} - -static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms, - const char *name, - const int *shape, int ndim) -{ - mlx_linear_t l = {0}; - l.w = load_tensor(ms, name, shape, ndim); - return l; -} - -static int linear_ok(mlx_linear_t l) -{ - return arr_ok(l.w) || arr_ok(l.wt); -} - -static void free_linear(mlx_linear_t *l) -{ - mlx_array_free(l->w); - mlx_array_free(l->wt); - mlx_array_free(l->scales); - mlx_array_free(l->biases); - memset(l, 0, sizeof(*l)); -} - -static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) -{ - if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt)) - return 0; - - mlx_array transposed = mlx_array_new(); - mlx_transpose(&transposed, l->w, s); - if (!arr_ok(transposed)) { - mlx_array_free(transposed); - fprintf(stderr, "mlx: failed to prepare transposed linear weight\n"); - return -1; - } - - l->wt = mlx_array_new(); - mlx_contiguous(&l->wt, transposed, false, s); - mlx_array_free(transposed); - if (!arr_ok(l->wt)) { - mlx_array_free(l->wt); - l->wt = (mlx_array){0}; - fprintf(stderr, "mlx: failed to materialize transposed linear weight\n"); - return -1; - } - mlx_array_eval(l->wt); - mlx_synchronize(s); - mlx_array_free(l->w); - l->w = (mlx_array){0}; - return 0; -} - -static int quantize_linear(mlx_linear_t *l, int bits, int group_size, - mlx_stream s) -{ - if (!l || !arr_ok(l->w) || bits == 0) return 0; - if (bits != 8) { - fprintf(stderr, "mlx: quantized weights support only 8 bits\n"); - return -1; - } - if (group_size <= 0) group_size = 64; - - mlx_optional_int gs = {.value = group_size, .has_value = true}; - mlx_optional_int qb = {.value = bits, .has_value = true}; - mlx_array null_arr = (mlx_array){0}; - mlx_vector_array q = mlx_vector_array_new(); - mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s); - if (!q.ctx || mlx_vector_array_size(q) < 2) { - mlx_vector_array_free(q); - fprintf(stderr, "mlx: quantize failed\n"); - return -1; - } - - mlx_array qw = (mlx_array){0}; - mlx_array scales = (mlx_array){0}; - mlx_array biases = (mlx_array){0}; - mlx_vector_array_get(&qw, q, 0); - mlx_vector_array_get(&scales, q, 1); - if (mlx_vector_array_size(q) >= 3) - mlx_vector_array_get(&biases, q, 2); - mlx_vector_array_free(q); - - if (!arr_ok(qw) || !arr_ok(scales)) { - mlx_array_free(qw); - mlx_array_free(scales); - mlx_array_free(biases); - fprintf(stderr, "mlx: quantize returned invalid arrays\n"); - return -1; - } - mlx_array_eval(qw); - mlx_array_eval(scales); - if (arr_ok(biases)) mlx_array_eval(biases); - mlx_synchronize(s); - - mlx_array_free(l->w); - mlx_array_free(l->wt); - l->w = qw; - l->wt = (mlx_array){0}; - l->scales = scales; - l->biases = biases; - l->quantized = 1; - l->bits = bits; - l->group_size = group_size; - return 0; -} - -/* y = x @ W^T */ -static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) -{ - if (W->quantized) { - if (!arr_ok(W->w)) return (mlx_array){0}; - mlx_array res = mlx_array_new(); - mlx_optional_int gs = {.value = W->group_size, .has_value = true}; - mlx_optional_int bits = {.value = W->bits, .has_value = true}; - mlx_array null_arr = (mlx_array){0}; - mlx_quantized_matmul(&res, x, W->w, W->scales, - arr_ok(W->biases) ? W->biases : null_arr, - true, gs, bits, "affine", s); - return res; - } - mlx_array Wt; - int free_wt = 0; - if (arr_ok(W->wt)) { - Wt = W->wt; - } else if (arr_ok(W->w)) { - Wt = mlx_array_new(); - mlx_transpose(&Wt, W->w, s); - free_wt = 1; - } else { - return (mlx_array){0}; - } - mlx_array res = mlx_array_new(); - if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && - mlx_array_dtype(x) == MLX_FLOAT32) { - /* F32 activations with BF16 weights: upcast the weight operand and run - * the matmul in F32 (so accumulation is F32). Only the weights are - * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers), - * matching the CPU/CUDA accuracy while keeping BF16 weight storage. */ - mlx_array Wf = mlx_array_new(); - mlx_astype(&Wf, Wt, MLX_FLOAT32, s); - mlx_matmul(&res, x, Wf, s); - mlx_array_free(Wf); - } else { - mlx_matmul(&res, x, Wt, s); - } - if (free_wt) mlx_array_free(Wt); - return res; -} - -const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx) -{ - return ctx ? &ctx->config : NULL; -} - -/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the - * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the - * residual stream in F32 while keeping the large matmul weights in BF16. */ -static void upcast_f32_inplace(mlx_array *a, mlx_stream s) -{ - if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return; - mlx_array f = mlx_array_new(); - mlx_astype(&f, *a, MLX_FLOAT32, s); - mlx_array_free(*a); - *a = f; -} - -/* ======================================================================== - * Load - * ======================================================================== */ - -/* We duplicate a minimal config parser here so MLX can load standalone. - * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */ - -static const char *skip_ws_m(const char *p) -{ - while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; - return p; -} - -static const char *json_find_m(const char *json, const char *key) -{ - size_t klen = strlen(key); - const char *p = json; - while ((p = strstr(p, key)) != NULL) { - if (p > json && *(p - 1) == '"') { - const char *a = p + klen; - if (*a == '"') { - a = skip_ws_m(a + 1); - if (*a == ':') return skip_ws_m(a + 1); - } - } - p += klen; - } - return NULL; -} - -static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir) -{ - char path[1024]; - int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir); - if (path_len < 0 || (size_t)path_len >= sizeof(path)) { - fprintf(stderr, "mlx: model path too long\n"); - return -1; - } - FILE *f = fopen(path, "rb"); - if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; } - if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - char *buf = (char *)malloc((size_t)sz + 1); - if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); free(buf); return -1; - } - fclose(f); - buf[sz] = '\0'; - -#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0) -#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0) - GI("hidden_size", hidden_size, 0); - GI("num_hidden_layers", n_layers, 0); - GI("num_attention_heads", n_heads, 0); - GI("num_key_value_heads", n_kv_heads, 0); - GI("head_dim", head_dim, PPLX_HEAD_DIM); - GI("intermediate_size", intermediate_size, 0); - GI("vocab_size", vocab_size, PPLX_VOCAB_SIZE); - GF("rms_norm_eps", rms_norm_eps, 1e-6f); - GF("rope_theta", rope_theta, 1000000.0f); -#undef GI -#undef GF - - cfg->q_dim = 0; - cfg->kv_dim = 0; - if (cfg->n_heads > 0 && cfg->head_dim > 0 && - cfg->n_heads <= INT_MAX / cfg->head_dim) - cfg->q_dim = cfg->n_heads * cfg->head_dim; - if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && - cfg->n_kv_heads <= INT_MAX / cfg->head_dim) - cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; - free(buf); - - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || - cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || - cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || - cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || - (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || - !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { - fprintf(stderr, "mlx: invalid config in %s " - "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", - path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, - cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); - return -1; - } - if (cfg->n_layers > PPLX_MAX_LAYERS) { - fprintf(stderr, "mlx: too many layers (%d > %d)\n", - cfg->n_layers, PPLX_MAX_LAYERS); - return -1; - } - return 0; -} - -static int mlx_model_dir_has_file(const char *model_dir, const char *rel) -{ - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) return 0; - FILE *f = fopen(path, "rb"); - if (!f) return 0; - fclose(f); - return 1; -} - -static int mlx_model_dir_has_late_projection(const char *model_dir) -{ - return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") && - mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors"); -} - -static pplx_mlx_options_t normalize_mlx_options(const pplx_mlx_options_t *opts) -{ - pplx_mlx_options_t out = {0}; - if (opts) out = *opts; - if (out.quantize_bits != 0 && - out.quantize_bits != 8) { - fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); - out.quantize_bits = -1; - } - if (out.quantize_bits && out.quantize_group_size <= 0) - out.quantize_group_size = 64; - return out; -} - -static int quantize_layer(mlx_layer_t *l, int bits, int group_size, - mlx_stream s) -{ - if (bits == 0) return 0; - return quantize_linear(&l->wq, bits, group_size, s) || - quantize_linear(&l->wk, bits, group_size, s) || - quantize_linear(&l->wv, bits, group_size, s) || - quantize_linear(&l->wo, bits, group_size, s) || - quantize_linear(&l->gate_proj, bits, group_size, s) || - quantize_linear(&l->up_proj, bits, group_size, s) || - quantize_linear(&l->down_proj, bits, group_size, s); -} - -static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) -{ - return prepare_linear_transpose(&l->wq, s) || - prepare_linear_transpose(&l->wk, s) || - prepare_linear_transpose(&l->wv, s) || - prepare_linear_transpose(&l->wo, s) || - prepare_linear_transpose(&l->gate_proj, s) || - prepare_linear_transpose(&l->up_proj, s) || - prepare_linear_transpose(&l->down_proj, s); -} - -static pplx_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, - int layer_start, int layer_end, - const pplx_mlx_options_t *opts, - int allow_late) -{ - pplx_mlx_options_t options = normalize_mlx_options(opts); - if (options.quantize_bits < 0) return NULL; - if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "mlx: late-interaction models require token-level " - "MaxSim support and are not valid pooled embedding models " - "yet\n"); - return NULL; - } - - pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); - if (!ctx) return NULL; - ctx->quantize_bits = options.quantize_bits; - ctx->quantize_group_size = options.quantize_group_size; - - if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } - - const pplx_config_t *c = &ctx->config; - if (layer_end == -1) - layer_end = c->n_layers; - if (layer_start < 0 || layer_start >= layer_end || - layer_end > c->n_layers) { - fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", - layer_start, layer_end, c->n_layers); - free(ctx); - return NULL; - } - ctx->layer_start = layer_start; - ctx->layer_end = layer_end; - - ctx->ms = multi_safetensors_open(model_dir); - if (!ctx->ms) { - fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); - free(ctx); return NULL; - } - - int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; - int inter = c->intermediate_size, hd = c->head_dim; - - /* Only the first stage needs token embeddings. */ - if (layer_start == 0) { - int emb_shape[] = {c->vocab_size, h}; - ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", - emb_shape, 2); - if (!arr_ok(ctx->embed_tokens)) goto fail; - } - - /* Layers */ - ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); - if (!ctx->layers) goto fail; - - char name[256]; - for (int i = layer_start; i < layer_end; i++) { - mlx_layer_t *l = &ctx->layers[i]; - -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) - - LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); - LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); - LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); - LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); -#undef LD - -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!arr_ok(l->fld)) goto fail; \ -} while (0) - - LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); - LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); - LD(input_norm, "layers.%d.input_layernorm.weight", h); - LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); -#undef LD - -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) - - LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); - LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); - LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); -#undef LD - - if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); - } - - if (layer_end == c->n_layers) { - int ns[] = {h}; - ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); - if (!arr_ok(ctx->norm)) goto fail; - } - - ctx->stream = mlx_default_gpu_stream_new(); - ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w); - /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the - * tiny norm vectors so they match the F32 activations in mlx_forward_layers; - * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */ - if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) { - for (int i = layer_start; i < layer_end; i++) { - mlx_layer_t *l = &ctx->layers[i]; - upcast_f32_inplace(&l->q_norm, ctx->stream); - upcast_f32_inplace(&l->k_norm, ctx->stream); - upcast_f32_inplace(&l->input_norm, ctx->stream); - upcast_f32_inplace(&l->post_attn_norm, ctx->stream); - } - upcast_f32_inplace(&ctx->norm, ctx->stream); - } - if (ctx->quantize_bits) { - for (int i = layer_start; i < layer_end; i++) { - if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, - ctx->quantize_group_size, ctx->stream) != 0) - goto fail; - } - mlx_clear_cache(); - if (pplx_verbose >= 1) - fprintf(stderr, "mlx: quantized linear weights to %d-bit " - "(group_size=%d)\n", - ctx->quantize_bits, ctx->quantize_group_size); - } else { - for (int i = layer_start; i < layer_end; i++) { - if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0) - goto fail; - } - } - - if (pplx_verbose >= 2) { - size_t active = 0, cache = 0, peak = 0; - mlx_get_active_memory(&active); - mlx_get_cache_memory(&cache); - mlx_get_peak_memory(&peak); - fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB " - "peak=%.2f GiB\n", - (double)active / (1024.0 * 1024.0 * 1024.0), - (double)cache / (1024.0 * 1024.0 * 1024.0), - (double)peak / (1024.0 * 1024.0 * 1024.0)); - } - - if (pplx_verbose >= 1) - fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", - layer_start, layer_end, h); - - return ctx; - -fail: - pplx_mlx_free(ctx); - return NULL; -} - -pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) -{ - return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); -} - -pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, - const pplx_mlx_options_t *opts) -{ - return mlx_load_range_ex(model_dir, 0, -1, opts, 0); -} - -pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, - int layer_start, int layer_end) -{ - return mlx_load_range_ex(model_dir, layer_start, layer_end, NULL, 0); -} - -pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( - const char *model_dir, int layer_start, int layer_end, - const pplx_mlx_options_t *opts) -{ - return mlx_load_range_ex(model_dir, layer_start, layer_end, opts, 0); -} - -/* ======================================================================== - * Free - * ======================================================================== */ - -static void free_mlx_layer(mlx_layer_t *l) -{ - free_linear(&l->wq); free_linear(&l->wk); - free_linear(&l->wv); free_linear(&l->wo); - mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); - mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); - free_linear(&l->gate_proj); free_linear(&l->up_proj); - free_linear(&l->down_proj); -} - -void pplx_mlx_free(pplx_mlx_ctx_t *ctx) -{ - if (!ctx) return; - mlx_array_free(ctx->embed_tokens); - if (ctx->layers) { - for (int i = 0; i < ctx->config.n_layers; i++) - free_mlx_layer(&ctx->layers[i]); - free(ctx->layers); - } - mlx_array_free(ctx->norm); - if (ctx->stream.ctx) mlx_stream_free(ctx->stream); - if (ctx->ms) multi_safetensors_close(ctx->ms); - free(ctx); -} - -pplx_mlx_late_ctx_t *pplx_mlx_late_load_with_options( - const char *model_dir, const pplx_mlx_options_t *opts) -{ - if (!mlx_model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "mlx late: missing 1_Dense projection\n"); - return NULL; - } - - pplx_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); - if (!late) return NULL; - - late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1); - if (!late->base) goto fail; - - char dense_dir[1024]; - int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir); - if (n < 0 || (size_t)n >= sizeof(dense_dir)) { - fprintf(stderr, "mlx late: model path too long\n"); - goto fail; - } - - late->dense_ms = multi_safetensors_open(dense_dir); - if (!late->dense_ms) { - fprintf(stderr, "mlx late: failed to open safetensors in %s\n", - dense_dir); - goto fail; - } - - const safetensor_t *t = multi_safetensors_find(late->dense_ms, - "linear.weight", NULL); - if (!t) { - fprintf(stderr, "mlx late: missing linear.weight\n"); - goto fail; - } - if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX || - t->shape[1] != late->base->config.hidden_size) { - fprintf(stderr, "mlx late: bad projection shape\n"); - goto fail; - } - - int shape[] = { (int)t->shape[0], late->base->config.hidden_size }; - late->projection = load_linear_tensor(late->dense_ms, "linear.weight", - shape, 2); - if (!linear_ok(late->projection)) goto fail; - if (prepare_linear_transpose(&late->projection, late->base->stream) != 0) - goto fail; - late->token_dim = shape[0]; - - if (pplx_verbose >= 1) - fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", - late->token_dim); - return late; - -fail: - pplx_mlx_late_free(late); - return NULL; -} - -pplx_mlx_late_ctx_t *pplx_mlx_late_load(const char *model_dir) -{ - return pplx_mlx_late_load_with_options(model_dir, NULL); -} - -void pplx_mlx_late_free(pplx_mlx_late_ctx_t *ctx) -{ - if (!ctx) return; - free_linear(&ctx->projection); - if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms); - pplx_mlx_free(ctx->base); - free(ctx); -} - -const pplx_config_t *pplx_mlx_late_config(const pplx_mlx_late_ctx_t *ctx) -{ - return ctx && ctx->base ? &ctx->base->config : NULL; -} - -int pplx_mlx_late_token_dim(const pplx_mlx_late_ctx_t *ctx) -{ - return ctx ? ctx->token_dim : 0; -} - -/* ======================================================================== - * Forward pass - * ======================================================================== */ - -static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, - int batch, int max_seq, int has_padding, - mlx_array attn_mask, - int layer_start, int layer_end) -{ - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - int n_heads = c->n_heads; - int n_kv_heads = c->n_kv_heads; - int head_dim = c->head_dim; - int q_dim = c->q_dim; - float scale = 1.0f / sqrtf((float)head_dim); - mlx_array null_arr = (mlx_array){0}; - - /* For non-quantized BF16 weights, run the residual stream, norms, and - * attention in F32; matmuls downcast their operands to BF16 internally - * (see linear). This keeps weight storage at BF16 while avoiding the - * cross-layer accumulation error of a fully BF16 forward. F32 and quantized - * models are unaffected. */ - mlx_array local_mask = attn_mask; - int free_mask = 0; - int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 && - !ctx->layers[layer_start].wq.quantized; - if (mixed_bf16) { - if (mlx_array_dtype(x) != MLX_FLOAT32) { - mlx_array xf = mlx_array_new(); - mlx_astype(&xf, x, MLX_FLOAT32, S); - mlx_array_free(x); - x = xf; - } - if (has_padding && arr_ok(attn_mask) && - mlx_array_dtype(attn_mask) != MLX_FLOAT32) { - local_mask = mlx_array_new(); - mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S); - free_mask = 1; - } - } - - for (int layer = layer_start; layer < layer_end; layer++) { - mlx_layer_t *l = &ctx->layers[layer]; - - mlx_array xn = mlx_array_new(); - mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); - - mlx_array q_flat = linear(xn, &l->wq, S); - mlx_array k_flat = linear(xn, &l->wk, S); - mlx_array v_flat = linear(xn, &l->wv, S); - mlx_array_free(xn); - - int q_shape[] = {batch, max_seq, n_heads, head_dim}; - int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; - mlx_array q = mlx_array_new(); - mlx_array k = mlx_array_new(); - mlx_array v = mlx_array_new(); - mlx_reshape(&q, q_flat, q_shape, 4, S); - mlx_reshape(&k, k_flat, k_shape, 4, S); - mlx_reshape(&v, v_flat, k_shape, 4, S); - mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); - - mlx_array qn = mlx_array_new(); - mlx_array kn = mlx_array_new(); - mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); - mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); - mlx_array_free(q); mlx_array_free(k); - - /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ - int perm[] = {0, 2, 1, 3}; - mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); - mlx_transpose_axes(&qt, qn, perm, 4, S); - mlx_transpose_axes(&kt, kn, perm, 4, S); - mlx_transpose_axes(&vt, v, perm, 4, S); - mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v); - - mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; - mlx_array qr = mlx_array_new(), kr = mlx_array_new(); - mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); - mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); - mlx_array_free(qt); mlx_array_free(kt); - - mlx_array attn = mlx_array_new(); - mlx_fast_scaled_dot_product_attention( - &attn, qr, kr, vt, scale, "", - has_padding ? local_mask : null_arr, null_arr, S); - mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); - - mlx_array attn_t = mlx_array_new(); - mlx_transpose_axes(&attn_t, attn, perm, 4, S); - mlx_array_free(attn); - - int flat_shape[] = {batch, max_seq, q_dim}; - mlx_array attn_flat = mlx_array_new(); - mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S); - mlx_array_free(attn_t); - - mlx_array proj = linear(attn_flat, &l->wo, S); - mlx_array_free(attn_flat); - mlx_array x2 = mlx_array_new(); - mlx_add(&x2, x, proj, S); - mlx_array_free(x); mlx_array_free(proj); - x = x2; - - mlx_array xn2 = mlx_array_new(); - mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); - - mlx_array gate = linear(xn2, &l->gate_proj, S); - mlx_array up = linear(xn2, &l->up_proj, S); - mlx_array_free(xn2); - - mlx_array gate_sig = mlx_array_new(); - mlx_sigmoid(&gate_sig, gate, S); - mlx_array silu_gate = mlx_array_new(); - mlx_multiply(&silu_gate, gate, gate_sig, S); - mlx_array_free(gate); mlx_array_free(gate_sig); - - mlx_array mid = mlx_array_new(); - mlx_multiply(&mid, silu_gate, up, S); - mlx_array_free(silu_gate); mlx_array_free(up); - - mlx_array ffn = linear(mid, &l->down_proj, S); - mlx_array_free(mid); - - mlx_array x3 = mlx_array_new(); - mlx_add(&x3, x, ffn, S); - mlx_array_free(x); mlx_array_free(ffn); - x = x3; - } - if (free_mask) mlx_array_free(local_mask); - return x; -} - -pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device( - pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, - int normalize) -{ - if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || - ctx->token_dim <= 0 || !linear_ok(ctx->projection)) - return NULL; - - pplx_mlx_ctx_t *base = ctx->base; - if (base->layer_start != 0 || base->layer_end != base->config.n_layers || - !arr_ok(base->embed_tokens) || !arr_ok(base->norm)) - return NULL; - - const pplx_config_t *c = &base->config; - mlx_stream S = base->stream; - for (int i = 0; i < n_tokens; i++) { - if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { - fprintf(stderr, "mlx late: invalid token id %d at position %d\n", - token_ids[i], i); - return NULL; - } - } - - size_t id_bytes; - if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0) - return NULL; - - int *ids_data = (int *)malloc(id_bytes); - if (!ids_data) return NULL; - memcpy(ids_data, token_ids, id_bytes); - - int ids_shape[] = {1, n_tokens}; - mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32); - free(ids_data); - if (!arr_ok(ids)) return NULL; - - pplx_mlx_late_vectors_t *result = NULL; - mlx_array x = mlx_array_new(); - mlx_array x_normed = (mlx_array){0}; - mlx_array projected = (mlx_array){0}; - mlx_array reshaped = (mlx_array){0}; - - mlx_take_axis(&x, base->embed_tokens, ids, 0, S); - mlx_array_free(ids); - if (!arr_ok(x)) goto cleanup; - - x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, - 0, c->n_layers); - if (!arr_ok(x)) goto cleanup; - - x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S); - mlx_array_free(x); - x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; - - projected = linear(x_normed, &ctx->projection, S); - mlx_array_free(x_normed); - x_normed = (mlx_array){0}; - if (!arr_ok(projected)) goto cleanup; - - if (normalize) { - mlx_array sq = mlx_array_new(); - mlx_square(&sq, projected, S); - if (!arr_ok(sq)) { - mlx_array_free(sq); - goto cleanup; - } - - mlx_array norm2 = mlx_array_new(); - mlx_sum_axis(&norm2, sq, 2, true, S); - mlx_array_free(sq); - if (!arr_ok(norm2)) { - mlx_array_free(norm2); - goto cleanup; - } - - mlx_array inv_norm = mlx_array_new(); - mlx_rsqrt(&inv_norm, norm2, S); - mlx_array_free(norm2); - if (!arr_ok(inv_norm)) { - mlx_array_free(inv_norm); - goto cleanup; - } - - mlx_array normalized = mlx_array_new(); - mlx_multiply(&normalized, projected, inv_norm, S); - mlx_array_free(inv_norm); - mlx_array_free(projected); - projected = normalized; - if (!arr_ok(projected)) goto cleanup; - } - - int vec_shape[] = {n_tokens, ctx->token_dim}; - reshaped = mlx_array_new(); - mlx_reshape(&reshaped, projected, vec_shape, 2, S); - if (!arr_ok(reshaped)) goto cleanup; - - result = (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*result)); - if (!result) goto cleanup; - result->owner = ctx; - result->vectors = reshaped; - result->tokens = n_tokens; - result->dim = ctx->token_dim; - reshaped = (mlx_array){0}; - -cleanup: - mlx_array_free(reshaped); - mlx_array_free(projected); - mlx_array_free(x_normed); - mlx_array_free(x); - return result; -} - -void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs) -{ - if (!vecs) return; - mlx_array_free(vecs->vectors); - free(vecs); -} - -int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs) -{ - return vecs ? vecs->tokens : 0; -} - -int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs) -{ - return vecs ? vecs->dim : 0; -} - -int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs, - float *out_vectors) -{ - if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || - vecs->tokens <= 0 || vecs->dim <= 0) - return -1; - - size_t values, bytes; - if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 || - mlx_mul_size(values, sizeof(float), &bytes) != 0) - return -1; - - mlx_stream S = vecs->owner->base->stream; - mlx_array out_f32 = mlx_array_new(); - mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S); - if (!arr_ok(out_f32)) { - mlx_array_free(out_f32); - return -1; - } - - int rc = -1; - mlx_array_eval(out_f32); - const float *data = mlx_array_data_float32(out_f32); - if (data) { - memcpy(out_vectors, data, bytes); - rc = 0; - } - mlx_array_free(out_f32); - return rc; -} - -pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat( - pplx_mlx_late_ctx_t *ctx, - const pplx_mlx_late_vectors_t *const *items, int count) -{ - if (!ctx || !ctx->base || !items || count <= 0) return NULL; - int dim = 0; - int total = 0; - for (int i = 0; i < count; i++) { - const pplx_mlx_late_vectors_t *v = items[i]; - if (!v || v->owner != ctx || !arr_ok(v->vectors) || - v->tokens <= 0 || v->dim <= 0) - return NULL; - if (i == 0) dim = v->dim; - if (v->dim != dim || total > INT_MAX - v->tokens) - return NULL; - total += v->tokens; - } - - mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts)); - if (!parts) return NULL; - for (int i = 0; i < count; i++) - parts[i] = items[i]->vectors; - - mlx_stream S = ctx->base->stream; - mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count); - free(parts); - if (!vec.ctx) return NULL; - - mlx_array joined = mlx_array_new(); - mlx_concatenate_axis(&joined, vec, 0, S); - mlx_vector_array_free(vec); - if (!arr_ok(joined)) { - mlx_array_free(joined); - return NULL; - } - - pplx_mlx_late_vectors_t *out = - (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out)); - if (!out) { - mlx_array_free(joined); - return NULL; - } - out->owner = ctx; - out->vectors = joined; - out->tokens = total; - out->dim = dim; - return out; -} - -pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select( - pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs, - const int *token_indices, int count) -{ - if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || - !arr_ok(vecs->vectors) || !token_indices || count <= 0 || - vecs->tokens <= 0 || vecs->dim <= 0) - return NULL; - - for (int i = 0; i < count; i++) { - if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens) - return NULL; - } - - size_t idx_bytes; - if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0) - return NULL; - int *idx_data = (int *)malloc(idx_bytes); - if (!idx_data) return NULL; - memcpy(idx_data, token_indices, idx_bytes); - - int idx_shape[] = {count}; - mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32); - free(idx_data); - if (!arr_ok(idx)) return NULL; - - mlx_stream S = ctx->base->stream; - mlx_array selected = mlx_array_new(); - mlx_take_axis(&selected, vecs->vectors, idx, 0, S); - mlx_array_free(idx); - if (!arr_ok(selected)) { - mlx_array_free(selected); - return NULL; - } - - pplx_mlx_late_vectors_t *out = - (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out)); - if (!out) { - mlx_array_free(selected); - return NULL; - } - out->owner = ctx; - out->vectors = selected; - out->tokens = count; - out->dim = vecs->dim; - return out; -} - -int pplx_mlx_late_maxsim_batch_device( - pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query, - const pplx_mlx_late_vectors_t *docs, const int *doc_offsets, - int docs_count, float *scores) -{ - if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || - !scores || query->owner != ctx || docs->owner != ctx || - query->dim <= 0 || query->dim != docs->dim || - query->tokens <= 0 || docs->tokens <= 0 || - !arr_ok(query->vectors) || !arr_ok(docs->vectors) || - doc_offsets[0] != 0) - return -1; - - int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths)); - if (!lengths) return -1; - int max_doc_tokens = 0; - for (int i = 0; i < docs_count; i++) { - if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] || - doc_offsets[i + 1] > docs->tokens) { - free(lengths); - return -1; - } - int n = doc_offsets[i + 1] - doc_offsets[i]; - lengths[i] = n; - if (n > max_doc_tokens) max_doc_tokens = n; - } - if (max_doc_tokens <= 0) { - free(lengths); - return -1; - } - - mlx_stream S = ctx->base->stream; - int dim = query->dim; - int rc = -1; - mlx_array *doc_rows = - (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); - if (!doc_rows) { - free(lengths); - return -1; - } - - mlx_vector_array doc_vec = (mlx_vector_array){0}; - mlx_array pad_value = mlx_array_new_float32(0.0f); - mlx_array doc_batch = (mlx_array){0}; - mlx_array query_3d = (mlx_array){0}; - mlx_array query_batched = (mlx_array){0}; - mlx_array docs_t = (mlx_array){0}; - mlx_array sims = (mlx_array){0}; - mlx_array lengths_arr = (mlx_array){0}; - mlx_array positions = (mlx_array){0}; - mlx_array lengths_2d = (mlx_array){0}; - mlx_array positions_2d = (mlx_array){0}; - mlx_array valid = (mlx_array){0}; - mlx_array valid_3d = (mlx_array){0}; - mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX); - mlx_array masked_sims = (mlx_array){0}; - mlx_array maxes = (mlx_array){0}; - mlx_array sums = (mlx_array){0}; - mlx_array sums_f32 = (mlx_array){0}; - - for (int i = 0; i < docs_count; i++) { - int n = lengths[i]; - int start[] = {doc_offsets[i], 0}; - int stop[] = {doc_offsets[i + 1], dim}; - int strides[] = {1, 1}; - - mlx_array doc = mlx_array_new(); - mlx_array padded = (mlx_array){0}; - - mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S); - if (!arr_ok(doc)) goto cleanup; - - if (n < max_doc_tokens) { - int axes[] = {0}; - int low[] = {0}; - int high[] = {max_doc_tokens - n}; - padded = mlx_array_new(); - mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, - "constant", S); - mlx_array_free(doc); - doc = padded; - padded = (mlx_array){0}; - if (!arr_ok(doc)) goto cleanup; - } - - doc_rows[i] = mlx_array_new(); - mlx_expand_dims(&doc_rows[i], doc, 0, S); - mlx_array_free(doc); - if (!arr_ok(doc_rows[i])) goto cleanup; - } - - doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count); - if (!doc_vec.ctx) goto cleanup; - - doc_batch = mlx_array_new(); - mlx_concatenate_axis(&doc_batch, doc_vec, 0, S); - if (!arr_ok(doc_batch)) goto cleanup; - - query_3d = mlx_array_new(); - mlx_expand_dims(&query_3d, query->vectors, 0, S); - if (!arr_ok(query_3d)) goto cleanup; - - int q_shape[] = {docs_count, query->tokens, dim}; - query_batched = mlx_array_new(); - mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S); - if (!arr_ok(query_batched)) goto cleanup; - - int perm[] = {0, 2, 1}; - docs_t = mlx_array_new(); - mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S); - if (!arr_ok(docs_t)) goto cleanup; - - sims = mlx_array_new(); - mlx_matmul(&sims, query_batched, docs_t, S); - if (!arr_ok(sims)) goto cleanup; - - int len_shape[] = {docs_count}; - lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32); - if (!arr_ok(lengths_arr)) goto cleanup; - - positions = mlx_array_new(); - mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S); - if (!arr_ok(positions)) goto cleanup; - - lengths_2d = mlx_array_new(); - mlx_expand_dims(&lengths_2d, lengths_arr, 1, S); - if (!arr_ok(lengths_2d)) goto cleanup; - - positions_2d = mlx_array_new(); - mlx_expand_dims(&positions_2d, positions, 0, S); - if (!arr_ok(positions_2d)) goto cleanup; - - valid = mlx_array_new(); - mlx_less(&valid, positions_2d, lengths_2d, S); - if (!arr_ok(valid)) goto cleanup; - - valid_3d = mlx_array_new(); - mlx_expand_dims(&valid_3d, valid, 1, S); - if (!arr_ok(valid_3d)) goto cleanup; - - masked_sims = mlx_array_new(); - mlx_where(&masked_sims, valid_3d, sims, invalid_value, S); - if (!arr_ok(masked_sims)) goto cleanup; - - maxes = mlx_array_new(); - mlx_max_axis(&maxes, masked_sims, 2, false, S); - if (!arr_ok(maxes)) goto cleanup; - - sums = mlx_array_new(); - mlx_sum_axis(&sums, maxes, 1, false, S); - if (!arr_ok(sums)) goto cleanup; - - sums_f32 = mlx_array_new(); - mlx_astype(&sums_f32, sums, MLX_FLOAT32, S); - if (!arr_ok(sums_f32)) goto cleanup; - - mlx_array_eval(sums_f32); - const float *data = mlx_array_data_float32(sums_f32); - if (!data) goto cleanup; - memcpy(scores, data, (size_t)docs_count * sizeof(float)); - rc = 0; - -cleanup: - mlx_array_free(sums_f32); - mlx_array_free(sums); - mlx_array_free(maxes); - mlx_array_free(masked_sims); - mlx_array_free(invalid_value); - mlx_array_free(valid_3d); - mlx_array_free(valid); - mlx_array_free(positions_2d); - mlx_array_free(lengths_2d); - mlx_array_free(positions); - mlx_array_free(lengths_arr); - mlx_array_free(sims); - mlx_array_free(docs_t); - mlx_array_free(query_batched); - mlx_array_free(query_3d); - mlx_array_free(doc_batch); - if (doc_vec.ctx) mlx_vector_array_free(doc_vec); - mlx_array_free(pad_value); - for (int i = 0; i < docs_count; i++) - mlx_array_free(doc_rows[i]); - free(doc_rows); - free(lengths); - return rc; -} - -int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors) -{ - pplx_mlx_late_vectors_t *vecs = - pplx_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, - normalize); - if (!vecs) return -1; - int rc = pplx_mlx_late_vectors_copy(vecs, out_vectors); - pplx_mlx_late_vectors_free(vecs); - return rc; -} - -static int pplx_mlx_embed_batch_dense(pplx_mlx_ctx_t *ctx, - const pplx_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) - return -1; - - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - - int hidden = c->hidden_size; - int max_seq = 0; - int has_padding = 0; - - for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - } - if (max_seq <= 0) return -1; - - for (int b = 0; b < batch; b++) { - if (inputs[b].n_tokens != max_seq) has_padding = 1; - } - - size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 || - mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 || - mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 || - mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || - mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || - mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) - return -1; - int *padded_ids = (int *)malloc(ids_bytes); - float *pool_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; - float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; - float *len_data = has_padding ? (float *)malloc(len_bytes) : NULL; - if (!padded_ids || (has_padding && (!pool_mask_data || !attn_mask_data || !len_data))) { - free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); - return -1; - } - - for (int b = 0; b < batch; b++) { - if (has_padding) len_data[b] = (float)inputs[b].n_tokens; - for (int t = 0; t < max_seq; t++) { - size_t idx = (size_t)b * max_seq + t; - if (t < inputs[b].n_tokens) { - int id = inputs[b].ids[t]; - if (id < 0 || id >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - id, b, t); - free(padded_ids); free(pool_mask_data); - free(attn_mask_data); free(len_data); - return -1; - } - padded_ids[idx] = id; - if (has_padding) { - pool_mask_data[idx] = 1.0f; - attn_mask_data[idx] = 0.0f; - } - } else { - padded_ids[idx] = 0; - pool_mask_data[idx] = 0.0f; - attn_mask_data[idx] = -1.0e9f; - } - } - } - - int ids_shape[] = {batch, max_seq}; - int pool_mask_shape[] = {batch, max_seq, 1}; - int attn_mask_shape[] = {batch, 1, 1, max_seq}; - int len_shape[] = {batch, 1}; - - mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - mlx_array pool_mask = has_padding - ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array lengths = has_padding - ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) - : (mlx_array){0}; - free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); - - if (!arr_ok(ids) || (has_padding && (!arr_ok(pool_mask) || - !arr_ok(attn_mask) || !arr_ok(lengths)))) { - mlx_array_free(ids); mlx_array_free(pool_mask); - if (has_padding) mlx_array_free(attn_mask); - mlx_array_free(lengths); - return -1; - } - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - mlx_array attn_mask_bf16 = mlx_array_new(); - mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); - mlx_array_free(attn_mask); - attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) { - mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths); - return -1; - } - } - - /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */ - mlx_array x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); - mlx_array_free(ids); - - /* 2. Transformer layers. */ - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - 0, c->n_layers); - - /* 3. Final RMSNorm. */ - mlx_array x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); - mlx_array_free(x); - - mlx_array emb = mlx_array_new(); - if (has_padding) { - /* 4. Masked mean pool over T. */ - mlx_array masked = mlx_array_new(); - mlx_multiply(&masked, x_normed, pool_mask, S); - mlx_array_free(x_normed); - mlx_array_free(pool_mask); - - mlx_array emb_sum = mlx_array_new(); - mlx_sum_axis(&emb_sum, masked, 1, false, S); - mlx_array_free(masked); - - mlx_divide(&emb, emb_sum, lengths, S); - mlx_array_free(emb_sum); mlx_array_free(lengths); - mlx_array_free(attn_mask); - } else { - /* Equal-length batches do not need masks or a separate divide. */ - mlx_mean_axis(&emb, x_normed, 1, false, S); - mlx_array_free(x_normed); - } - - /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are - * intentionally unnormalized; callers should use cosine similarity. */ - mlx_array emb_f32 = mlx_array_new(); - mlx_astype(&emb_f32, emb, MLX_FLOAT32, S); - mlx_array_free(emb); - - mlx_array_eval(emb_f32); - const float *data = mlx_array_data_float32(emb_f32); - int rc = -1; - if (data) { - memcpy(out_embeddings, data, out_bytes); - rc = 0; - } - mlx_array_free(emb_f32); - return rc; -} - -typedef struct { - int index; - const pplx_input_t *input; -} mlx_batch_item_t; - -static int mlx_batch_item_cmp(const void *a, const void *b) -{ - const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a; - const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b; - int da = ia->input->n_tokens; - int db = ib->input->n_tokens; - return (da > db) - (da < db); -} - -static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) -{ - if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5) - return 1; - return padded * 5 > tokens * 11; -} - -static int mlx_batch_should_split(const pplx_input_t *inputs, int batch, - int *out_total_tokens, int *out_max_seq) -{ - int max_seq = 0; - int total_tokens = 0; - for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0; - total_tokens += inputs[b].n_tokens; - } - if (out_total_tokens) *out_total_tokens = total_tokens; - if (out_max_seq) *out_max_seq = max_seq; - if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0; - - size_t padded; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) - return 0; - - return mlx_padding_exceeds_limit(padded, (size_t)total_tokens); -} - -static int mlx_should_add_to_group(const pplx_mlx_ctx_t *ctx, - int group_count, int group_max, - int next_tokens) -{ - if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0) - return 1; - - int next_count = group_count + 1; - int next_max = next_tokens > group_max ? next_tokens : group_max; - size_t merged, split_left, split_total; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, &merged) != 0 || - mlx_mul_size((size_t)group_count, (size_t)group_max, &split_left) != 0) - return 0; - /* Treat one extra MLX launch as roughly 256 padded token rows for 4B. - * This preserves small mixed groups while splitting larger padded tails. */ - if (split_left > SIZE_MAX - (size_t)next_tokens || - split_left + (size_t)next_tokens > SIZE_MAX - 256) - return 0; - split_total = split_left + (size_t)next_tokens + 256; - return merged <= split_total; -} - -int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) - return -1; - - if (!mlx_batch_should_split(inputs, batch, NULL, NULL)) - return pplx_mlx_embed_batch_dense(ctx, inputs, batch, out_embeddings); - - const int hidden = ctx->config.hidden_size; - mlx_batch_item_t *items = (mlx_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - pplx_input_t *group_inputs = (pplx_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); - size_t group_values, group_bytes; - if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 || - mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); - return -1; - } - float *group_out = (float *)malloc(group_bytes); - if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); - return -1; - } - for (int i = 0; i < batch; i++) { - items[i].index = i; - items[i].input = &inputs[i]; - } - qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp); - - int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { - int group_max = 0; - int group_tokens = 0; - int end = start; - while (end < batch) { - int next_tokens = items[end].input->n_tokens; - int next_count = end - start + 1; - int next_max = next_tokens > group_max ? next_tokens : group_max; - int next_total = group_tokens + next_tokens; - if (end > start) { - size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) - break; - } - group_max = next_max; - group_tokens = next_total; - end++; - } - - int group_count = end - start; - for (int i = 0; i < group_count; i++) - group_inputs[i] = *items[start + i].input; - rc = pplx_mlx_embed_batch_dense(ctx, group_inputs, group_count, - group_out); - if (rc != 0) break; - for (int i = 0; i < group_count; i++) { - int dst = items[start + i].index; - memcpy(out_embeddings + (size_t)dst * hidden, - group_out + (size_t)i * hidden, - (size_t)hidden * sizeof(float)); - } - start = end; - } - - free(items); free(group_inputs); free(group_out); - return rc; -} - -int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding) -{ - pplx_input_t input = { token_ids, n_tokens }; - return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding); -} - -int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states) -{ - if (!ctx || !inputs || batch <= 0 || !out_states || - layer_start < 0 || layer_start > layer_end || - layer_end > ctx->config.n_layers || - layer_start < ctx->layer_start || layer_end > ctx->layer_end || - (layer_start == 0 && input_states) || - (layer_start != 0 && !input_states) || - (layer_start == 0 && !arr_ok(ctx->embed_tokens)) || - (apply_final_norm && - (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm)))) - return -1; - - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int max_seq = 0; - int total_seq = 0; - int has_padding = 0; - for (int b = 0; b < batch; b++) { - if ((layer_start == 0 && !inputs[b].ids) || inputs[b].n_tokens <= 0 || - total_seq > INT_MAX - inputs[b].n_tokens) - return -1; - total_seq += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - } - for (int b = 0; b < batch; b++) { - if (inputs[b].n_tokens != max_seq) has_padding = 1; - } - - size_t rows, ids_bytes, mask_bytes, state_bytes, row_bytes; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || - mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || - mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || - mlx_mul_size((size_t)hidden, sizeof(float), &row_bytes) != 0) - return -1; - if (rows == 0 || row_bytes == 0 || (has_padding && mask_bytes == 0)) - return -1; - if (layer_start != 0 && - mlx_mul_size(rows, (size_t)hidden, &state_bytes) != 0) - return -1; - if (layer_start == 0) - state_bytes = 0; - - int *padded_ids = layer_start == 0 - ? (int *)malloc(ids_bytes) : NULL; - float *padded_states = layer_start != 0 - ? (float *)calloc(1, state_bytes) : NULL; - float *attn_mask_data = has_padding - ? (float *)malloc(mask_bytes) : NULL; - if ((layer_start == 0 && !padded_ids) || - (layer_start != 0 && !padded_states) || - (has_padding && !attn_mask_data)) { - free(padded_ids); free(padded_states); free(attn_mask_data); - return -1; - } - - int packed_offset = 0; - for (int b = 0; b < batch; b++) { - for (int t = 0; t < max_seq; t++) { - size_t dense_row = (size_t)b * max_seq + t; - if (t < inputs[b].n_tokens) { - if (layer_start == 0) { - int id = inputs[b].ids[t]; - if (id < 0 || id >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - id, b, t); - free(padded_ids); free(padded_states); - free(attn_mask_data); - return -1; - } - padded_ids[dense_row] = id; - } else { - memcpy(padded_states + dense_row * hidden, - input_states + (size_t)(packed_offset + t) * hidden, - row_bytes); - } - if (has_padding) attn_mask_data[dense_row] = 0.0f; - } else { - if (layer_start == 0) padded_ids[dense_row] = 0; - if (has_padding) attn_mask_data[dense_row] = -1.0e9f; - } - } - packed_offset += inputs[b].n_tokens; - } - - int attn_mask_shape[] = {batch, 1, 1, max_seq}; - mlx_array attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - free(attn_mask_data); - if (has_padding && !arr_ok(attn_mask)) { - free(padded_ids); free(padded_states); - return -1; - } - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - mlx_array attn_mask_bf16 = mlx_array_new(); - mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); - mlx_array_free(attn_mask); - attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) { - free(padded_ids); free(padded_states); - return -1; - } - } - - mlx_array x; - if (layer_start == 0) { - int ids_shape[] = {batch, max_seq}; - mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - free(padded_ids); - if (!arr_ok(ids)) { - if (has_padding) mlx_array_free(attn_mask); - return -1; - } - x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); - mlx_array_free(ids); - } else { - int states_shape[] = {batch, max_seq, hidden}; - mlx_array states = mlx_array_new_data(padded_states, states_shape, 3, - MLX_FLOAT32); - free(padded_states); - if (!arr_ok(states)) { - if (has_padding) mlx_array_free(attn_mask); - return -1; - } - if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - x = mlx_array_new(); - mlx_astype(&x, states, MLX_BFLOAT16, S); - mlx_array_free(states); - } else { - x = states; - } - } - - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - layer_start, layer_end); - if (has_padding) mlx_array_free(attn_mask); - - if (apply_final_norm) { - mlx_array x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); - mlx_array_free(x); - x = x_normed; - } - - mlx_array x_f32 = mlx_array_new(); - mlx_astype(&x_f32, x, MLX_FLOAT32, S); - mlx_array_free(x); - mlx_array_eval(x_f32); - - const float *data = mlx_array_data_float32(x_f32); - if (!data) { - mlx_array_free(x_f32); - return -1; - } - packed_offset = 0; - for (int b = 0; b < batch; b++) { - memcpy(out_states + (size_t)packed_offset * hidden, - data + (size_t)b * max_seq * hidden, - (size_t)inputs[b].n_tokens * row_bytes); - packed_offset += inputs[b].n_tokens; - } - mlx_array_free(x_f32); - return 0; -} - -typedef struct { - int index; - int span_offset; - const pplx_context_input_t *input; -} mlx_context_batch_item_t; - -static int mlx_context_batch_item_cmp(const void *a, const void *b) -{ - const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a; - const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b; - int da = ia->input->input.n_tokens; - int db = ib->input->input.n_tokens; - return (da > db) - (da < db); -} - -static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) -{ - if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4) - return 1; - return padded * 4 > tokens * 5; -} - -static int mlx_context_batch_should_split(const pplx_context_input_t *inputs, - int batch, int *out_total_spans) -{ - int max_seq = 0; - int total_tokens = 0; - int total_spans = 0; - for (int b = 0; b < batch; b++) { - int n_tokens = inputs[b].input.n_tokens; - if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || - inputs[b].n_spans <= 0) - return 0; - if (n_tokens > max_seq) max_seq = n_tokens; - if (total_tokens > INT_MAX - n_tokens || - total_spans > INT_MAX - inputs[b].n_spans) - return 0; - total_tokens += n_tokens; - total_spans += inputs[b].n_spans; - } - if (out_total_spans) *out_total_spans = total_spans; - if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0; - - size_t padded; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) - return 0; - return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens); -} - -static int pplx_mlx_embed_spans_batch_dense(pplx_mlx_ctx_t *ctx, - const pplx_context_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || - ctx->layer_end != ctx->config.n_layers || - !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) - return -1; - - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int max_seq = 0; - int total_spans = 0; - int has_padding = 0; - for (int b = 0; b < batch; b++) { - const pplx_context_input_t *input = &inputs[b]; - int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans) - return -1; - total_spans += input->n_spans; - if (n_tokens > max_seq) max_seq = n_tokens; - for (int t = 0; t < n_tokens; t++) { - if (input->input.ids[t] < 0 || - input->input.ids[t] >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - input->input.ids[t], b, t); - return -1; - } - } - for (int s = 0; s < input->n_spans; s++) { - if (input->spans[s].start < 0 || - input->spans[s].start > n_tokens || - input->spans[s].n_tokens <= 0 || - input->spans[s].n_tokens > n_tokens - input->spans[s].start) - return -1; - } - } - for (int b = 0; b < batch; b++) - if (inputs[b].input.n_tokens != max_seq) has_padding = 1; - - size_t rows, ids_bytes, mask_bytes, index_bytes; - size_t span_float_bytes, out_values, out_bytes; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || - mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || - mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || - mlx_mul_size((size_t)total_spans, sizeof(int), &index_bytes) != 0 || - mlx_mul_size((size_t)total_spans, sizeof(float), &span_float_bytes) != 0 || - mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 || - mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) - return -1; - if (rows > INT_MAX) return -1; - int *padded_ids = (int *)malloc(ids_bytes); - float *attn_mask_data = has_padding - ? (float *)malloc(mask_bytes) : NULL; - if (!padded_ids || (has_padding && !attn_mask_data)) { - free(padded_ids); - free(attn_mask_data); - return -1; - } - for (int b = 0; b < batch; b++) { - const pplx_input_t *input = &inputs[b].input; - for (int t = 0; t < max_seq; t++) { - size_t row = (size_t)b * max_seq + t; - if (t < input->n_tokens) { - padded_ids[row] = input->ids[t]; - if (has_padding) attn_mask_data[row] = 0.0f; - } else { - padded_ids[row] = 0; - if (has_padding) attn_mask_data[row] = -1.0e9f; - } - } - } - - mlx_array ids = (mlx_array){0}; - mlx_array attn_mask = (mlx_array){0}; - mlx_array x = (mlx_array){0}; - mlx_array x_normed = (mlx_array){0}; - mlx_array prefix = (mlx_array){0}; - mlx_array flat_prefix = (mlx_array){0}; - mlx_array end_idx = (mlx_array){0}; - mlx_array start_idx = (mlx_array){0}; - mlx_array lengths = (mlx_array){0}; - mlx_array start_mask = (mlx_array){0}; - mlx_array end_values = (mlx_array){0}; - mlx_array start_values = (mlx_array){0}; - mlx_array masked_start = (mlx_array){0}; - mlx_array span_sums = (mlx_array){0}; - mlx_array embeddings = (mlx_array){0}; - mlx_array embeddings_f32 = (mlx_array){0}; - mlx_array *pooled = NULL; - mlx_vector_array pooled_vec = (mlx_vector_array){0}; - int *end_idx_data = NULL; - int *start_idx_data = NULL; - float *lengths_data = NULL; - float *start_mask_data = NULL; - int rc = -1; - - int ids_shape[] = {batch, max_seq}; - int attn_mask_shape[] = {batch, 1, 1, max_seq}; - ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - free(padded_ids); - free(attn_mask_data); - if (!arr_ok(ids)) goto cleanup; - if (has_padding && !arr_ok(attn_mask)) goto cleanup; - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - mlx_array attn_mask_bf16 = mlx_array_new(); - mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); - mlx_array_free(attn_mask); - attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) goto cleanup; - } - - x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); - mlx_array_free(ids); - ids = (mlx_array){0}; - if (!arr_ok(x)) goto cleanup; - - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - 0, c->n_layers); - if (has_padding) { - mlx_array_free(attn_mask); - attn_mask = (mlx_array){0}; - } - if (!arr_ok(x)) goto cleanup; - - x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); - mlx_array_free(x); - x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; - - if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled)); - if (!pooled) goto cleanup; - - int pooled_count = 0; - for (int b = 0; b < batch; b++) { - for (int s = 0; s < inputs[b].n_spans; s++) { - const pplx_span_t *span = &inputs[b].spans[s]; - int start[] = {b, span->start, 0}; - int stop[] = {b + 1, span->start + span->n_tokens, hidden}; - int strides[] = {1, 1, 1}; - mlx_array slice = mlx_array_new(); - mlx_slice(&slice, x_normed, start, 3, stop, 3, strides, 3, S); - if (!arr_ok(slice)) { - mlx_array_free(slice); - goto cleanup; - } - - pooled[pooled_count] = mlx_array_new(); - mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S); - mlx_array_free(slice); - if (!arr_ok(pooled[pooled_count])) goto cleanup; - pooled_count++; - } - } - - pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans); - if (!pooled_vec.ctx) goto cleanup; - embeddings = mlx_array_new(); - mlx_concatenate_axis(&embeddings, pooled_vec, 0, S); - if (!arr_ok(embeddings)) goto cleanup; - } else { - end_idx_data = (int *)malloc(index_bytes); - start_idx_data = (int *)malloc(index_bytes); - lengths_data = (float *)malloc(span_float_bytes); - start_mask_data = (float *)malloc(span_float_bytes); - if (!end_idx_data || !start_idx_data || !lengths_data || - !start_mask_data) - goto cleanup; - - int span_count = 0; - for (int b = 0; b < batch; b++) { - for (int s = 0; s < inputs[b].n_spans; s++) { - const pplx_span_t *span = &inputs[b].spans[s]; - int base = b * max_seq; - int start = span->start; - int stop = span->start + span->n_tokens; - end_idx_data[span_count] = base + stop - 1; - start_idx_data[span_count] = start > 0 ? base + start - 1 : base; - lengths_data[span_count] = (float)span->n_tokens; - start_mask_data[span_count] = start > 0 ? 1.0f : 0.0f; - span_count++; - } - } - - int index_shape[] = {total_spans}; - int scale_shape[] = {total_spans, 1}; - end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32); - start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, - MLX_INT32); - lengths = mlx_array_new_data(lengths_data, scale_shape, 2, - MLX_FLOAT32); - start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, - MLX_FLOAT32); - free(end_idx_data); end_idx_data = NULL; - free(start_idx_data); start_idx_data = NULL; - free(lengths_data); lengths_data = NULL; - free(start_mask_data); start_mask_data = NULL; - if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || - !arr_ok(start_mask)) - goto cleanup; - - prefix = mlx_array_new(); - mlx_cumsum(&prefix, x_normed, 1, false, true, S); - mlx_array_free(x_normed); - x_normed = (mlx_array){0}; - if (!arr_ok(prefix)) goto cleanup; - - int flat_shape[] = {(int)rows, hidden}; - flat_prefix = mlx_array_new(); - mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S); - mlx_array_free(prefix); - prefix = (mlx_array){0}; - if (!arr_ok(flat_prefix)) goto cleanup; - - end_values = mlx_array_new(); - mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S); - if (!arr_ok(end_values)) goto cleanup; - - start_values = mlx_array_new(); - mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S); - if (!arr_ok(start_values)) goto cleanup; - - masked_start = mlx_array_new(); - mlx_multiply(&masked_start, start_values, start_mask, S); - if (!arr_ok(masked_start)) goto cleanup; - - span_sums = mlx_array_new(); - mlx_subtract(&span_sums, end_values, masked_start, S); - if (!arr_ok(span_sums)) goto cleanup; - - embeddings = mlx_array_new(); - mlx_divide(&embeddings, span_sums, lengths, S); - if (!arr_ok(embeddings)) goto cleanup; - } - - embeddings_f32 = mlx_array_new(); - mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S); - if (!arr_ok(embeddings_f32)) goto cleanup; - - mlx_array_eval(embeddings_f32); - const float *data = mlx_array_data_float32(embeddings_f32); - if (data) { - memcpy(out_embeddings, data, out_bytes); - rc = 0; - } - -cleanup: - free(start_mask_data); - free(lengths_data); - free(start_idx_data); - free(end_idx_data); - mlx_array_free(embeddings_f32); - mlx_array_free(embeddings); - if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec); - if (pooled) { - for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]); - free(pooled); - } - mlx_array_free(span_sums); - mlx_array_free(masked_start); - mlx_array_free(start_values); - mlx_array_free(end_values); - mlx_array_free(start_mask); - mlx_array_free(lengths); - mlx_array_free(start_idx); - mlx_array_free(end_idx); - mlx_array_free(flat_prefix); - mlx_array_free(prefix); - mlx_array_free(x_normed); - mlx_array_free(x); - mlx_array_free(attn_mask); - mlx_array_free(ids); - return rc; -} - -int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, - const pplx_context_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings) - return -1; - - int total_spans = 0; - if (!mlx_context_batch_should_split(inputs, batch, &total_spans)) - return pplx_mlx_embed_spans_batch_dense(ctx, inputs, batch, - out_embeddings); - - const int hidden = ctx->config.hidden_size; - mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - pplx_context_input_t *group_inputs = (pplx_context_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); - size_t group_values, group_bytes; - if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 || - mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); - return -1; - } - float *group_out = (float *)malloc(group_bytes); - if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); - return -1; - } - - int span_offset = 0; - for (int i = 0; i < batch; i++) { - items[i].index = i; - items[i].span_offset = span_offset; - items[i].input = &inputs[i]; - span_offset += inputs[i].n_spans; - } - qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp); - - int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { - int group_max = 0; - int group_tokens = 0; - int end = start; - while (end < batch) { - int next_tokens = items[end].input->input.n_tokens; - int next_count = end - start + 1; - int next_max = next_tokens > group_max ? next_tokens : group_max; - int next_total = group_tokens + next_tokens; - if (end > start) { - size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_context_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) - break; - } - group_max = next_max; - group_tokens = next_total; - end++; - } - - int group_count = end - start; - for (int i = 0; i < group_count; i++) - group_inputs[i] = *items[start + i].input; - rc = pplx_mlx_embed_spans_batch_dense(ctx, group_inputs, - group_count, group_out); - if (rc != 0) break; - int src_span = 0; - for (int i = 0; i < group_count; i++) { - int n_spans = group_inputs[i].n_spans; - int dst_span = items[start + i].span_offset; - memcpy(out_embeddings + (size_t)dst_span * hidden, - group_out + (size_t)src_span * hidden, - (size_t)n_spans * hidden * sizeof(float)); - src_span += n_spans; - } - start = end; - } - - free(items); free(group_inputs); free(group_out); - return rc; -} - -int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, const pplx_span_t *spans, - int n_spans, float *out_embeddings) -{ - pplx_context_input_t input = { - .input = { token_ids, n_tokens }, - .spans = spans, - .n_spans = n_spans, - }; - return pplx_mlx_embed_spans_batch(ctx, &input, 1, out_embeddings); -} - -float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) -{ - if (!ctx || !token_ids || n_tokens <= 0) return NULL; - - int hidden = ctx->config.hidden_size; - size_t out_bytes; - if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0) - return NULL; - float *out = (float *)malloc(out_bytes); - if (!out) return NULL; - - if (pplx_mlx_embed_into(ctx, token_ids, n_tokens, out) != 0) { - free(out); - return NULL; - } - return out; -} |
