aboutsummaryrefslogtreecommitdiff
path: root/embed_mlx.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 08:24:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit2c37727e653d2340d8977dd5db4addc767dd81ea (patch)
tree578ad6affac511f847a346a0efc6d40ebf6a279f /embed_mlx.c
parentec417bcb52f2aa8e24aeec312e8ddbf3a0798056 (diff)
downloadffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.tar.gz
ffwd-2c37727e653d2340d8977dd5db4addc767dd81ea.zip
gitignore: track new binary and library names
Diffstat (limited to 'embed_mlx.c')
-rw-r--r--embed_mlx.c2296
1 files changed, 2296 insertions, 0 deletions
diff --git a/embed_mlx.c b/embed_mlx.c
new file mode 100644
index 0000000..602571d
--- /dev/null
+++ b/embed_mlx.c
@@ -0,0 +1,2296 @@
+#include "pplx_embed_mlx.h"
+#include "pplx_embed.h"
+#include "qwen_safetensors.h"
+
+#include <mlx/c/mlx.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+/* ========================================================================
+ * Per-layer MLX weight arrays
+ * ======================================================================== */
+
+typedef struct {
+ mlx_array w;
+ mlx_array wt;
+ mlx_array scales;
+ mlx_array biases;
+ int quantized;
+ int bits;
+ int group_size;
+} mlx_linear_t;
+
+typedef struct {
+ mlx_linear_t wq, wk, wv, wo;
+ mlx_array q_norm, k_norm;
+ mlx_array input_norm, post_attn_norm;
+ mlx_linear_t gate_proj, up_proj, down_proj;
+} mlx_layer_t;
+
+struct pplx_mlx_ctx {
+ pplx_config_t config;
+ mlx_array embed_tokens;
+ mlx_layer_t *layers; /* heap-allocated [n_layers] */
+ mlx_array norm;
+ mlx_stream stream;
+ multi_safetensors_t *ms;
+ int layer_start; /* loaded transformer range, inclusive */
+ int layer_end; /* loaded transformer range, exclusive */
+ int quantize_bits;
+ int quantize_group_size;
+ mlx_dtype weight_dtype;
+};
+
+struct pplx_mlx_late_ctx {
+ pplx_mlx_ctx_t *base;
+ multi_safetensors_t *dense_ms;
+ mlx_linear_t projection;
+ int token_dim;
+};
+
+struct pplx_mlx_late_vectors {
+ pplx_mlx_late_ctx_t *owner;
+ mlx_array vectors; /* [tokens, dim] */
+ int tokens;
+ int dim;
+};
+
+/* ========================================================================
+ * Helpers
+ * ======================================================================== */
+
+static int arr_ok(mlx_array a) { return a.ctx != NULL; }
+
+static int mlx_mul_size(size_t a, size_t b, size_t *out)
+{
+ if (a != 0 && b > SIZE_MAX / a) return -1;
+ *out = a * b;
+ return 0;
+}
+
+static mlx_dtype mlx_weight_dtype(const pplx_mlx_ctx_t *ctx)
+{
+ return ctx->weight_dtype;
+}
+
+static size_t safetensor_dtype_size(safetensor_dtype_t dtype)
+{
+ switch (dtype) {
+ case DTYPE_F32: return sizeof(float);
+ case DTYPE_BF16: return sizeof(uint16_t);
+ default: return 0;
+ }
+}
+
+static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
+ const safetensor_t *t,
+ const char *name, const int *shape,
+ int ndim)
+{
+ size_t elem_size = safetensor_dtype_size(t->dtype);
+ if (elem_size == 0) {
+ fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
+ return 0;
+ }
+ if (t->ndim != ndim) {
+ fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n",
+ name, t->ndim, ndim);
+ return 0;
+ }
+
+ size_t numel = 1;
+ for (int i = 0; i < ndim; i++) {
+ if (shape[i] <= 0 || t->shape[i] != shape[i]) {
+ fprintf(stderr, "mlx: bad shape for %s at dim %d: "
+ "got %lld, expected %d\n",
+ name, i, (long long)t->shape[i], shape[i]);
+ return 0;
+ }
+ if (mlx_mul_size(numel, (size_t)shape[i], &numel) != 0) {
+ fprintf(stderr, "mlx: shape too large for %s\n", name);
+ return 0;
+ }
+ }
+
+ size_t bytes;
+ if (mlx_mul_size(numel, elem_size, &bytes) != 0) {
+ fprintf(stderr, "mlx: tensor data too large for %s\n", name);
+ return 0;
+ }
+ if (t->data_size != bytes) {
+ fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n",
+ name, t->data_size, bytes);
+ return 0;
+ }
+
+ if (sf->header_size > SIZE_MAX - 8) return 0;
+ size_t data_start = 8 + sf->header_size;
+ if (data_start > sf->file_size ||
+ t->data_offset > sf->file_size - data_start ||
+ t->data_size > sf->file_size - data_start - t->data_offset) {
+ fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name);
+ return 0;
+ }
+ return 1;
+}
+
+static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
+ const int *shape, int ndim)
+{
+ safetensors_file_t *sf = NULL;
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ if (!t) {
+ fprintf(stderr, "mlx: tensor not found: %s\n", name);
+ return (mlx_array){0};
+ }
+ if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim))
+ return (mlx_array){0};
+ mlx_dtype dtype;
+ if (t->dtype == DTYPE_F32) {
+ dtype = MLX_FLOAT32;
+ } else if (t->dtype == DTYPE_BF16) {
+ dtype = MLX_BFLOAT16;
+ } else {
+ fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
+ return (mlx_array){0};
+ }
+ return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype);
+}
+
+static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms,
+ const char *name,
+ const int *shape, int ndim)
+{
+ mlx_linear_t l = {0};
+ l.w = load_tensor(ms, name, shape, ndim);
+ return l;
+}
+
+static int linear_ok(mlx_linear_t l)
+{
+ return arr_ok(l.w) || arr_ok(l.wt);
+}
+
+static void free_linear(mlx_linear_t *l)
+{
+ mlx_array_free(l->w);
+ mlx_array_free(l->wt);
+ mlx_array_free(l->scales);
+ mlx_array_free(l->biases);
+ memset(l, 0, sizeof(*l));
+}
+
+static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s)
+{
+ if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt))
+ return 0;
+
+ mlx_array transposed = mlx_array_new();
+ mlx_transpose(&transposed, l->w, s);
+ if (!arr_ok(transposed)) {
+ mlx_array_free(transposed);
+ fprintf(stderr, "mlx: failed to prepare transposed linear weight\n");
+ return -1;
+ }
+
+ l->wt = mlx_array_new();
+ mlx_contiguous(&l->wt, transposed, false, s);
+ mlx_array_free(transposed);
+ if (!arr_ok(l->wt)) {
+ mlx_array_free(l->wt);
+ l->wt = (mlx_array){0};
+ fprintf(stderr, "mlx: failed to materialize transposed linear weight\n");
+ return -1;
+ }
+ mlx_array_eval(l->wt);
+ mlx_synchronize(s);
+ mlx_array_free(l->w);
+ l->w = (mlx_array){0};
+ return 0;
+}
+
+static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
+ mlx_stream s)
+{
+ if (!l || !arr_ok(l->w) || bits == 0) return 0;
+ if (bits != 8) {
+ fprintf(stderr, "mlx: quantized weights support only 8 bits\n");
+ return -1;
+ }
+ if (group_size <= 0) group_size = 64;
+
+ mlx_optional_int gs = {.value = group_size, .has_value = true};
+ mlx_optional_int qb = {.value = bits, .has_value = true};
+ mlx_array null_arr = (mlx_array){0};
+ mlx_vector_array q = mlx_vector_array_new();
+ mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s);
+ if (!q.ctx || mlx_vector_array_size(q) < 2) {
+ mlx_vector_array_free(q);
+ fprintf(stderr, "mlx: quantize failed\n");
+ return -1;
+ }
+
+ mlx_array qw = (mlx_array){0};
+ mlx_array scales = (mlx_array){0};
+ mlx_array biases = (mlx_array){0};
+ mlx_vector_array_get(&qw, q, 0);
+ mlx_vector_array_get(&scales, q, 1);
+ if (mlx_vector_array_size(q) >= 3)
+ mlx_vector_array_get(&biases, q, 2);
+ mlx_vector_array_free(q);
+
+ if (!arr_ok(qw) || !arr_ok(scales)) {
+ mlx_array_free(qw);
+ mlx_array_free(scales);
+ mlx_array_free(biases);
+ fprintf(stderr, "mlx: quantize returned invalid arrays\n");
+ return -1;
+ }
+ mlx_array_eval(qw);
+ mlx_array_eval(scales);
+ if (arr_ok(biases)) mlx_array_eval(biases);
+ mlx_synchronize(s);
+
+ mlx_array_free(l->w);
+ mlx_array_free(l->wt);
+ l->w = qw;
+ l->wt = (mlx_array){0};
+ l->scales = scales;
+ l->biases = biases;
+ l->quantized = 1;
+ l->bits = bits;
+ l->group_size = group_size;
+ return 0;
+}
+
+/* y = x @ W^T */
+static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
+{
+ if (W->quantized) {
+ if (!arr_ok(W->w)) return (mlx_array){0};
+ mlx_array res = mlx_array_new();
+ mlx_optional_int gs = {.value = W->group_size, .has_value = true};
+ mlx_optional_int bits = {.value = W->bits, .has_value = true};
+ mlx_array null_arr = (mlx_array){0};
+ mlx_quantized_matmul(&res, x, W->w, W->scales,
+ arr_ok(W->biases) ? W->biases : null_arr,
+ true, gs, bits, "affine", s);
+ return res;
+ }
+ mlx_array Wt;
+ int free_wt = 0;
+ if (arr_ok(W->wt)) {
+ Wt = W->wt;
+ } else if (arr_ok(W->w)) {
+ Wt = mlx_array_new();
+ mlx_transpose(&Wt, W->w, s);
+ free_wt = 1;
+ } else {
+ return (mlx_array){0};
+ }
+ mlx_array res = mlx_array_new();
+ if (mlx_array_dtype(Wt) == MLX_BFLOAT16 &&
+ mlx_array_dtype(x) == MLX_FLOAT32) {
+ /* F32 activations with BF16 weights: upcast the weight operand and run
+ * the matmul in F32 (so accumulation is F32). Only the weights are
+ * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers),
+ * matching the CPU/CUDA accuracy while keeping BF16 weight storage. */
+ mlx_array Wf = mlx_array_new();
+ mlx_astype(&Wf, Wt, MLX_FLOAT32, s);
+ mlx_matmul(&res, x, Wf, s);
+ mlx_array_free(Wf);
+ } else {
+ mlx_matmul(&res, x, Wt, s);
+ }
+ if (free_wt) mlx_array_free(Wt);
+ return res;
+}
+
+const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx)
+{
+ return ctx ? &ctx->config : NULL;
+}
+
+/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the
+ * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the
+ * residual stream in F32 while keeping the large matmul weights in BF16. */
+static void upcast_f32_inplace(mlx_array *a, mlx_stream s)
+{
+ if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return;
+ mlx_array f = mlx_array_new();
+ mlx_astype(&f, *a, MLX_FLOAT32, s);
+ mlx_array_free(*a);
+ *a = f;
+}
+
+/* ========================================================================
+ * Load
+ * ======================================================================== */
+
+/* We duplicate a minimal config parser here so MLX can load standalone.
+ * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */
+
+static const char *skip_ws_m(const char *p)
+{
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+ return p;
+}
+
+static const char *json_find_m(const char *json, const char *key)
+{
+ size_t klen = strlen(key);
+ const char *p = json;
+ while ((p = strstr(p, key)) != NULL) {
+ if (p > json && *(p - 1) == '"') {
+ const char *a = p + klen;
+ if (*a == '"') {
+ a = skip_ws_m(a + 1);
+ if (*a == ':') return skip_ws_m(a + 1);
+ }
+ }
+ p += klen;
+ }
+ return NULL;
+}
+
+static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir)
+{
+ char path[1024];
+ int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir);
+ if (path_len < 0 || (size_t)path_len >= sizeof(path)) {
+ fprintf(stderr, "mlx: model path too long\n");
+ return -1;
+ }
+ FILE *f = fopen(path, "rb");
+ if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; }
+ if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ fclose(f); free(buf); return -1;
+ }
+ fclose(f);
+ buf[sz] = '\0';
+
+#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0)
+#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0)
+ GI("hidden_size", hidden_size, 0);
+ GI("num_hidden_layers", n_layers, 0);
+ GI("num_attention_heads", n_heads, 0);
+ GI("num_key_value_heads", n_kv_heads, 0);
+ GI("head_dim", head_dim, PPLX_HEAD_DIM);
+ GI("intermediate_size", intermediate_size, 0);
+ GI("vocab_size", vocab_size, PPLX_VOCAB_SIZE);
+ GF("rms_norm_eps", rms_norm_eps, 1e-6f);
+ GF("rope_theta", rope_theta, 1000000.0f);
+#undef GI
+#undef GF
+
+ cfg->q_dim = 0;
+ cfg->kv_dim = 0;
+ if (cfg->n_heads > 0 && cfg->head_dim > 0 &&
+ cfg->n_heads <= INT_MAX / cfg->head_dim)
+ cfg->q_dim = cfg->n_heads * cfg->head_dim;
+ if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 &&
+ cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
+ cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
+ free(buf);
+
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 ||
+ cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
+ cfg->head_dim <= 0 || cfg->intermediate_size <= 0 ||
+ cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
+ (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
+ !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
+ !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
+ fprintf(stderr, "mlx: invalid config in %s "
+ "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
+ path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
+ cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size);
+ return -1;
+ }
+ if (cfg->n_layers > PPLX_MAX_LAYERS) {
+ fprintf(stderr, "mlx: too many layers (%d > %d)\n",
+ cfg->n_layers, PPLX_MAX_LAYERS);
+ return -1;
+ }
+ return 0;
+}
+
+static int mlx_model_dir_has_file(const char *model_dir, const char *rel)
+{
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
+ if (n < 0 || (size_t)n >= sizeof(path)) return 0;
+ FILE *f = fopen(path, "rb");
+ if (!f) return 0;
+ fclose(f);
+ return 1;
+}
+
+static int mlx_model_dir_has_late_projection(const char *model_dir)
+{
+ return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") &&
+ mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors");
+}
+
+static pplx_mlx_options_t normalize_mlx_options(const pplx_mlx_options_t *opts)
+{
+ pplx_mlx_options_t out = {0};
+ if (opts) out = *opts;
+ if (out.quantize_bits != 0 &&
+ out.quantize_bits != 8) {
+ fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n");
+ out.quantize_bits = -1;
+ }
+ if (out.quantize_bits && out.quantize_group_size <= 0)
+ out.quantize_group_size = 64;
+ return out;
+}
+
+static int quantize_layer(mlx_layer_t *l, int bits, int group_size,
+ mlx_stream s)
+{
+ if (bits == 0) return 0;
+ return quantize_linear(&l->wq, bits, group_size, s) ||
+ quantize_linear(&l->wk, bits, group_size, s) ||
+ quantize_linear(&l->wv, bits, group_size, s) ||
+ quantize_linear(&l->wo, bits, group_size, s) ||
+ quantize_linear(&l->gate_proj, bits, group_size, s) ||
+ quantize_linear(&l->up_proj, bits, group_size, s) ||
+ quantize_linear(&l->down_proj, bits, group_size, s);
+}
+
+static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s)
+{
+ return prepare_linear_transpose(&l->wq, s) ||
+ prepare_linear_transpose(&l->wk, s) ||
+ prepare_linear_transpose(&l->wv, s) ||
+ prepare_linear_transpose(&l->wo, s) ||
+ prepare_linear_transpose(&l->gate_proj, s) ||
+ prepare_linear_transpose(&l->up_proj, s) ||
+ prepare_linear_transpose(&l->down_proj, s);
+}
+
+static pplx_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
+ int layer_start, int layer_end,
+ const pplx_mlx_options_t *opts,
+ int allow_late)
+{
+ pplx_mlx_options_t options = normalize_mlx_options(opts);
+ if (options.quantize_bits < 0) return NULL;
+ if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) {
+ fprintf(stderr, "mlx: late-interaction models require token-level "
+ "MaxSim support and are not valid pooled embedding models "
+ "yet\n");
+ return NULL;
+ }
+
+ pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t));
+ if (!ctx) return NULL;
+ ctx->quantize_bits = options.quantize_bits;
+ ctx->quantize_group_size = options.quantize_group_size;
+
+ if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; }
+
+ const pplx_config_t *c = &ctx->config;
+ if (layer_end == -1)
+ layer_end = c->n_layers;
+ if (layer_start < 0 || layer_start >= layer_end ||
+ layer_end > c->n_layers) {
+ fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n",
+ layer_start, layer_end, c->n_layers);
+ free(ctx);
+ return NULL;
+ }
+ ctx->layer_start = layer_start;
+ ctx->layer_end = layer_end;
+
+ ctx->ms = multi_safetensors_open(model_dir);
+ if (!ctx->ms) {
+ fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
+ free(ctx); return NULL;
+ }
+
+ int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
+ int inter = c->intermediate_size, hd = c->head_dim;
+
+ /* Only the first stage needs token embeddings. */
+ if (layer_start == 0) {
+ int emb_shape[] = {c->vocab_size, h};
+ ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight",
+ emb_shape, 2);
+ if (!arr_ok(ctx->embed_tokens)) goto fail;
+ }
+
+ /* Layers */
+ ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
+ if (!ctx->layers) goto fail;
+
+ char name[256];
+ for (int i = layer_start; i < layer_end; i++) {
+ mlx_layer_t *l = &ctx->layers[i];
+
+#define LD(fld, fmt, ...) do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
+ if (!linear_ok(l->fld)) goto fail; \
+} while (0)
+
+ LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h);
+ LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h);
+ LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h);
+ LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd);
+#undef LD
+
+#define LD(fld, fmt, ...) do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
+ if (!arr_ok(l->fld)) goto fail; \
+} while (0)
+
+ LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd);
+ LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd);
+ LD(input_norm, "layers.%d.input_layernorm.weight", h);
+ LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h);
+#undef LD
+
+#define LD(fld, fmt, ...) do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
+ if (!linear_ok(l->fld)) goto fail; \
+} while (0)
+
+ LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h);
+ LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h);
+ LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter);
+#undef LD
+
+ if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i);
+ }
+
+ if (layer_end == c->n_layers) {
+ int ns[] = {h};
+ ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1);
+ if (!arr_ok(ctx->norm)) goto fail;
+ }
+
+ ctx->stream = mlx_default_gpu_stream_new();
+ ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w);
+ /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the
+ * tiny norm vectors so they match the F32 activations in mlx_forward_layers;
+ * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */
+ if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) {
+ for (int i = layer_start; i < layer_end; i++) {
+ mlx_layer_t *l = &ctx->layers[i];
+ upcast_f32_inplace(&l->q_norm, ctx->stream);
+ upcast_f32_inplace(&l->k_norm, ctx->stream);
+ upcast_f32_inplace(&l->input_norm, ctx->stream);
+ upcast_f32_inplace(&l->post_attn_norm, ctx->stream);
+ }
+ upcast_f32_inplace(&ctx->norm, ctx->stream);
+ }
+ if (ctx->quantize_bits) {
+ for (int i = layer_start; i < layer_end; i++) {
+ if (quantize_layer(&ctx->layers[i], ctx->quantize_bits,
+ ctx->quantize_group_size, ctx->stream) != 0)
+ goto fail;
+ }
+ mlx_clear_cache();
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "mlx: quantized linear weights to %d-bit "
+ "(group_size=%d)\n",
+ ctx->quantize_bits, ctx->quantize_group_size);
+ } else {
+ for (int i = layer_start; i < layer_end; i++) {
+ if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0)
+ goto fail;
+ }
+ }
+
+ if (pplx_verbose >= 2) {
+ size_t active = 0, cache = 0, peak = 0;
+ mlx_get_active_memory(&active);
+ mlx_get_cache_memory(&cache);
+ mlx_get_peak_memory(&peak);
+ fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB "
+ "peak=%.2f GiB\n",
+ (double)active / (1024.0 * 1024.0 * 1024.0),
+ (double)cache / (1024.0 * 1024.0 * 1024.0),
+ (double)peak / (1024.0 * 1024.0 * 1024.0));
+ }
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n",
+ layer_start, layer_end, h);
+
+ return ctx;
+
+fail:
+ pplx_mlx_free(ctx);
+ return NULL;
+}
+
+pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
+{
+ return mlx_load_range_ex(model_dir, 0, -1, NULL, 0);
+}
+
+pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir,
+ const pplx_mlx_options_t *opts)
+{
+ return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
+}
+
+pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir,
+ int layer_start, int layer_end)
+{
+ return mlx_load_range_ex(model_dir, layer_start, layer_end, NULL, 0);
+}
+
+pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options(
+ const char *model_dir, int layer_start, int layer_end,
+ const pplx_mlx_options_t *opts)
+{
+ return mlx_load_range_ex(model_dir, layer_start, layer_end, opts, 0);
+}
+
+/* ========================================================================
+ * Free
+ * ======================================================================== */
+
+static void free_mlx_layer(mlx_layer_t *l)
+{
+ free_linear(&l->wq); free_linear(&l->wk);
+ free_linear(&l->wv); free_linear(&l->wo);
+ mlx_array_free(l->q_norm); mlx_array_free(l->k_norm);
+ mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm);
+ free_linear(&l->gate_proj); free_linear(&l->up_proj);
+ free_linear(&l->down_proj);
+}
+
+void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
+{
+ if (!ctx) return;
+ mlx_array_free(ctx->embed_tokens);
+ if (ctx->layers) {
+ for (int i = 0; i < ctx->config.n_layers; i++)
+ free_mlx_layer(&ctx->layers[i]);
+ free(ctx->layers);
+ }
+ mlx_array_free(ctx->norm);
+ if (ctx->stream.ctx) mlx_stream_free(ctx->stream);
+ if (ctx->ms) multi_safetensors_close(ctx->ms);
+ free(ctx);
+}
+
+pplx_mlx_late_ctx_t *pplx_mlx_late_load_with_options(
+ const char *model_dir, const pplx_mlx_options_t *opts)
+{
+ if (!mlx_model_dir_has_late_projection(model_dir)) {
+ fprintf(stderr, "mlx late: missing 1_Dense projection\n");
+ return NULL;
+ }
+
+ pplx_mlx_late_ctx_t *late = calloc(1, sizeof(*late));
+ if (!late) return NULL;
+
+ late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1);
+ if (!late->base) goto fail;
+
+ char dense_dir[1024];
+ int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir);
+ if (n < 0 || (size_t)n >= sizeof(dense_dir)) {
+ fprintf(stderr, "mlx late: model path too long\n");
+ goto fail;
+ }
+
+ late->dense_ms = multi_safetensors_open(dense_dir);
+ if (!late->dense_ms) {
+ fprintf(stderr, "mlx late: failed to open safetensors in %s\n",
+ dense_dir);
+ goto fail;
+ }
+
+ const safetensor_t *t = multi_safetensors_find(late->dense_ms,
+ "linear.weight", NULL);
+ if (!t) {
+ fprintf(stderr, "mlx late: missing linear.weight\n");
+ goto fail;
+ }
+ if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX ||
+ t->shape[1] != late->base->config.hidden_size) {
+ fprintf(stderr, "mlx late: bad projection shape\n");
+ goto fail;
+ }
+
+ int shape[] = { (int)t->shape[0], late->base->config.hidden_size };
+ late->projection = load_linear_tensor(late->dense_ms, "linear.weight",
+ shape, 2);
+ if (!linear_ok(late->projection)) goto fail;
+ if (prepare_linear_transpose(&late->projection, late->base->stream) != 0)
+ goto fail;
+ late->token_dim = shape[0];
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n",
+ late->token_dim);
+ return late;
+
+fail:
+ pplx_mlx_late_free(late);
+ return NULL;
+}
+
+pplx_mlx_late_ctx_t *pplx_mlx_late_load(const char *model_dir)
+{
+ return pplx_mlx_late_load_with_options(model_dir, NULL);
+}
+
+void pplx_mlx_late_free(pplx_mlx_late_ctx_t *ctx)
+{
+ if (!ctx) return;
+ free_linear(&ctx->projection);
+ if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms);
+ pplx_mlx_free(ctx->base);
+ free(ctx);
+}
+
+const pplx_config_t *pplx_mlx_late_config(const pplx_mlx_late_ctx_t *ctx)
+{
+ return ctx && ctx->base ? &ctx->base->config : NULL;
+}
+
+int pplx_mlx_late_token_dim(const pplx_mlx_late_ctx_t *ctx)
+{
+ return ctx ? ctx->token_dim : 0;
+}
+
+/* ========================================================================
+ * Forward pass
+ * ======================================================================== */
+
+static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x,
+ int batch, int max_seq, int has_padding,
+ mlx_array attn_mask,
+ int layer_start, int layer_end)
+{
+ const pplx_config_t *c = &ctx->config;
+ mlx_stream S = ctx->stream;
+ int n_heads = c->n_heads;
+ int n_kv_heads = c->n_kv_heads;
+ int head_dim = c->head_dim;
+ int q_dim = c->q_dim;
+ float scale = 1.0f / sqrtf((float)head_dim);
+ mlx_array null_arr = (mlx_array){0};
+
+ /* For non-quantized BF16 weights, run the residual stream, norms, and
+ * attention in F32; matmuls downcast their operands to BF16 internally
+ * (see linear). This keeps weight storage at BF16 while avoiding the
+ * cross-layer accumulation error of a fully BF16 forward. F32 and quantized
+ * models are unaffected. */
+ mlx_array local_mask = attn_mask;
+ int free_mask = 0;
+ int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 &&
+ !ctx->layers[layer_start].wq.quantized;
+ if (mixed_bf16) {
+ if (mlx_array_dtype(x) != MLX_FLOAT32) {
+ mlx_array xf = mlx_array_new();
+ mlx_astype(&xf, x, MLX_FLOAT32, S);
+ mlx_array_free(x);
+ x = xf;
+ }
+ if (has_padding && arr_ok(attn_mask) &&
+ mlx_array_dtype(attn_mask) != MLX_FLOAT32) {
+ local_mask = mlx_array_new();
+ mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S);
+ free_mask = 1;
+ }
+ }
+
+ for (int layer = layer_start; layer < layer_end; layer++) {
+ mlx_layer_t *l = &ctx->layers[layer];
+
+ mlx_array xn = mlx_array_new();
+ mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S);
+
+ mlx_array q_flat = linear(xn, &l->wq, S);
+ mlx_array k_flat = linear(xn, &l->wk, S);
+ mlx_array v_flat = linear(xn, &l->wv, S);
+ mlx_array_free(xn);
+
+ int q_shape[] = {batch, max_seq, n_heads, head_dim};
+ int k_shape[] = {batch, max_seq, n_kv_heads, head_dim};
+ mlx_array q = mlx_array_new();
+ mlx_array k = mlx_array_new();
+ mlx_array v = mlx_array_new();
+ mlx_reshape(&q, q_flat, q_shape, 4, S);
+ mlx_reshape(&k, k_flat, k_shape, 4, S);
+ mlx_reshape(&v, v_flat, k_shape, 4, S);
+ mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat);
+
+ mlx_array qn = mlx_array_new();
+ mlx_array kn = mlx_array_new();
+ mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
+ mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
+ mlx_array_free(q); mlx_array_free(k);
+
+ /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */
+ int perm[] = {0, 2, 1, 3};
+ mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
+ mlx_transpose_axes(&qt, qn, perm, 4, S);
+ mlx_transpose_axes(&kt, kn, perm, 4, S);
+ mlx_transpose_axes(&vt, v, perm, 4, S);
+ mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v);
+
+ mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
+ mlx_array qr = mlx_array_new(), kr = mlx_array_new();
+ mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
+ mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
+ mlx_array_free(qt); mlx_array_free(kt);
+
+ mlx_array attn = mlx_array_new();
+ mlx_fast_scaled_dot_product_attention(
+ &attn, qr, kr, vt, scale, "",
+ has_padding ? local_mask : null_arr, null_arr, S);
+ mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
+
+ mlx_array attn_t = mlx_array_new();
+ mlx_transpose_axes(&attn_t, attn, perm, 4, S);
+ mlx_array_free(attn);
+
+ int flat_shape[] = {batch, max_seq, q_dim};
+ mlx_array attn_flat = mlx_array_new();
+ mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S);
+ mlx_array_free(attn_t);
+
+ mlx_array proj = linear(attn_flat, &l->wo, S);
+ mlx_array_free(attn_flat);
+ mlx_array x2 = mlx_array_new();
+ mlx_add(&x2, x, proj, S);
+ mlx_array_free(x); mlx_array_free(proj);
+ x = x2;
+
+ mlx_array xn2 = mlx_array_new();
+ mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
+
+ mlx_array gate = linear(xn2, &l->gate_proj, S);
+ mlx_array up = linear(xn2, &l->up_proj, S);
+ mlx_array_free(xn2);
+
+ mlx_array gate_sig = mlx_array_new();
+ mlx_sigmoid(&gate_sig, gate, S);
+ mlx_array silu_gate = mlx_array_new();
+ mlx_multiply(&silu_gate, gate, gate_sig, S);
+ mlx_array_free(gate); mlx_array_free(gate_sig);
+
+ mlx_array mid = mlx_array_new();
+ mlx_multiply(&mid, silu_gate, up, S);
+ mlx_array_free(silu_gate); mlx_array_free(up);
+
+ mlx_array ffn = linear(mid, &l->down_proj, S);
+ mlx_array_free(mid);
+
+ mlx_array x3 = mlx_array_new();
+ mlx_add(&x3, x, ffn, S);
+ mlx_array_free(x); mlx_array_free(ffn);
+ x = x3;
+ }
+ if (free_mask) mlx_array_free(local_mask);
+ return x;
+}
+
+pplx_mlx_late_vectors_t *pplx_mlx_late_encode_tokens_device(
+ pplx_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens,
+ int normalize)
+{
+ if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 ||
+ ctx->token_dim <= 0 || !linear_ok(ctx->projection))
+ return NULL;
+
+ pplx_mlx_ctx_t *base = ctx->base;
+ if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
+ !arr_ok(base->embed_tokens) || !arr_ok(base->norm))
+ return NULL;
+
+ const pplx_config_t *c = &base->config;
+ mlx_stream S = base->stream;
+ for (int i = 0; i < n_tokens; i++) {
+ if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
+ fprintf(stderr, "mlx late: invalid token id %d at position %d\n",
+ token_ids[i], i);
+ return NULL;
+ }
+ }
+
+ size_t id_bytes;
+ if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0)
+ return NULL;
+
+ int *ids_data = (int *)malloc(id_bytes);
+ if (!ids_data) return NULL;
+ memcpy(ids_data, token_ids, id_bytes);
+
+ int ids_shape[] = {1, n_tokens};
+ mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32);
+ free(ids_data);
+ if (!arr_ok(ids)) return NULL;
+
+ pplx_mlx_late_vectors_t *result = NULL;
+ mlx_array x = mlx_array_new();
+ mlx_array x_normed = (mlx_array){0};
+ mlx_array projected = (mlx_array){0};
+ mlx_array reshaped = (mlx_array){0};
+
+ mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
+ mlx_array_free(ids);
+ if (!arr_ok(x)) goto cleanup;
+
+ x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0},
+ 0, c->n_layers);
+ if (!arr_ok(x)) goto cleanup;
+
+ x_normed = mlx_array_new();
+ mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S);
+ mlx_array_free(x);
+ x = (mlx_array){0};
+ if (!arr_ok(x_normed)) goto cleanup;
+
+ projected = linear(x_normed, &ctx->projection, S);
+ mlx_array_free(x_normed);
+ x_normed = (mlx_array){0};
+ if (!arr_ok(projected)) goto cleanup;
+
+ if (normalize) {
+ mlx_array sq = mlx_array_new();
+ mlx_square(&sq, projected, S);
+ if (!arr_ok(sq)) {
+ mlx_array_free(sq);
+ goto cleanup;
+ }
+
+ mlx_array norm2 = mlx_array_new();
+ mlx_sum_axis(&norm2, sq, 2, true, S);
+ mlx_array_free(sq);
+ if (!arr_ok(norm2)) {
+ mlx_array_free(norm2);
+ goto cleanup;
+ }
+
+ mlx_array inv_norm = mlx_array_new();
+ mlx_rsqrt(&inv_norm, norm2, S);
+ mlx_array_free(norm2);
+ if (!arr_ok(inv_norm)) {
+ mlx_array_free(inv_norm);
+ goto cleanup;
+ }
+
+ mlx_array normalized = mlx_array_new();
+ mlx_multiply(&normalized, projected, inv_norm, S);
+ mlx_array_free(inv_norm);
+ mlx_array_free(projected);
+ projected = normalized;
+ if (!arr_ok(projected)) goto cleanup;
+ }
+
+ int vec_shape[] = {n_tokens, ctx->token_dim};
+ reshaped = mlx_array_new();
+ mlx_reshape(&reshaped, projected, vec_shape, 2, S);
+ if (!arr_ok(reshaped)) goto cleanup;
+
+ result = (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*result));
+ if (!result) goto cleanup;
+ result->owner = ctx;
+ result->vectors = reshaped;
+ result->tokens = n_tokens;
+ result->dim = ctx->token_dim;
+ reshaped = (mlx_array){0};
+
+cleanup:
+ mlx_array_free(reshaped);
+ mlx_array_free(projected);
+ mlx_array_free(x_normed);
+ mlx_array_free(x);
+ return result;
+}
+
+void pplx_mlx_late_vectors_free(pplx_mlx_late_vectors_t *vecs)
+{
+ if (!vecs) return;
+ mlx_array_free(vecs->vectors);
+ free(vecs);
+}
+
+int pplx_mlx_late_vectors_token_count(const pplx_mlx_late_vectors_t *vecs)
+{
+ return vecs ? vecs->tokens : 0;
+}
+
+int pplx_mlx_late_vectors_dim(const pplx_mlx_late_vectors_t *vecs)
+{
+ return vecs ? vecs->dim : 0;
+}
+
+int pplx_mlx_late_vectors_copy(const pplx_mlx_late_vectors_t *vecs,
+ float *out_vectors)
+{
+ if (!vecs || !out_vectors || !arr_ok(vecs->vectors) ||
+ vecs->tokens <= 0 || vecs->dim <= 0)
+ return -1;
+
+ size_t values, bytes;
+ if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 ||
+ mlx_mul_size(values, sizeof(float), &bytes) != 0)
+ return -1;
+
+ mlx_stream S = vecs->owner->base->stream;
+ mlx_array out_f32 = mlx_array_new();
+ mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S);
+ if (!arr_ok(out_f32)) {
+ mlx_array_free(out_f32);
+ return -1;
+ }
+
+ int rc = -1;
+ mlx_array_eval(out_f32);
+ const float *data = mlx_array_data_float32(out_f32);
+ if (data) {
+ memcpy(out_vectors, data, bytes);
+ rc = 0;
+ }
+ mlx_array_free(out_f32);
+ return rc;
+}
+
+pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_concat(
+ pplx_mlx_late_ctx_t *ctx,
+ const pplx_mlx_late_vectors_t *const *items, int count)
+{
+ if (!ctx || !ctx->base || !items || count <= 0) return NULL;
+ int dim = 0;
+ int total = 0;
+ for (int i = 0; i < count; i++) {
+ const pplx_mlx_late_vectors_t *v = items[i];
+ if (!v || v->owner != ctx || !arr_ok(v->vectors) ||
+ v->tokens <= 0 || v->dim <= 0)
+ return NULL;
+ if (i == 0) dim = v->dim;
+ if (v->dim != dim || total > INT_MAX - v->tokens)
+ return NULL;
+ total += v->tokens;
+ }
+
+ mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts));
+ if (!parts) return NULL;
+ for (int i = 0; i < count; i++)
+ parts[i] = items[i]->vectors;
+
+ mlx_stream S = ctx->base->stream;
+ mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count);
+ free(parts);
+ if (!vec.ctx) return NULL;
+
+ mlx_array joined = mlx_array_new();
+ mlx_concatenate_axis(&joined, vec, 0, S);
+ mlx_vector_array_free(vec);
+ if (!arr_ok(joined)) {
+ mlx_array_free(joined);
+ return NULL;
+ }
+
+ pplx_mlx_late_vectors_t *out =
+ (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ if (!out) {
+ mlx_array_free(joined);
+ return NULL;
+ }
+ out->owner = ctx;
+ out->vectors = joined;
+ out->tokens = total;
+ out->dim = dim;
+ return out;
+}
+
+pplx_mlx_late_vectors_t *pplx_mlx_late_vectors_select(
+ pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *vecs,
+ const int *token_indices, int count)
+{
+ if (!ctx || !ctx->base || !vecs || vecs->owner != ctx ||
+ !arr_ok(vecs->vectors) || !token_indices || count <= 0 ||
+ vecs->tokens <= 0 || vecs->dim <= 0)
+ return NULL;
+
+ for (int i = 0; i < count; i++) {
+ if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens)
+ return NULL;
+ }
+
+ size_t idx_bytes;
+ if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0)
+ return NULL;
+ int *idx_data = (int *)malloc(idx_bytes);
+ if (!idx_data) return NULL;
+ memcpy(idx_data, token_indices, idx_bytes);
+
+ int idx_shape[] = {count};
+ mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32);
+ free(idx_data);
+ if (!arr_ok(idx)) return NULL;
+
+ mlx_stream S = ctx->base->stream;
+ mlx_array selected = mlx_array_new();
+ mlx_take_axis(&selected, vecs->vectors, idx, 0, S);
+ mlx_array_free(idx);
+ if (!arr_ok(selected)) {
+ mlx_array_free(selected);
+ return NULL;
+ }
+
+ pplx_mlx_late_vectors_t *out =
+ (pplx_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ if (!out) {
+ mlx_array_free(selected);
+ return NULL;
+ }
+ out->owner = ctx;
+ out->vectors = selected;
+ out->tokens = count;
+ out->dim = vecs->dim;
+ return out;
+}
+
+int pplx_mlx_late_maxsim_batch_device(
+ pplx_mlx_late_ctx_t *ctx, const pplx_mlx_late_vectors_t *query,
+ const pplx_mlx_late_vectors_t *docs, const int *doc_offsets,
+ int docs_count, float *scores)
+{
+ if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 ||
+ !scores || query->owner != ctx || docs->owner != ctx ||
+ query->dim <= 0 || query->dim != docs->dim ||
+ query->tokens <= 0 || docs->tokens <= 0 ||
+ !arr_ok(query->vectors) || !arr_ok(docs->vectors) ||
+ doc_offsets[0] != 0)
+ return -1;
+
+ int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths));
+ if (!lengths) return -1;
+ int max_doc_tokens = 0;
+ for (int i = 0; i < docs_count; i++) {
+ if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] ||
+ doc_offsets[i + 1] > docs->tokens) {
+ free(lengths);
+ return -1;
+ }
+ int n = doc_offsets[i + 1] - doc_offsets[i];
+ lengths[i] = n;
+ if (n > max_doc_tokens) max_doc_tokens = n;
+ }
+ if (max_doc_tokens <= 0) {
+ free(lengths);
+ return -1;
+ }
+
+ mlx_stream S = ctx->base->stream;
+ int dim = query->dim;
+ int rc = -1;
+ mlx_array *doc_rows =
+ (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
+ if (!doc_rows) {
+ free(lengths);
+ return -1;
+ }
+
+ mlx_vector_array doc_vec = (mlx_vector_array){0};
+ mlx_array pad_value = mlx_array_new_float32(0.0f);
+ mlx_array doc_batch = (mlx_array){0};
+ mlx_array query_3d = (mlx_array){0};
+ mlx_array query_batched = (mlx_array){0};
+ mlx_array docs_t = (mlx_array){0};
+ mlx_array sims = (mlx_array){0};
+ mlx_array lengths_arr = (mlx_array){0};
+ mlx_array positions = (mlx_array){0};
+ mlx_array lengths_2d = (mlx_array){0};
+ mlx_array positions_2d = (mlx_array){0};
+ mlx_array valid = (mlx_array){0};
+ mlx_array valid_3d = (mlx_array){0};
+ mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX);
+ mlx_array masked_sims = (mlx_array){0};
+ mlx_array maxes = (mlx_array){0};
+ mlx_array sums = (mlx_array){0};
+ mlx_array sums_f32 = (mlx_array){0};
+
+ for (int i = 0; i < docs_count; i++) {
+ int n = lengths[i];
+ int start[] = {doc_offsets[i], 0};
+ int stop[] = {doc_offsets[i + 1], dim};
+ int strides[] = {1, 1};
+
+ mlx_array doc = mlx_array_new();
+ mlx_array padded = (mlx_array){0};
+
+ mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S);
+ if (!arr_ok(doc)) goto cleanup;
+
+ if (n < max_doc_tokens) {
+ int axes[] = {0};
+ int low[] = {0};
+ int high[] = {max_doc_tokens - n};
+ padded = mlx_array_new();
+ mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value,
+ "constant", S);
+ mlx_array_free(doc);
+ doc = padded;
+ padded = (mlx_array){0};
+ if (!arr_ok(doc)) goto cleanup;
+ }
+
+ doc_rows[i] = mlx_array_new();
+ mlx_expand_dims(&doc_rows[i], doc, 0, S);
+ mlx_array_free(doc);
+ if (!arr_ok(doc_rows[i])) goto cleanup;
+ }
+
+ doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count);
+ if (!doc_vec.ctx) goto cleanup;
+
+ doc_batch = mlx_array_new();
+ mlx_concatenate_axis(&doc_batch, doc_vec, 0, S);
+ if (!arr_ok(doc_batch)) goto cleanup;
+
+ query_3d = mlx_array_new();
+ mlx_expand_dims(&query_3d, query->vectors, 0, S);
+ if (!arr_ok(query_3d)) goto cleanup;
+
+ int q_shape[] = {docs_count, query->tokens, dim};
+ query_batched = mlx_array_new();
+ mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S);
+ if (!arr_ok(query_batched)) goto cleanup;
+
+ int perm[] = {0, 2, 1};
+ docs_t = mlx_array_new();
+ mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S);
+ if (!arr_ok(docs_t)) goto cleanup;
+
+ sims = mlx_array_new();
+ mlx_matmul(&sims, query_batched, docs_t, S);
+ if (!arr_ok(sims)) goto cleanup;
+
+ int len_shape[] = {docs_count};
+ lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32);
+ if (!arr_ok(lengths_arr)) goto cleanup;
+
+ positions = mlx_array_new();
+ mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S);
+ if (!arr_ok(positions)) goto cleanup;
+
+ lengths_2d = mlx_array_new();
+ mlx_expand_dims(&lengths_2d, lengths_arr, 1, S);
+ if (!arr_ok(lengths_2d)) goto cleanup;
+
+ positions_2d = mlx_array_new();
+ mlx_expand_dims(&positions_2d, positions, 0, S);
+ if (!arr_ok(positions_2d)) goto cleanup;
+
+ valid = mlx_array_new();
+ mlx_less(&valid, positions_2d, lengths_2d, S);
+ if (!arr_ok(valid)) goto cleanup;
+
+ valid_3d = mlx_array_new();
+ mlx_expand_dims(&valid_3d, valid, 1, S);
+ if (!arr_ok(valid_3d)) goto cleanup;
+
+ masked_sims = mlx_array_new();
+ mlx_where(&masked_sims, valid_3d, sims, invalid_value, S);
+ if (!arr_ok(masked_sims)) goto cleanup;
+
+ maxes = mlx_array_new();
+ mlx_max_axis(&maxes, masked_sims, 2, false, S);
+ if (!arr_ok(maxes)) goto cleanup;
+
+ sums = mlx_array_new();
+ mlx_sum_axis(&sums, maxes, 1, false, S);
+ if (!arr_ok(sums)) goto cleanup;
+
+ sums_f32 = mlx_array_new();
+ mlx_astype(&sums_f32, sums, MLX_FLOAT32, S);
+ if (!arr_ok(sums_f32)) goto cleanup;
+
+ mlx_array_eval(sums_f32);
+ const float *data = mlx_array_data_float32(sums_f32);
+ if (!data) goto cleanup;
+ memcpy(scores, data, (size_t)docs_count * sizeof(float));
+ rc = 0;
+
+cleanup:
+ mlx_array_free(sums_f32);
+ mlx_array_free(sums);
+ mlx_array_free(maxes);
+ mlx_array_free(masked_sims);
+ mlx_array_free(invalid_value);
+ mlx_array_free(valid_3d);
+ mlx_array_free(valid);
+ mlx_array_free(positions_2d);
+ mlx_array_free(lengths_2d);
+ mlx_array_free(positions);
+ mlx_array_free(lengths_arr);
+ mlx_array_free(sims);
+ mlx_array_free(docs_t);
+ mlx_array_free(query_batched);
+ mlx_array_free(query_3d);
+ mlx_array_free(doc_batch);
+ if (doc_vec.ctx) mlx_vector_array_free(doc_vec);
+ mlx_array_free(pad_value);
+ for (int i = 0; i < docs_count; i++)
+ mlx_array_free(doc_rows[i]);
+ free(doc_rows);
+ free(lengths);
+ return rc;
+}
+
+int pplx_mlx_late_encode_tokens(pplx_mlx_late_ctx_t *ctx,
+ const int *token_ids, int n_tokens,
+ int normalize, float *out_vectors)
+{
+ pplx_mlx_late_vectors_t *vecs =
+ pplx_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens,
+ normalize);
+ if (!vecs) return -1;
+ int rc = pplx_mlx_late_vectors_copy(vecs, out_vectors);
+ pplx_mlx_late_vectors_free(vecs);
+ return rc;
+}
+
+static int pplx_mlx_embed_batch_dense(pplx_mlx_ctx_t *ctx,
+ const pplx_input_t *inputs, int batch,
+ float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
+ ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers)
+ return -1;
+
+ const pplx_config_t *c = &ctx->config;
+ mlx_stream S = ctx->stream;
+
+ int hidden = c->hidden_size;
+ int max_seq = 0;
+ int has_padding = 0;
+
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ }
+ if (max_seq <= 0) return -1;
+
+ for (int b = 0; b < batch; b++) {
+ if (inputs[b].n_tokens != max_seq) has_padding = 1;
+ }
+
+ size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes;
+ if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 ||
+ mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 ||
+ mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 ||
+ mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 ||
+ mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 ||
+ mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
+ return -1;
+ int *padded_ids = (int *)malloc(ids_bytes);
+ float *pool_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
+ float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
+ float *len_data = has_padding ? (float *)malloc(len_bytes) : NULL;
+ if (!padded_ids || (has_padding && (!pool_mask_data || !attn_mask_data || !len_data))) {
+ free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+ return -1;
+ }
+
+ for (int b = 0; b < batch; b++) {
+ if (has_padding) len_data[b] = (float)inputs[b].n_tokens;
+ for (int t = 0; t < max_seq; t++) {
+ size_t idx = (size_t)b * max_seq + t;
+ if (t < inputs[b].n_tokens) {
+ int id = inputs[b].ids[t];
+ if (id < 0 || id >= c->vocab_size) {
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
+ id, b, t);
+ free(padded_ids); free(pool_mask_data);
+ free(attn_mask_data); free(len_data);
+ return -1;
+ }
+ padded_ids[idx] = id;
+ if (has_padding) {
+ pool_mask_data[idx] = 1.0f;
+ attn_mask_data[idx] = 0.0f;
+ }
+ } else {
+ padded_ids[idx] = 0;
+ pool_mask_data[idx] = 0.0f;
+ attn_mask_data[idx] = -1.0e9f;
+ }
+ }
+ }
+
+ int ids_shape[] = {batch, max_seq};
+ int pool_mask_shape[] = {batch, max_seq, 1};
+ int attn_mask_shape[] = {batch, 1, 1, max_seq};
+ int len_shape[] = {batch, 1};
+
+ mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
+ mlx_array pool_mask = has_padding
+ ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
+ : (mlx_array){0};
+ mlx_array attn_mask = has_padding
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ mlx_array lengths = has_padding
+ ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32)
+ : (mlx_array){0};
+ free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+
+ if (!arr_ok(ids) || (has_padding && (!arr_ok(pool_mask) ||
+ !arr_ok(attn_mask) || !arr_ok(lengths)))) {
+ mlx_array_free(ids); mlx_array_free(pool_mask);
+ if (has_padding) mlx_array_free(attn_mask);
+ mlx_array_free(lengths);
+ return -1;
+ }
+ if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
+ mlx_array attn_mask_bf16 = mlx_array_new();
+ mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
+ mlx_array_free(attn_mask);
+ attn_mask = attn_mask_bf16;
+ if (!arr_ok(attn_mask)) {
+ mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths);
+ return -1;
+ }
+ }
+
+ /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */
+ mlx_array x = mlx_array_new();
+ mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
+ mlx_array_free(ids);
+
+ /* 2. Transformer layers. */
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
+ 0, c->n_layers);
+
+ /* 3. Final RMSNorm. */
+ mlx_array x_normed = mlx_array_new();
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
+ mlx_array_free(x);
+
+ mlx_array emb = mlx_array_new();
+ if (has_padding) {
+ /* 4. Masked mean pool over T. */
+ mlx_array masked = mlx_array_new();
+ mlx_multiply(&masked, x_normed, pool_mask, S);
+ mlx_array_free(x_normed);
+ mlx_array_free(pool_mask);
+
+ mlx_array emb_sum = mlx_array_new();
+ mlx_sum_axis(&emb_sum, masked, 1, false, S);
+ mlx_array_free(masked);
+
+ mlx_divide(&emb, emb_sum, lengths, S);
+ mlx_array_free(emb_sum); mlx_array_free(lengths);
+ mlx_array_free(attn_mask);
+ } else {
+ /* Equal-length batches do not need masks or a separate divide. */
+ mlx_mean_axis(&emb, x_normed, 1, false, S);
+ mlx_array_free(x_normed);
+ }
+
+ /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are
+ * intentionally unnormalized; callers should use cosine similarity. */
+ mlx_array emb_f32 = mlx_array_new();
+ mlx_astype(&emb_f32, emb, MLX_FLOAT32, S);
+ mlx_array_free(emb);
+
+ mlx_array_eval(emb_f32);
+ const float *data = mlx_array_data_float32(emb_f32);
+ int rc = -1;
+ if (data) {
+ memcpy(out_embeddings, data, out_bytes);
+ rc = 0;
+ }
+ mlx_array_free(emb_f32);
+ return rc;
+}
+
+typedef struct {
+ int index;
+ const pplx_input_t *input;
+} mlx_batch_item_t;
+
+static int mlx_batch_item_cmp(const void *a, const void *b)
+{
+ const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a;
+ const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b;
+ int da = ia->input->n_tokens;
+ int db = ib->input->n_tokens;
+ return (da > db) - (da < db);
+}
+
+static int mlx_padding_exceeds_limit(size_t padded, size_t tokens)
+{
+ if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5)
+ return 1;
+ return padded * 5 > tokens * 11;
+}
+
+static int mlx_batch_should_split(const pplx_input_t *inputs, int batch,
+ int *out_total_tokens, int *out_max_seq)
+{
+ int max_seq = 0;
+ int total_tokens = 0;
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0;
+ total_tokens += inputs[b].n_tokens;
+ }
+ if (out_total_tokens) *out_total_tokens = total_tokens;
+ if (out_max_seq) *out_max_seq = max_seq;
+ if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0;
+
+ size_t padded;
+ if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
+ return 0;
+
+ return mlx_padding_exceeds_limit(padded, (size_t)total_tokens);
+}
+
+static int mlx_should_add_to_group(const pplx_mlx_ctx_t *ctx,
+ int group_count, int group_max,
+ int next_tokens)
+{
+ if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0)
+ return 1;
+
+ int next_count = group_count + 1;
+ int next_max = next_tokens > group_max ? next_tokens : group_max;
+ size_t merged, split_left, split_total;
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max, &merged) != 0 ||
+ mlx_mul_size((size_t)group_count, (size_t)group_max, &split_left) != 0)
+ return 0;
+ /* Treat one extra MLX launch as roughly 256 padded token rows for 4B.
+ * This preserves small mixed groups while splitting larger padded tails. */
+ if (split_left > SIZE_MAX - (size_t)next_tokens ||
+ split_left + (size_t)next_tokens > SIZE_MAX - 256)
+ return 0;
+ split_total = split_left + (size_t)next_tokens + 256;
+ return merged <= split_total;
+}
+
+int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
+
+ if (!mlx_batch_should_split(inputs, batch, NULL, NULL))
+ return pplx_mlx_embed_batch_dense(ctx, inputs, batch, out_embeddings);
+
+ const int hidden = ctx->config.hidden_size;
+ mlx_batch_item_t *items = (mlx_batch_item_t *)malloc(
+ (size_t)batch * sizeof(*items));
+ pplx_input_t *group_inputs = (pplx_input_t *)malloc(
+ (size_t)batch * sizeof(*group_inputs));
+ size_t group_values, group_bytes;
+ if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 ||
+ mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
+ free(items); free(group_inputs);
+ return -1;
+ }
+ float *group_out = (float *)malloc(group_bytes);
+ if (!items || !group_inputs || !group_out) {
+ free(items); free(group_inputs); free(group_out);
+ return -1;
+ }
+ for (int i = 0; i < batch; i++) {
+ items[i].index = i;
+ items[i].input = &inputs[i];
+ }
+ qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp);
+
+ int rc = 0;
+ for (int start = 0; start < batch && rc == 0; ) {
+ int group_max = 0;
+ int group_tokens = 0;
+ int end = start;
+ while (end < batch) {
+ int next_tokens = items[end].input->n_tokens;
+ int next_count = end - start + 1;
+ int next_max = next_tokens > group_max ? next_tokens : group_max;
+ int next_total = group_tokens + next_tokens;
+ if (end > start) {
+ size_t next_padded;
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max,
+ &next_padded) != 0 ||
+ mlx_padding_exceeds_limit(next_padded,
+ (size_t)next_total) ||
+ !mlx_should_add_to_group(ctx, end - start,
+ group_max, next_tokens))
+ break;
+ }
+ group_max = next_max;
+ group_tokens = next_total;
+ end++;
+ }
+
+ int group_count = end - start;
+ for (int i = 0; i < group_count; i++)
+ group_inputs[i] = *items[start + i].input;
+ rc = pplx_mlx_embed_batch_dense(ctx, group_inputs, group_count,
+ group_out);
+ if (rc != 0) break;
+ for (int i = 0; i < group_count; i++) {
+ int dst = items[start + i].index;
+ memcpy(out_embeddings + (size_t)dst * hidden,
+ group_out + (size_t)i * hidden,
+ (size_t)hidden * sizeof(float));
+ }
+ start = end;
+ }
+
+ free(items); free(group_inputs); free(group_out);
+ return rc;
+}
+
+int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
+ int n_tokens, float *out_embedding)
+{
+ pplx_input_t input = { token_ids, n_tokens };
+ return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding);
+}
+
+int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
+ const pplx_input_t *inputs, int batch,
+ const float *input_states,
+ int layer_start, int layer_end,
+ int apply_final_norm,
+ float *out_states)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_states ||
+ layer_start < 0 || layer_start > layer_end ||
+ layer_end > ctx->config.n_layers ||
+ layer_start < ctx->layer_start || layer_end > ctx->layer_end ||
+ (layer_start == 0 && input_states) ||
+ (layer_start != 0 && !input_states) ||
+ (layer_start == 0 && !arr_ok(ctx->embed_tokens)) ||
+ (apply_final_norm &&
+ (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm))))
+ return -1;
+
+ const pplx_config_t *c = &ctx->config;
+ mlx_stream S = ctx->stream;
+ int hidden = c->hidden_size;
+ int max_seq = 0;
+ int total_seq = 0;
+ int has_padding = 0;
+ for (int b = 0; b < batch; b++) {
+ if ((layer_start == 0 && !inputs[b].ids) || inputs[b].n_tokens <= 0 ||
+ total_seq > INT_MAX - inputs[b].n_tokens)
+ return -1;
+ total_seq += inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ }
+ for (int b = 0; b < batch; b++) {
+ if (inputs[b].n_tokens != max_seq) has_padding = 1;
+ }
+
+ size_t rows, ids_bytes, mask_bytes, state_bytes, row_bytes;
+ if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 ||
+ mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 ||
+ mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 ||
+ mlx_mul_size((size_t)hidden, sizeof(float), &row_bytes) != 0)
+ return -1;
+ if (rows == 0 || row_bytes == 0 || (has_padding && mask_bytes == 0))
+ return -1;
+ if (layer_start != 0 &&
+ mlx_mul_size(rows, (size_t)hidden, &state_bytes) != 0)
+ return -1;
+ if (layer_start == 0)
+ state_bytes = 0;
+
+ int *padded_ids = layer_start == 0
+ ? (int *)malloc(ids_bytes) : NULL;
+ float *padded_states = layer_start != 0
+ ? (float *)calloc(1, state_bytes) : NULL;
+ float *attn_mask_data = has_padding
+ ? (float *)malloc(mask_bytes) : NULL;
+ if ((layer_start == 0 && !padded_ids) ||
+ (layer_start != 0 && !padded_states) ||
+ (has_padding && !attn_mask_data)) {
+ free(padded_ids); free(padded_states); free(attn_mask_data);
+ return -1;
+ }
+
+ int packed_offset = 0;
+ for (int b = 0; b < batch; b++) {
+ for (int t = 0; t < max_seq; t++) {
+ size_t dense_row = (size_t)b * max_seq + t;
+ if (t < inputs[b].n_tokens) {
+ if (layer_start == 0) {
+ int id = inputs[b].ids[t];
+ if (id < 0 || id >= c->vocab_size) {
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
+ id, b, t);
+ free(padded_ids); free(padded_states);
+ free(attn_mask_data);
+ return -1;
+ }
+ padded_ids[dense_row] = id;
+ } else {
+ memcpy(padded_states + dense_row * hidden,
+ input_states + (size_t)(packed_offset + t) * hidden,
+ row_bytes);
+ }
+ if (has_padding) attn_mask_data[dense_row] = 0.0f;
+ } else {
+ if (layer_start == 0) padded_ids[dense_row] = 0;
+ if (has_padding) attn_mask_data[dense_row] = -1.0e9f;
+ }
+ }
+ packed_offset += inputs[b].n_tokens;
+ }
+
+ int attn_mask_shape[] = {batch, 1, 1, max_seq};
+ mlx_array attn_mask = has_padding
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ free(attn_mask_data);
+ if (has_padding && !arr_ok(attn_mask)) {
+ free(padded_ids); free(padded_states);
+ return -1;
+ }
+ if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
+ mlx_array attn_mask_bf16 = mlx_array_new();
+ mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
+ mlx_array_free(attn_mask);
+ attn_mask = attn_mask_bf16;
+ if (!arr_ok(attn_mask)) {
+ free(padded_ids); free(padded_states);
+ return -1;
+ }
+ }
+
+ mlx_array x;
+ if (layer_start == 0) {
+ int ids_shape[] = {batch, max_seq};
+ mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
+ free(padded_ids);
+ if (!arr_ok(ids)) {
+ if (has_padding) mlx_array_free(attn_mask);
+ return -1;
+ }
+ x = mlx_array_new();
+ mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
+ mlx_array_free(ids);
+ } else {
+ int states_shape[] = {batch, max_seq, hidden};
+ mlx_array states = mlx_array_new_data(padded_states, states_shape, 3,
+ MLX_FLOAT32);
+ free(padded_states);
+ if (!arr_ok(states)) {
+ if (has_padding) mlx_array_free(attn_mask);
+ return -1;
+ }
+ if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
+ x = mlx_array_new();
+ mlx_astype(&x, states, MLX_BFLOAT16, S);
+ mlx_array_free(states);
+ } else {
+ x = states;
+ }
+ }
+
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
+ layer_start, layer_end);
+ if (has_padding) mlx_array_free(attn_mask);
+
+ if (apply_final_norm) {
+ mlx_array x_normed = mlx_array_new();
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
+ mlx_array_free(x);
+ x = x_normed;
+ }
+
+ mlx_array x_f32 = mlx_array_new();
+ mlx_astype(&x_f32, x, MLX_FLOAT32, S);
+ mlx_array_free(x);
+ mlx_array_eval(x_f32);
+
+ const float *data = mlx_array_data_float32(x_f32);
+ if (!data) {
+ mlx_array_free(x_f32);
+ return -1;
+ }
+ packed_offset = 0;
+ for (int b = 0; b < batch; b++) {
+ memcpy(out_states + (size_t)packed_offset * hidden,
+ data + (size_t)b * max_seq * hidden,
+ (size_t)inputs[b].n_tokens * row_bytes);
+ packed_offset += inputs[b].n_tokens;
+ }
+ mlx_array_free(x_f32);
+ return 0;
+}
+
+typedef struct {
+ int index;
+ int span_offset;
+ const pplx_context_input_t *input;
+} mlx_context_batch_item_t;
+
+static int mlx_context_batch_item_cmp(const void *a, const void *b)
+{
+ const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a;
+ const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b;
+ int da = ia->input->input.n_tokens;
+ int db = ib->input->input.n_tokens;
+ return (da > db) - (da < db);
+}
+
+static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens)
+{
+ if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4)
+ return 1;
+ return padded * 4 > tokens * 5;
+}
+
+static int mlx_context_batch_should_split(const pplx_context_input_t *inputs,
+ int batch, int *out_total_spans)
+{
+ int max_seq = 0;
+ int total_tokens = 0;
+ int total_spans = 0;
+ for (int b = 0; b < batch; b++) {
+ int n_tokens = inputs[b].input.n_tokens;
+ if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans ||
+ inputs[b].n_spans <= 0)
+ return 0;
+ if (n_tokens > max_seq) max_seq = n_tokens;
+ if (total_tokens > INT_MAX - n_tokens ||
+ total_spans > INT_MAX - inputs[b].n_spans)
+ return 0;
+ total_tokens += n_tokens;
+ total_spans += inputs[b].n_spans;
+ }
+ if (out_total_spans) *out_total_spans = total_spans;
+ if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0;
+
+ size_t padded;
+ if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
+ return 0;
+ return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens);
+}
+
+static int pplx_mlx_embed_spans_batch_dense(pplx_mlx_ctx_t *ctx,
+ const pplx_context_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
+ ctx->layer_start != 0 ||
+ ctx->layer_end != ctx->config.n_layers ||
+ !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
+ return -1;
+
+ const pplx_config_t *c = &ctx->config;
+ mlx_stream S = ctx->stream;
+ int hidden = c->hidden_size;
+ int max_seq = 0;
+ int total_spans = 0;
+ int has_padding = 0;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ int n_tokens = input->input.n_tokens;
+ if (!input->input.ids || n_tokens <= 0 || !input->spans ||
+ input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans)
+ return -1;
+ total_spans += input->n_spans;
+ if (n_tokens > max_seq) max_seq = n_tokens;
+ for (int t = 0; t < n_tokens; t++) {
+ if (input->input.ids[t] < 0 ||
+ input->input.ids[t] >= c->vocab_size) {
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
+ input->input.ids[t], b, t);
+ return -1;
+ }
+ }
+ for (int s = 0; s < input->n_spans; s++) {
+ if (input->spans[s].start < 0 ||
+ input->spans[s].start > n_tokens ||
+ input->spans[s].n_tokens <= 0 ||
+ input->spans[s].n_tokens > n_tokens - input->spans[s].start)
+ return -1;
+ }
+ }
+ for (int b = 0; b < batch; b++)
+ if (inputs[b].input.n_tokens != max_seq) has_padding = 1;
+
+ size_t rows, ids_bytes, mask_bytes, index_bytes;
+ size_t span_float_bytes, out_values, out_bytes;
+ if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 ||
+ mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 ||
+ mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 ||
+ mlx_mul_size((size_t)total_spans, sizeof(int), &index_bytes) != 0 ||
+ mlx_mul_size((size_t)total_spans, sizeof(float), &span_float_bytes) != 0 ||
+ mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 ||
+ mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
+ return -1;
+ if (rows > INT_MAX) return -1;
+ int *padded_ids = (int *)malloc(ids_bytes);
+ float *attn_mask_data = has_padding
+ ? (float *)malloc(mask_bytes) : NULL;
+ if (!padded_ids || (has_padding && !attn_mask_data)) {
+ free(padded_ids);
+ free(attn_mask_data);
+ return -1;
+ }
+ for (int b = 0; b < batch; b++) {
+ const pplx_input_t *input = &inputs[b].input;
+ for (int t = 0; t < max_seq; t++) {
+ size_t row = (size_t)b * max_seq + t;
+ if (t < input->n_tokens) {
+ padded_ids[row] = input->ids[t];
+ if (has_padding) attn_mask_data[row] = 0.0f;
+ } else {
+ padded_ids[row] = 0;
+ if (has_padding) attn_mask_data[row] = -1.0e9f;
+ }
+ }
+ }
+
+ mlx_array ids = (mlx_array){0};
+ mlx_array attn_mask = (mlx_array){0};
+ mlx_array x = (mlx_array){0};
+ mlx_array x_normed = (mlx_array){0};
+ mlx_array prefix = (mlx_array){0};
+ mlx_array flat_prefix = (mlx_array){0};
+ mlx_array end_idx = (mlx_array){0};
+ mlx_array start_idx = (mlx_array){0};
+ mlx_array lengths = (mlx_array){0};
+ mlx_array start_mask = (mlx_array){0};
+ mlx_array end_values = (mlx_array){0};
+ mlx_array start_values = (mlx_array){0};
+ mlx_array masked_start = (mlx_array){0};
+ mlx_array span_sums = (mlx_array){0};
+ mlx_array embeddings = (mlx_array){0};
+ mlx_array embeddings_f32 = (mlx_array){0};
+ mlx_array *pooled = NULL;
+ mlx_vector_array pooled_vec = (mlx_vector_array){0};
+ int *end_idx_data = NULL;
+ int *start_idx_data = NULL;
+ float *lengths_data = NULL;
+ float *start_mask_data = NULL;
+ int rc = -1;
+
+ int ids_shape[] = {batch, max_seq};
+ int attn_mask_shape[] = {batch, 1, 1, max_seq};
+ ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
+ attn_mask = has_padding
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ free(padded_ids);
+ free(attn_mask_data);
+ if (!arr_ok(ids)) goto cleanup;
+ if (has_padding && !arr_ok(attn_mask)) goto cleanup;
+ if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
+ mlx_array attn_mask_bf16 = mlx_array_new();
+ mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
+ mlx_array_free(attn_mask);
+ attn_mask = attn_mask_bf16;
+ if (!arr_ok(attn_mask)) goto cleanup;
+ }
+
+ x = mlx_array_new();
+ mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
+ mlx_array_free(ids);
+ ids = (mlx_array){0};
+ if (!arr_ok(x)) goto cleanup;
+
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
+ 0, c->n_layers);
+ if (has_padding) {
+ mlx_array_free(attn_mask);
+ attn_mask = (mlx_array){0};
+ }
+ if (!arr_ok(x)) goto cleanup;
+
+ x_normed = mlx_array_new();
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
+ mlx_array_free(x);
+ x = (mlx_array){0};
+ if (!arr_ok(x_normed)) goto cleanup;
+
+ if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
+ pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled));
+ if (!pooled) goto cleanup;
+
+ int pooled_count = 0;
+ for (int b = 0; b < batch; b++) {
+ for (int s = 0; s < inputs[b].n_spans; s++) {
+ const pplx_span_t *span = &inputs[b].spans[s];
+ int start[] = {b, span->start, 0};
+ int stop[] = {b + 1, span->start + span->n_tokens, hidden};
+ int strides[] = {1, 1, 1};
+ mlx_array slice = mlx_array_new();
+ mlx_slice(&slice, x_normed, start, 3, stop, 3, strides, 3, S);
+ if (!arr_ok(slice)) {
+ mlx_array_free(slice);
+ goto cleanup;
+ }
+
+ pooled[pooled_count] = mlx_array_new();
+ mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S);
+ mlx_array_free(slice);
+ if (!arr_ok(pooled[pooled_count])) goto cleanup;
+ pooled_count++;
+ }
+ }
+
+ pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans);
+ if (!pooled_vec.ctx) goto cleanup;
+ embeddings = mlx_array_new();
+ mlx_concatenate_axis(&embeddings, pooled_vec, 0, S);
+ if (!arr_ok(embeddings)) goto cleanup;
+ } else {
+ end_idx_data = (int *)malloc(index_bytes);
+ start_idx_data = (int *)malloc(index_bytes);
+ lengths_data = (float *)malloc(span_float_bytes);
+ start_mask_data = (float *)malloc(span_float_bytes);
+ if (!end_idx_data || !start_idx_data || !lengths_data ||
+ !start_mask_data)
+ goto cleanup;
+
+ int span_count = 0;
+ for (int b = 0; b < batch; b++) {
+ for (int s = 0; s < inputs[b].n_spans; s++) {
+ const pplx_span_t *span = &inputs[b].spans[s];
+ int base = b * max_seq;
+ int start = span->start;
+ int stop = span->start + span->n_tokens;
+ end_idx_data[span_count] = base + stop - 1;
+ start_idx_data[span_count] = start > 0 ? base + start - 1 : base;
+ lengths_data[span_count] = (float)span->n_tokens;
+ start_mask_data[span_count] = start > 0 ? 1.0f : 0.0f;
+ span_count++;
+ }
+ }
+
+ int index_shape[] = {total_spans};
+ int scale_shape[] = {total_spans, 1};
+ end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32);
+ start_idx = mlx_array_new_data(start_idx_data, index_shape, 1,
+ MLX_INT32);
+ lengths = mlx_array_new_data(lengths_data, scale_shape, 2,
+ MLX_FLOAT32);
+ start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2,
+ MLX_FLOAT32);
+ free(end_idx_data); end_idx_data = NULL;
+ free(start_idx_data); start_idx_data = NULL;
+ free(lengths_data); lengths_data = NULL;
+ free(start_mask_data); start_mask_data = NULL;
+ if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) ||
+ !arr_ok(start_mask))
+ goto cleanup;
+
+ prefix = mlx_array_new();
+ mlx_cumsum(&prefix, x_normed, 1, false, true, S);
+ mlx_array_free(x_normed);
+ x_normed = (mlx_array){0};
+ if (!arr_ok(prefix)) goto cleanup;
+
+ int flat_shape[] = {(int)rows, hidden};
+ flat_prefix = mlx_array_new();
+ mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S);
+ mlx_array_free(prefix);
+ prefix = (mlx_array){0};
+ if (!arr_ok(flat_prefix)) goto cleanup;
+
+ end_values = mlx_array_new();
+ mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S);
+ if (!arr_ok(end_values)) goto cleanup;
+
+ start_values = mlx_array_new();
+ mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S);
+ if (!arr_ok(start_values)) goto cleanup;
+
+ masked_start = mlx_array_new();
+ mlx_multiply(&masked_start, start_values, start_mask, S);
+ if (!arr_ok(masked_start)) goto cleanup;
+
+ span_sums = mlx_array_new();
+ mlx_subtract(&span_sums, end_values, masked_start, S);
+ if (!arr_ok(span_sums)) goto cleanup;
+
+ embeddings = mlx_array_new();
+ mlx_divide(&embeddings, span_sums, lengths, S);
+ if (!arr_ok(embeddings)) goto cleanup;
+ }
+
+ embeddings_f32 = mlx_array_new();
+ mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S);
+ if (!arr_ok(embeddings_f32)) goto cleanup;
+
+ mlx_array_eval(embeddings_f32);
+ const float *data = mlx_array_data_float32(embeddings_f32);
+ if (data) {
+ memcpy(out_embeddings, data, out_bytes);
+ rc = 0;
+ }
+
+cleanup:
+ free(start_mask_data);
+ free(lengths_data);
+ free(start_idx_data);
+ free(end_idx_data);
+ mlx_array_free(embeddings_f32);
+ mlx_array_free(embeddings);
+ if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec);
+ if (pooled) {
+ for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]);
+ free(pooled);
+ }
+ mlx_array_free(span_sums);
+ mlx_array_free(masked_start);
+ mlx_array_free(start_values);
+ mlx_array_free(end_values);
+ mlx_array_free(start_mask);
+ mlx_array_free(lengths);
+ mlx_array_free(start_idx);
+ mlx_array_free(end_idx);
+ mlx_array_free(flat_prefix);
+ mlx_array_free(prefix);
+ mlx_array_free(x_normed);
+ mlx_array_free(x);
+ mlx_array_free(attn_mask);
+ mlx_array_free(ids);
+ return rc;
+}
+
+int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx,
+ const pplx_context_input_t *inputs, int batch,
+ float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
+
+ int total_spans = 0;
+ if (!mlx_context_batch_should_split(inputs, batch, &total_spans))
+ return pplx_mlx_embed_spans_batch_dense(ctx, inputs, batch,
+ out_embeddings);
+
+ const int hidden = ctx->config.hidden_size;
+ mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc(
+ (size_t)batch * sizeof(*items));
+ pplx_context_input_t *group_inputs = (pplx_context_input_t *)malloc(
+ (size_t)batch * sizeof(*group_inputs));
+ size_t group_values, group_bytes;
+ if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 ||
+ mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
+ free(items); free(group_inputs);
+ return -1;
+ }
+ float *group_out = (float *)malloc(group_bytes);
+ if (!items || !group_inputs || !group_out) {
+ free(items); free(group_inputs); free(group_out);
+ return -1;
+ }
+
+ int span_offset = 0;
+ for (int i = 0; i < batch; i++) {
+ items[i].index = i;
+ items[i].span_offset = span_offset;
+ items[i].input = &inputs[i];
+ span_offset += inputs[i].n_spans;
+ }
+ qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp);
+
+ int rc = 0;
+ for (int start = 0; start < batch && rc == 0; ) {
+ int group_max = 0;
+ int group_tokens = 0;
+ int end = start;
+ while (end < batch) {
+ int next_tokens = items[end].input->input.n_tokens;
+ int next_count = end - start + 1;
+ int next_max = next_tokens > group_max ? next_tokens : group_max;
+ int next_total = group_tokens + next_tokens;
+ if (end > start) {
+ size_t next_padded;
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max,
+ &next_padded) != 0 ||
+ mlx_context_padding_exceeds_limit(next_padded,
+ (size_t)next_total) ||
+ !mlx_should_add_to_group(ctx, end - start,
+ group_max, next_tokens))
+ break;
+ }
+ group_max = next_max;
+ group_tokens = next_total;
+ end++;
+ }
+
+ int group_count = end - start;
+ for (int i = 0; i < group_count; i++)
+ group_inputs[i] = *items[start + i].input;
+ rc = pplx_mlx_embed_spans_batch_dense(ctx, group_inputs,
+ group_count, group_out);
+ if (rc != 0) break;
+ int src_span = 0;
+ for (int i = 0; i < group_count; i++) {
+ int n_spans = group_inputs[i].n_spans;
+ int dst_span = items[start + i].span_offset;
+ memcpy(out_embeddings + (size_t)dst_span * hidden,
+ group_out + (size_t)src_span * hidden,
+ (size_t)n_spans * hidden * sizeof(float));
+ src_span += n_spans;
+ }
+ start = end;
+ }
+
+ free(items); free(group_inputs); free(group_out);
+ return rc;
+}
+
+int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
+ int n_tokens, const pplx_span_t *spans,
+ int n_spans, float *out_embeddings)
+{
+ pplx_context_input_t input = {
+ .input = { token_ids, n_tokens },
+ .spans = spans,
+ .n_spans = n_spans,
+ };
+ return pplx_mlx_embed_spans_batch(ctx, &input, 1, out_embeddings);
+}
+
+float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ int hidden = ctx->config.hidden_size;
+ size_t out_bytes;
+ if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0)
+ return NULL;
+ float *out = (float *)malloc(out_bytes);
+ if (!out) return NULL;
+
+ if (pplx_mlx_embed_into(ctx, token_ids, n_tokens, out) != 0) {
+ free(out);
+ return NULL;
+ }
+ return out;
+}