diff options
Diffstat (limited to 'src/mlx.c')
| -rw-r--r-- | src/mlx.c | 2752 |
1 files changed, 2752 insertions, 0 deletions
diff --git a/src/mlx.c b/src/mlx.c new file mode 100644 index 0000000..bfa466c --- /dev/null +++ b/src/mlx.c @@ -0,0 +1,2752 @@ +#include "mlx.h" +#include "config.h" +#include "embed.h" +#include "safetensors.h" + +#include <mlx/c/mlx.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <math.h> +#include <limits.h> +#include <float.h> + +/* ======================================================================== + * Per-layer MLX weight arrays + * ======================================================================== */ + +typedef struct { + mlx_array w; + mlx_array wt; + mlx_array scales; + mlx_array biases; + int quantized; + int bits; + int group_size; +} mlx_linear_t; + +typedef struct { + mlx_linear_t wq, wk, wv, wo; + mlx_array q_bias, k_bias, v_bias; + mlx_array q_norm, k_norm; + mlx_array input_norm, post_attn_norm; + mlx_linear_t gate_proj, up_proj, down_proj; + /* BERT family (unset for Qwen3; see embed_internal.h). wo/up_proj/down_proj + * and input_norm/post_attn_norm are reused as the BERT attention output, + * FFN matrices, and the two block LayerNorm weights. */ + mlx_array o_bias, attn_ln_bias, ffn_inter_bias, ffn_out_bias, ffn_ln_bias; +} mlx_layer_t; + +struct embed_mlx_ctx { + embed_config_t config; + mlx_array embed_tokens; + mlx_layer_t *layers; /* heap-allocated [n_layers] */ + mlx_array norm; + /* BERT family (unset for Qwen3): learned positions, token-type, embed LN. */ + mlx_array position_embeddings, token_type_embeddings, embed_ln_w, embed_ln_b; + mlx_stream stream; + multi_safetensors_t *ms; + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ + int quantize_bits; + int quantize_group_size; + mlx_dtype weight_dtype; +}; + +struct embed_mlx_late_ctx { + embed_mlx_ctx_t *base; + multi_safetensors_t *dense_ms; + mlx_linear_t projection; + int token_dim; +}; + +struct embed_mlx_late_vectors { + embed_mlx_late_ctx_t *owner; + mlx_array vectors; /* [tokens, dim] */ + int tokens; + int dim; +}; + +/* ======================================================================== + * Helpers + * ======================================================================== */ + +static int arr_ok(mlx_array a) { return a.ctx != NULL; } + +static int mlx_mul_size(size_t a, size_t b, size_t *out) { + if (a != 0 && b > SIZE_MAX / a) + return -1; + *out = a * b; + return 0; +} + +static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; } + +static size_t safetensor_dtype_size(safetensor_dtype_t dtype) { + switch (dtype) { + case DTYPE_F32: + return sizeof(float); + case DTYPE_BF16: + return sizeof(uint16_t); + default: + return 0; + } +} + +static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, + const safetensor_t *t, + const char *name, + const int *shape, + int ndim) { + size_t elem_size = safetensor_dtype_size(t->dtype); + if (elem_size == 0) { + fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); + return 0; + } + if (t->ndim != ndim) { + fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); + return 0; + } + + size_t numel = 1; + for (int i = 0; i < ndim; i++) { + if (shape[i] <= 0 || t->shape[i] != shape[i]) { + fprintf(stderr, + "mlx: bad shape for %s at dim %d: " + "got %lld, expected %d\n", + name, i, (long long)t->shape[i], shape[i]); + return 0; + } + if (mlx_mul_size(numel, (size_t)shape[i], &numel) != 0) { + fprintf(stderr, "mlx: shape too large for %s\n", name); + return 0; + } + } + + size_t bytes; + if (mlx_mul_size(numel, elem_size, &bytes) != 0) { + fprintf(stderr, "mlx: tensor data too large for %s\n", name); + return 0; + } + if (t->data_size != bytes) { + fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, + bytes); + return 0; + } + + if (sf->header_size > SIZE_MAX - 8) + return 0; + size_t data_start = 8 + sf->header_size; + if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || + t->data_size > sf->file_size - data_start - t->data_offset) { + fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name); + return 0; + } + return 1; +} + +static mlx_array +load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { + safetensors_file_t *sf = NULL; + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + if (!t) { + fprintf(stderr, "mlx: tensor not found: %s\n", name); + return (mlx_array){0}; + } + if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim)) + return (mlx_array){0}; + mlx_dtype dtype; + if (t->dtype == DTYPE_F32) { + dtype = MLX_FLOAT32; + } else if (t->dtype == DTYPE_BF16) { + dtype = MLX_BFLOAT16; + } else { + fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); + return (mlx_array){0}; + } + return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); +} + +static mlx_linear_t +load_linear_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { + mlx_linear_t l = {0}; + l.w = load_tensor(ms, name, shape, ndim); + return l; +} + +static int linear_ok(mlx_linear_t l) { return arr_ok(l.w) || arr_ok(l.wt); } + +static void free_linear(mlx_linear_t *l) { + mlx_array_free(l->w); + mlx_array_free(l->wt); + mlx_array_free(l->scales); + mlx_array_free(l->biases); + memset(l, 0, sizeof(*l)); +} + +static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) { + if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt)) + return 0; + + mlx_array transposed = mlx_array_new(); + mlx_transpose(&transposed, l->w, s); + if (!arr_ok(transposed)) { + mlx_array_free(transposed); + fprintf(stderr, "mlx: failed to prepare transposed linear weight\n"); + return -1; + } + + l->wt = mlx_array_new(); + mlx_contiguous(&l->wt, transposed, false, s); + mlx_array_free(transposed); + if (!arr_ok(l->wt)) { + mlx_array_free(l->wt); + l->wt = (mlx_array){0}; + fprintf(stderr, "mlx: failed to materialize transposed linear weight\n"); + return -1; + } + mlx_array_eval(l->wt); + mlx_synchronize(s); + mlx_array_free(l->w); + l->w = (mlx_array){0}; + return 0; +} + +static int quantize_linear(mlx_linear_t *l, int bits, int group_size, mlx_stream s) { + if (!l || !arr_ok(l->w) || bits == 0) + return 0; + if (bits != 8) { + fprintf(stderr, "mlx: quantized weights support only 8 bits\n"); + return -1; + } + if (group_size <= 0) + group_size = 64; + + mlx_optional_int gs = {.value = group_size, .has_value = true}; + mlx_optional_int qb = {.value = bits, .has_value = true}; + mlx_array null_arr = (mlx_array){0}; + mlx_vector_array q = mlx_vector_array_new(); + mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s); + if (!q.ctx || mlx_vector_array_size(q) < 2) { + mlx_vector_array_free(q); + fprintf(stderr, "mlx: quantize failed\n"); + return -1; + } + + mlx_array qw = (mlx_array){0}; + mlx_array scales = (mlx_array){0}; + mlx_array biases = (mlx_array){0}; + mlx_vector_array_get(&qw, q, 0); + mlx_vector_array_get(&scales, q, 1); + if (mlx_vector_array_size(q) >= 3) + mlx_vector_array_get(&biases, q, 2); + mlx_vector_array_free(q); + + if (!arr_ok(qw) || !arr_ok(scales)) { + mlx_array_free(qw); + mlx_array_free(scales); + mlx_array_free(biases); + fprintf(stderr, "mlx: quantize returned invalid arrays\n"); + return -1; + } + mlx_array_eval(qw); + mlx_array_eval(scales); + if (arr_ok(biases)) + mlx_array_eval(biases); + mlx_synchronize(s); + + mlx_array_free(l->w); + mlx_array_free(l->wt); + l->w = qw; + l->wt = (mlx_array){0}; + l->scales = scales; + l->biases = biases; + l->quantized = 1; + l->bits = bits; + l->group_size = group_size; + return 0; +} + +/* y = x @ W^T */ +static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) { + if (W->quantized) { + if (!arr_ok(W->w)) + return (mlx_array){0}; + mlx_array res = mlx_array_new(); + mlx_optional_int gs = {.value = W->group_size, .has_value = true}; + mlx_optional_int bits = {.value = W->bits, .has_value = true}; + mlx_array null_arr = (mlx_array){0}; + mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr, + true, gs, bits, "affine", s); + return res; + } + mlx_array Wt; + int free_wt = 0; + if (arr_ok(W->wt)) { + Wt = W->wt; + } else if (arr_ok(W->w)) { + Wt = mlx_array_new(); + mlx_transpose(&Wt, W->w, s); + free_wt = 1; + } else { + return (mlx_array){0}; + } + mlx_array res = mlx_array_new(); + if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && mlx_array_dtype(x) == MLX_FLOAT32) { + /* F32 activations with BF16 weights: upcast the weight operand and run + * the matmul in F32 (so accumulation is F32). Only the weights are + * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers), + * matching the CPU/CUDA accuracy while keeping BF16 weight storage. */ + mlx_array Wf = mlx_array_new(); + mlx_astype(&Wf, Wt, MLX_FLOAT32, s); + mlx_matmul(&res, x, Wf, s); + mlx_array_free(Wf); + } else { + mlx_matmul(&res, x, Wt, s); + } + if (free_wt) + mlx_array_free(Wt); + return res; +} + +const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) { + return ctx ? &ctx->config : NULL; +} + +/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the + * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the + * residual stream in F32 while keeping the large matmul weights in BF16. */ +static void upcast_f32_inplace(mlx_array *a, mlx_stream s) { + if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) + return; + mlx_array f = mlx_array_new(); + mlx_astype(&f, *a, MLX_FLOAT32, s); + mlx_array_free(*a); + *a = f; +} + +/* ======================================================================== + * Load + * ======================================================================== */ + +static int mlx_model_dir_has_file(const char *model_dir, const char *rel) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; + FILE *f = fopen(path, "rb"); + if (!f) + return 0; + fclose(f); + return 1; +} + +static int mlx_model_dir_has_late_projection(const char *model_dir) { + return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") && + mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors"); +} + +static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) { + embed_mlx_options_t out = {0}; + if (opts) + out = *opts; + if (out.quantize_bits != 0 && out.quantize_bits != 8) { + fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); + out.quantize_bits = -1; + } + if (out.quantize_bits && out.quantize_group_size <= 0) + out.quantize_group_size = 64; + return out; +} + +static int quantize_layer(mlx_layer_t *l, int bits, int group_size, mlx_stream s) { + if (bits == 0) + return 0; + return quantize_linear(&l->wq, bits, group_size, s) || + quantize_linear(&l->wk, bits, group_size, s) || + quantize_linear(&l->wv, bits, group_size, s) || + quantize_linear(&l->wo, bits, group_size, s) || + quantize_linear(&l->gate_proj, bits, group_size, s) || + quantize_linear(&l->up_proj, bits, group_size, s) || + quantize_linear(&l->down_proj, bits, group_size, s); +} + +static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) { + return prepare_linear_transpose(&l->wq, s) || prepare_linear_transpose(&l->wk, s) || + prepare_linear_transpose(&l->wv, s) || prepare_linear_transpose(&l->wo, s) || + prepare_linear_transpose(&l->gate_proj, s) || prepare_linear_transpose(&l->up_proj, s) || + prepare_linear_transpose(&l->down_proj, s); +} + +/* BERT-family MLX loader, mirroring load_bert_weights on the CPU side. The + * shared post-load below (stream, transposes, optional BF16 upcast) runs after + * either family fills ctx->layers. */ +static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int layer_end) { + const embed_config_t *c = &ctx->config; + int h = c->hidden_size, inter = c->intermediate_size; + char name[256]; + + const char *p; + if (multi_safetensors_find(ctx->ms, "embeddings.word_embeddings.weight", NULL)) + p = ""; + else if (multi_safetensors_find(ctx->ms, "bert.embeddings.word_embeddings.weight", NULL)) + p = "bert."; + else { + fprintf(stderr, "mlx: BERT word embeddings not found\n"); + return -1; + } + + if (layer_start == 0) { + int emb_shape[] = {c->vocab_size, h}; + snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p); + ctx->embed_tokens = load_tensor(ctx->ms, name, emb_shape, 2); + int pos_shape[] = {c->max_position_embeddings, h}; + snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p); + ctx->position_embeddings = load_tensor(ctx->ms, name, pos_shape, 2); + int type_shape[] = {2, h}; + snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p); + ctx->token_type_embeddings = load_tensor(ctx->ms, name, type_shape, 2); + int ln_shape[] = {h}; + snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p); + ctx->embed_ln_w = load_tensor(ctx->ms, name, ln_shape, 1); + snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); + ctx->embed_ln_b = load_tensor(ctx->ms, name, ln_shape, 1); + if (!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->position_embeddings) || + !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->embed_ln_w) || + !arr_ok(ctx->embed_ln_b)) + return -1; + } + + ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); + if (!ctx->layers) + return -1; + + for (int i = layer_start; i < layer_end; i++) { + mlx_layer_t *l = &ctx->layers[i]; +#define LDL(fld, suffix, ...) \ + do { \ + snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!linear_ok(l->fld)) \ + return -1; \ + } while (0) +#define LDV(fld, suffix, ...) \ + do { \ + snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!arr_ok(l->fld)) \ + return -1; \ + } while (0) + + LDL(wq, "attention.self.query.weight", h, h); + LDV(q_bias, "attention.self.query.bias", h); + LDL(wk, "attention.self.key.weight", h, h); + LDV(k_bias, "attention.self.key.bias", h); + LDL(wv, "attention.self.value.weight", h, h); + LDV(v_bias, "attention.self.value.bias", h); + LDL(wo, "attention.output.dense.weight", h, h); + LDV(o_bias, "attention.output.dense.bias", h); + LDV(input_norm, "attention.output.LayerNorm.weight", h); + LDV(attn_ln_bias, "attention.output.LayerNorm.bias", h); + LDL(up_proj, "intermediate.dense.weight", inter, h); + LDV(ffn_inter_bias, "intermediate.dense.bias", inter); + LDL(down_proj, "output.dense.weight", h, inter); + LDV(ffn_out_bias, "output.dense.bias", h); + LDV(post_attn_norm, "output.LayerNorm.weight", h); + LDV(ffn_ln_bias, "output.LayerNorm.bias", h); +#undef LDL +#undef LDV + } + return 0; +} + +static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, + int layer_start, + int layer_end, + const embed_mlx_options_t *opts, + int allow_late) { + embed_mlx_options_t options = normalize_mlx_options(opts); + if (options.quantize_bits < 0) + return NULL; + if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "mlx: late-interaction models require token-level " + "MaxSim support and are not valid pooled embedding models " + "yet\n"); + return NULL; + } + + embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t)); + if (!ctx) + return NULL; + ctx->quantize_bits = options.quantize_bits; + ctx->quantize_group_size = options.quantize_group_size; + + if (embed_config_parse(&ctx->config, model_dir) != 0) { + free(ctx); + return NULL; + } + + const embed_config_t *c = &ctx->config; + if (layer_end == -1) + layer_end = c->n_layers; + if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) { + fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", layer_start, + layer_end, c->n_layers); + free(ctx); + return NULL; + } + ctx->layer_start = layer_start; + ctx->layer_end = layer_end; + + ctx->ms = multi_safetensors_open(model_dir); + if (!ctx->ms) { + fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); + free(ctx); + return NULL; + } + + if (c->family == EMBED_FAMILY_BERT) { + if (load_mlx_bert_weights(ctx, layer_start, layer_end) != 0) + goto fail; + goto loaded; + } + + const char *weight_prefix = multi_safetensors_weight_prefix(ctx->ms, "embed_tokens.weight"); + if (!weight_prefix) { + fprintf(stderr, "mlx: neither embed_tokens.weight nor " + "model.embed_tokens.weight was found\n"); + goto fail; + } + + int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; + int inter = c->intermediate_size, hd = c->head_dim; + char name[256]; + + /* Only the first stage needs token embeddings. */ + if (layer_start == 0) { + int emb_shape[] = {c->vocab_size, h}; + snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix); + ctx->embed_tokens = load_tensor(ctx->ms, name, emb_shape, 2); + if (!arr_ok(ctx->embed_tokens)) + goto fail; + } + + /* Layers */ + ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); + if (!ctx->layers) + goto fail; + + for (int i = layer_start; i < layer_end; i++) { + mlx_layer_t *l = &ctx->layers[i]; + +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, weight_prefix, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!linear_ok(l->fld)) \ + goto fail; \ + } while (0) + + LD(wq, "%slayers.%d.self_attn.q_proj.weight", qd, h); + LD(wk, "%slayers.%d.self_attn.k_proj.weight", kvd, h); + LD(wv, "%slayers.%d.self_attn.v_proj.weight", kvd, h); + LD(wo, "%slayers.%d.self_attn.o_proj.weight", h, qd); +#undef LD + +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, weight_prefix, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!arr_ok(l->fld)) \ + goto fail; \ + } while (0) + + if (c->qkv_bias) { + LD(q_bias, "%slayers.%d.self_attn.q_proj.bias", qd); + LD(k_bias, "%slayers.%d.self_attn.k_proj.bias", kvd); + LD(v_bias, "%slayers.%d.self_attn.v_proj.bias", kvd); + } + if (c->qk_norm) { + LD(q_norm, "%slayers.%d.self_attn.q_norm.weight", hd); + LD(k_norm, "%slayers.%d.self_attn.k_norm.weight", hd); + } + LD(input_norm, "%slayers.%d.input_layernorm.weight", h); + LD(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", h); +#undef LD + +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, weight_prefix, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!linear_ok(l->fld)) \ + goto fail; \ + } while (0) + + LD(gate_proj, "%slayers.%d.mlp.gate_proj.weight", inter, h); + LD(up_proj, "%slayers.%d.mlp.up_proj.weight", inter, h); + LD(down_proj, "%slayers.%d.mlp.down_proj.weight", h, inter); +#undef LD + + if (embed_verbose >= 2) + fprintf(stderr, " mlx layer %d loaded\n", i); + } + + if (layer_end == c->n_layers) { + int ns[] = {h}; + snprintf(name, sizeof(name), "%snorm.weight", weight_prefix); + ctx->norm = load_tensor(ctx->ms, name, ns, 1); + if (!arr_ok(ctx->norm)) + goto fail; + } + +loaded: + ctx->stream = mlx_default_gpu_stream_new(); + ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w); + /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the + * tiny norm vectors so they match the F32 activations in mlx_forward_layers; + * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */ + if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) { + for (int i = layer_start; i < layer_end; i++) { + mlx_layer_t *l = &ctx->layers[i]; + upcast_f32_inplace(&l->q_bias, ctx->stream); + upcast_f32_inplace(&l->k_bias, ctx->stream); + upcast_f32_inplace(&l->v_bias, ctx->stream); + upcast_f32_inplace(&l->q_norm, ctx->stream); + upcast_f32_inplace(&l->k_norm, ctx->stream); + upcast_f32_inplace(&l->input_norm, ctx->stream); + upcast_f32_inplace(&l->post_attn_norm, ctx->stream); + /* BERT block biases (no-ops for Qwen, whose arrays are unset); the + * two BERT LayerNorm weights reuse input_norm/post_attn_norm above. */ + upcast_f32_inplace(&l->o_bias, ctx->stream); + upcast_f32_inplace(&l->attn_ln_bias, ctx->stream); + upcast_f32_inplace(&l->ffn_inter_bias, ctx->stream); + upcast_f32_inplace(&l->ffn_out_bias, ctx->stream); + upcast_f32_inplace(&l->ffn_ln_bias, ctx->stream); + } + upcast_f32_inplace(&ctx->norm, ctx->stream); + /* BERT embedding stage (no-ops for Qwen): learned position and token-type + * embeddings and the embedding LayerNorm all feed the F32 residual. */ + upcast_f32_inplace(&ctx->position_embeddings, ctx->stream); + upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream); + upcast_f32_inplace(&ctx->embed_ln_w, ctx->stream); + upcast_f32_inplace(&ctx->embed_ln_b, ctx->stream); + } + if (ctx->quantize_bits) { + for (int i = layer_start; i < layer_end; i++) { + if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, ctx->quantize_group_size, + ctx->stream) != 0) + goto fail; + } + mlx_clear_cache(); + if (embed_verbose >= 1) + fprintf(stderr, + "mlx: quantized linear weights to %d-bit " + "(group_size=%d)\n", + ctx->quantize_bits, ctx->quantize_group_size); + } else { + for (int i = layer_start; i < layer_end; i++) { + if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0) + goto fail; + } + } + + if (embed_verbose >= 2) { + size_t active = 0, cache = 0, peak = 0; + mlx_get_active_memory(&active); + mlx_get_cache_memory(&cache); + mlx_get_peak_memory(&peak); + fprintf(stderr, + "mlx: memory active=%.2f GiB cache=%.2f GiB " + "peak=%.2f GiB\n", + (double)active / (1024.0 * 1024.0 * 1024.0), + (double)cache / (1024.0 * 1024.0 * 1024.0), + (double)peak / (1024.0 * 1024.0 * 1024.0)); + } + + if (embed_verbose >= 1) + fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h); + + return ctx; + +fail: + embed_mlx_free(ctx); + return NULL; +} + +embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) { + return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); +} + +embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, + const embed_mlx_options_t *opts) { + return mlx_load_range_ex(model_dir, 0, -1, opts, 0); +} + +/* ======================================================================== + * Free + * ======================================================================== */ + +static void free_mlx_layer(mlx_layer_t *l) { + free_linear(&l->wq); + free_linear(&l->wk); + free_linear(&l->wv); + free_linear(&l->wo); + mlx_array_free(l->q_bias); + mlx_array_free(l->k_bias); + mlx_array_free(l->v_bias); + mlx_array_free(l->q_norm); + mlx_array_free(l->k_norm); + mlx_array_free(l->input_norm); + mlx_array_free(l->post_attn_norm); + free_linear(&l->gate_proj); + free_linear(&l->up_proj); + free_linear(&l->down_proj); + mlx_array_free(l->o_bias); + mlx_array_free(l->attn_ln_bias); + mlx_array_free(l->ffn_inter_bias); + mlx_array_free(l->ffn_out_bias); + mlx_array_free(l->ffn_ln_bias); +} + +void embed_mlx_free(embed_mlx_ctx_t *ctx) { + if (!ctx) + return; + mlx_array_free(ctx->embed_tokens); + if (ctx->layers) { + for (int i = 0; i < ctx->config.n_layers; i++) + free_mlx_layer(&ctx->layers[i]); + free(ctx->layers); + } + mlx_array_free(ctx->norm); + mlx_array_free(ctx->position_embeddings); + mlx_array_free(ctx->token_type_embeddings); + mlx_array_free(ctx->embed_ln_w); + mlx_array_free(ctx->embed_ln_b); + if (ctx->stream.ctx) + mlx_stream_free(ctx->stream); + if (ctx->ms) + multi_safetensors_close(ctx->ms); + free(ctx); +} + +embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir, + const embed_mlx_options_t *opts) { + if (!mlx_model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "mlx late: missing 1_Dense projection\n"); + return NULL; + } + + embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); + if (!late) + return NULL; + + late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1); + if (!late->base) + goto fail; + + char dense_dir[1024]; + int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir); + if (n < 0 || (size_t)n >= sizeof(dense_dir)) { + fprintf(stderr, "mlx late: model path too long\n"); + goto fail; + } + + late->dense_ms = multi_safetensors_open(dense_dir); + if (!late->dense_ms) { + fprintf(stderr, "mlx late: failed to open safetensors in %s\n", dense_dir); + goto fail; + } + + const safetensor_t *t = multi_safetensors_find(late->dense_ms, "linear.weight", NULL); + if (!t) { + fprintf(stderr, "mlx late: missing linear.weight\n"); + goto fail; + } + if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX || + t->shape[1] != late->base->config.hidden_size) { + fprintf(stderr, "mlx late: bad projection shape\n"); + goto fail; + } + + int shape[] = {(int)t->shape[0], late->base->config.hidden_size}; + late->projection = load_linear_tensor(late->dense_ms, "linear.weight", shape, 2); + if (!linear_ok(late->projection)) + goto fail; + if (prepare_linear_transpose(&late->projection, late->base->stream) != 0) + goto fail; + late->token_dim = shape[0]; + + if (embed_verbose >= 1) + fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim); + return late; + +fail: + embed_mlx_late_free(late); + return NULL; +} + +embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) { + return embed_mlx_late_load_with_options(model_dir, NULL); +} + +void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) { + if (!ctx) + return; + free_linear(&ctx->projection); + if (ctx->dense_ms) + multi_safetensors_close(ctx->dense_ms); + embed_mlx_free(ctx->base); + free(ctx); +} + +const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) { + return ctx && ctx->base ? &ctx->base->config : NULL; +} + +int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } + +/* ======================================================================== + * Forward pass + * ======================================================================== */ + +/* Exact erf GeLU: 0.5 * x * (1 + erf(x / sqrt(2))). Frees nothing it is given. */ +static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) { + mlx_array c = mlx_array_new_float32(0.70710678118654752f); + mlx_array scaled = mlx_array_new(); + mlx_multiply(&scaled, x, c, S); + mlx_array_free(c); + mlx_array e = mlx_array_new(); + mlx_erf(&e, scaled, S); + mlx_array_free(scaled); + mlx_array one = mlx_array_new_float32(1.0f); + mlx_array e1 = mlx_array_new(); + mlx_add(&e1, e, one, S); + mlx_array_free(e); + mlx_array_free(one); + mlx_array half = mlx_array_new_float32(0.5f); + mlx_array xh = mlx_array_new(); + mlx_multiply(&xh, x, half, S); + mlx_array_free(half); + mlx_array g = mlx_array_new(); + mlx_multiply(&g, xh, e1, S); + mlx_array_free(xh); + mlx_array_free(e1); + return g; +} + +/* Tanh-approximation GeLU (gelu_new / gelu_pytorch_tanh): + * 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). Frees nothing given. */ +static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) { + mlx_array x2 = mlx_array_new(); + mlx_multiply(&x2, x, x, S); + mlx_array x3 = mlx_array_new(); + mlx_multiply(&x3, x2, x, S); + mlx_array_free(x2); + mlx_array c = mlx_array_new_float32(0.044715f); + mlx_array cx3 = mlx_array_new(); + mlx_multiply(&cx3, x3, c, S); + mlx_array_free(x3); + mlx_array_free(c); + mlx_array sum = mlx_array_new(); + mlx_add(&sum, x, cx3, S); + mlx_array_free(cx3); + mlx_array s2pi = mlx_array_new_float32(0.79788456080286536f); + mlx_array inner = mlx_array_new(); + mlx_multiply(&inner, sum, s2pi, S); + mlx_array_free(sum); + mlx_array_free(s2pi); + mlx_array t = mlx_array_new(); + mlx_tanh(&t, inner, S); + mlx_array_free(inner); + mlx_array one = mlx_array_new_float32(1.0f); + mlx_array t1 = mlx_array_new(); + mlx_add(&t1, t, one, S); + mlx_array_free(t); + mlx_array_free(one); + mlx_array half = mlx_array_new_float32(0.5f); + mlx_array xh = mlx_array_new(); + mlx_multiply(&xh, x, half, S); + mlx_array_free(half); + mlx_array g = mlx_array_new(); + mlx_multiply(&g, xh, t1, S); + mlx_array_free(xh); + mlx_array_free(t1); + return g; +} + +/* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU + * feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the + * x it is given and returns the new hidden states. */ +static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx, + mlx_array x, + int batch, + int max_seq, + int has_padding, + mlx_array attn_mask, + int layer_start, + int layer_end) { + const embed_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + int n_heads = c->n_heads, head_dim = c->head_dim, q_dim = c->q_dim; + float scale = 1.0f / sqrtf((float)head_dim); + float eps = c->layer_norm_eps; + mlx_array null_arr = (mlx_array){0}; + + for (int layer = layer_start; layer < layer_end; layer++) { + mlx_layer_t *l = &ctx->layers[layer]; + + /* Self-attention reads x directly (post-norm). */ + mlx_array q_flat = linear(x, &l->wq, S); + mlx_array k_flat = linear(x, &l->wk, S); + mlx_array v_flat = linear(x, &l->wv, S); + mlx_array t = mlx_array_new(); + mlx_add(&t, q_flat, l->q_bias, S); + mlx_array_free(q_flat); + q_flat = t; + t = mlx_array_new(); + mlx_add(&t, k_flat, l->k_bias, S); + mlx_array_free(k_flat); + k_flat = t; + t = mlx_array_new(); + mlx_add(&t, v_flat, l->v_bias, S); + mlx_array_free(v_flat); + v_flat = t; + + int s4[] = {batch, max_seq, n_heads, head_dim}; + mlx_array q = mlx_array_new(), k = mlx_array_new(), v = mlx_array_new(); + mlx_reshape(&q, q_flat, s4, 4, S); + mlx_reshape(&k, k_flat, s4, 4, S); + mlx_reshape(&v, v_flat, s4, 4, S); + mlx_array_free(q_flat); + mlx_array_free(k_flat); + mlx_array_free(v_flat); + + int perm[] = {0, 2, 1, 3}; + mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); + mlx_transpose_axes(&qt, q, perm, 4, S); + mlx_transpose_axes(&kt, k, perm, 4, S); + mlx_transpose_axes(&vt, v, perm, 4, S); + mlx_array_free(q); + mlx_array_free(k); + mlx_array_free(v); + + mlx_array attn = mlx_array_new(); + mlx_fast_scaled_dot_product_attention(&attn, qt, kt, vt, scale, "", + has_padding ? attn_mask : null_arr, null_arr, S); + mlx_array_free(qt); + mlx_array_free(kt); + mlx_array_free(vt); + + mlx_array attn_t = mlx_array_new(); + mlx_transpose_axes(&attn_t, attn, perm, 4, S); + mlx_array_free(attn); + int s3[] = {batch, max_seq, q_dim}; + mlx_array attn_flat = mlx_array_new(); + mlx_reshape(&attn_flat, attn_t, s3, 3, S); + mlx_array_free(attn_t); + + mlx_array proj = linear(attn_flat, &l->wo, S); + mlx_array_free(attn_flat); + mlx_array proj_b = mlx_array_new(); + mlx_add(&proj_b, proj, l->o_bias, S); + mlx_array_free(proj); + mlx_array res = mlx_array_new(); + mlx_add(&res, x, proj_b, S); + mlx_array_free(x); + mlx_array_free(proj_b); + x = mlx_array_new(); + mlx_fast_layer_norm(&x, res, l->input_norm, l->attn_ln_bias, eps, S); + mlx_array_free(res); + + /* Feed-forward: dense -> GeLU -> dense. */ + mlx_array mid = linear(x, &l->up_proj, S); + mlx_array mid_b = mlx_array_new(); + mlx_add(&mid_b, mid, l->ffn_inter_bias, S); + mlx_array_free(mid); + mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S) + : mlx_gelu_erf(mid_b, S); + mlx_array_free(mid_b); + mlx_array out = linear(g, &l->down_proj, S); + mlx_array_free(g); + mlx_array out_b = mlx_array_new(); + mlx_add(&out_b, out, l->ffn_out_bias, S); + mlx_array_free(out); + mlx_array res2 = mlx_array_new(); + mlx_add(&res2, x, out_b, S); + mlx_array_free(x); + mlx_array_free(out_b); + x = mlx_array_new(); + mlx_fast_layer_norm(&x, res2, l->post_attn_norm, l->ffn_ln_bias, eps, S); + mlx_array_free(res2); + } + return x; +} + +static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, + mlx_array x, + int batch, + int max_seq, + int has_padding, + mlx_array attn_mask, + int layer_start, + int layer_end) { + const embed_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + int n_heads = c->n_heads; + int n_kv_heads = c->n_kv_heads; + int head_dim = c->head_dim; + int q_dim = c->q_dim; + float scale = 1.0f / sqrtf((float)head_dim); + mlx_array null_arr = (mlx_array){0}; + const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : ""; + + /* For non-quantized BF16 weights, run the residual stream, norms, and + * attention in F32; matmuls downcast their operands to BF16 internally + * (see linear). This keeps weight storage at BF16 while avoiding the + * cross-layer accumulation error of a fully BF16 forward. F32 and quantized + * models are unaffected. */ + mlx_array local_mask = attn_mask; + int free_mask = 0; + int mixed_bf16 = + mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized; + if (mixed_bf16) { + if (mlx_array_dtype(x) != MLX_FLOAT32) { + mlx_array xf = mlx_array_new(); + mlx_astype(&xf, x, MLX_FLOAT32, S); + mlx_array_free(x); + x = xf; + } + if (has_padding && arr_ok(attn_mask) && mlx_array_dtype(attn_mask) != MLX_FLOAT32) { + local_mask = mlx_array_new(); + mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S); + free_mask = 1; + } + } + + for (int layer = layer_start; layer < layer_end; layer++) { + mlx_layer_t *l = &ctx->layers[layer]; + + mlx_array xn = mlx_array_new(); + mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); + + mlx_array q_flat = linear(xn, &l->wq, S); + mlx_array k_flat = linear(xn, &l->wk, S); + mlx_array v_flat = linear(xn, &l->wv, S); + mlx_array_free(xn); + if (c->qkv_bias) { + mlx_array biased = mlx_array_new(); + mlx_add(&biased, q_flat, l->q_bias, S); + mlx_array_free(q_flat); + q_flat = biased; + biased = mlx_array_new(); + mlx_add(&biased, k_flat, l->k_bias, S); + mlx_array_free(k_flat); + k_flat = biased; + biased = mlx_array_new(); + mlx_add(&biased, v_flat, l->v_bias, S); + mlx_array_free(v_flat); + v_flat = biased; + } + + int q_shape[] = {batch, max_seq, n_heads, head_dim}; + int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; + mlx_array q = mlx_array_new(); + mlx_array k = mlx_array_new(); + mlx_array v = mlx_array_new(); + mlx_reshape(&q, q_flat, q_shape, 4, S); + mlx_reshape(&k, k_flat, k_shape, 4, S); + mlx_reshape(&v, v_flat, k_shape, 4, S); + mlx_array_free(q_flat); + mlx_array_free(k_flat); + mlx_array_free(v_flat); + + mlx_array qn = q; + mlx_array kn = k; + if (c->qk_norm) { + qn = mlx_array_new(); + kn = mlx_array_new(); + mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); + mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); + mlx_array_free(q); + mlx_array_free(k); + } + + /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ + int perm[] = {0, 2, 1, 3}; + mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); + mlx_transpose_axes(&qt, qn, perm, 4, S); + mlx_transpose_axes(&kt, kn, perm, 4, S); + mlx_transpose_axes(&vt, v, perm, 4, S); + mlx_array_free(qn); + mlx_array_free(kn); + mlx_array_free(v); + + mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; + mlx_array qr = mlx_array_new(), kr = mlx_array_new(); + mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); + mlx_array_free(qt); + mlx_array_free(kt); + + mlx_array attn = mlx_array_new(); + mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, mask_mode, + has_padding ? local_mask : null_arr, null_arr, S); + mlx_array_free(qr); + mlx_array_free(kr); + mlx_array_free(vt); + + mlx_array attn_t = mlx_array_new(); + mlx_transpose_axes(&attn_t, attn, perm, 4, S); + mlx_array_free(attn); + + int flat_shape[] = {batch, max_seq, q_dim}; + mlx_array attn_flat = mlx_array_new(); + mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S); + mlx_array_free(attn_t); + + mlx_array proj = linear(attn_flat, &l->wo, S); + mlx_array_free(attn_flat); + mlx_array x2 = mlx_array_new(); + mlx_add(&x2, x, proj, S); + mlx_array_free(x); + mlx_array_free(proj); + x = x2; + + mlx_array xn2 = mlx_array_new(); + mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); + + mlx_array gate = linear(xn2, &l->gate_proj, S); + mlx_array up = linear(xn2, &l->up_proj, S); + mlx_array_free(xn2); + + mlx_array gate_sig = mlx_array_new(); + mlx_sigmoid(&gate_sig, gate, S); + mlx_array silu_gate = mlx_array_new(); + mlx_multiply(&silu_gate, gate, gate_sig, S); + mlx_array_free(gate); + mlx_array_free(gate_sig); + + mlx_array mid = mlx_array_new(); + mlx_multiply(&mid, silu_gate, up, S); + mlx_array_free(silu_gate); + mlx_array_free(up); + + mlx_array ffn = linear(mid, &l->down_proj, S); + mlx_array_free(mid); + + mlx_array x3 = mlx_array_new(); + mlx_add(&x3, x, ffn, S); + mlx_array_free(x); + mlx_array_free(ffn); + x = x3; + } + if (free_mask) + mlx_array_free(local_mask); + return x; +} + +embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize) { + if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 || + !linear_ok(ctx->projection)) + return NULL; + + embed_mlx_ctx_t *base = ctx->base; + if (base->layer_start != 0 || base->layer_end != base->config.n_layers || + !arr_ok(base->embed_tokens) || !arr_ok(base->norm)) + return NULL; + + const embed_config_t *c = &base->config; + mlx_stream S = base->stream; + for (int i = 0; i < n_tokens; i++) { + if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { + fprintf(stderr, "mlx late: invalid token id %d at position %d\n", token_ids[i], i); + return NULL; + } + } + + size_t id_bytes; + if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0) + return NULL; + + int *ids_data = (int *)malloc(id_bytes); + if (!ids_data) + return NULL; + memcpy(ids_data, token_ids, id_bytes); + + int ids_shape[] = {1, n_tokens}; + mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32); + free(ids_data); + if (!arr_ok(ids)) + return NULL; + + embed_mlx_late_vectors_t *result = NULL; + mlx_array x = mlx_array_new(); + mlx_array x_normed = (mlx_array){0}; + mlx_array projected = (mlx_array){0}; + mlx_array reshaped = (mlx_array){0}; + + mlx_take_axis(&x, base->embed_tokens, ids, 0, S); + mlx_array_free(ids); + if (!arr_ok(x)) + goto cleanup; + + x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, 0, c->n_layers); + if (!arr_ok(x)) + goto cleanup; + + x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S); + mlx_array_free(x); + x = (mlx_array){0}; + if (!arr_ok(x_normed)) + goto cleanup; + + projected = linear(x_normed, &ctx->projection, S); + mlx_array_free(x_normed); + x_normed = (mlx_array){0}; + if (!arr_ok(projected)) + goto cleanup; + + if (normalize) { + mlx_array sq = mlx_array_new(); + mlx_square(&sq, projected, S); + if (!arr_ok(sq)) { + mlx_array_free(sq); + goto cleanup; + } + + mlx_array norm2 = mlx_array_new(); + mlx_sum_axis(&norm2, sq, 2, true, S); + mlx_array_free(sq); + if (!arr_ok(norm2)) { + mlx_array_free(norm2); + goto cleanup; + } + + mlx_array inv_norm = mlx_array_new(); + mlx_rsqrt(&inv_norm, norm2, S); + mlx_array_free(norm2); + if (!arr_ok(inv_norm)) { + mlx_array_free(inv_norm); + goto cleanup; + } + + mlx_array normalized = mlx_array_new(); + mlx_multiply(&normalized, projected, inv_norm, S); + mlx_array_free(inv_norm); + mlx_array_free(projected); + projected = normalized; + if (!arr_ok(projected)) + goto cleanup; + } + + int vec_shape[] = {n_tokens, ctx->token_dim}; + reshaped = mlx_array_new(); + mlx_reshape(&reshaped, projected, vec_shape, 2, S); + if (!arr_ok(reshaped)) + goto cleanup; + + result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result)); + if (!result) + goto cleanup; + result->owner = ctx; + result->vectors = reshaped; + result->tokens = n_tokens; + result->dim = ctx->token_dim; + reshaped = (mlx_array){0}; + +cleanup: + mlx_array_free(reshaped); + mlx_array_free(projected); + mlx_array_free(x_normed); + mlx_array_free(x); + return result; +} + +void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) { + if (!vecs) + return; + mlx_array_free(vecs->vectors); + free(vecs); +} + +int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) { + return vecs ? vecs->tokens : 0; +} + +int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) { + return vecs ? vecs->dim : 0; +} + +int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) { + if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0) + return -1; + + size_t values, bytes; + if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 || + mlx_mul_size(values, sizeof(float), &bytes) != 0) + return -1; + + mlx_stream S = vecs->owner->base->stream; + mlx_array out_f32 = mlx_array_new(); + mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S); + if (!arr_ok(out_f32)) { + mlx_array_free(out_f32); + return -1; + } + + int rc = -1; + mlx_array_eval(out_f32); + const float *data = mlx_array_data_float32(out_f32); + if (data) { + memcpy(out_vectors, data, bytes); + rc = 0; + } + mlx_array_free(out_f32); + return rc; +} + +/* + * Batched late-interaction document encoder. + * + * Encodes every candidate in one padded transformer pass, then gathers each + * document's kept token vectors into a single packed [total_keep, token_dim] + * array in document order. out_offsets receives the prefix sum of kept counts + * (out_offsets[i + 1] - out_offsets[i] == n_keep[i]), the exact layout + * embed_mlx_late_maxsim_batch_device consumes. This collapses N per-candidate + * transformer calls into one forward. + * + * Right-padding keeps real tokens at positions 0..n_tokens[d]-1 so RoPE + * positions are unchanged; a bidirectional model gets a padding attention mask + * so real tokens never attend to pad slots (the same masking the dense pooled + * batch path uses). Kept-token vectors therefore match per-document encoding to + * within MLX's batched-GEMM rounding. + */ +embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + int *out_offsets) { + if (!ctx || !ctx->base || !doc_ids || !n_tokens || !keep || !n_keep || !out_offsets || + n_docs <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection)) + return NULL; + + embed_mlx_ctx_t *base = ctx->base; + if (base->layer_start != 0 || base->layer_end != base->config.n_layers || + !arr_ok(base->embed_tokens) || !arr_ok(base->norm)) + return NULL; + + const embed_config_t *c = &base->config; + mlx_stream S = base->stream; + int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; + + /* Validate inputs and size the batch: longest sequence and total kept + * tokens. out_offsets is the running prefix sum of kept counts. */ + int max_seq = 0; + int total_keep = 0; + out_offsets[0] = 0; + for (int d = 0; d < n_docs; d++) { + if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 || + n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) + return NULL; + if (n_tokens[d] > max_seq) + max_seq = n_tokens[d]; + for (int k = 0; k < n_keep[d]; k++) { + if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) + return NULL; + } + total_keep += n_keep[d]; + out_offsets[d + 1] = total_keep; + } + + /* The gather index lives in an MLX int32 array, so the flattened + * [n_docs * max_seq, dim] row count must stay within int range. */ + size_t elems, ids_bytes, mask_bytes, gather_bytes; + if (mlx_mul_size((size_t)n_docs, (size_t)max_seq, &elems) != 0 || elems > INT_MAX || + mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 || + mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 || + mlx_mul_size((size_t)total_keep, sizeof(int), &gather_bytes) != 0) + return NULL; + + int has_padding = 0; + for (int d = 0; d < n_docs; d++) + if (n_tokens[d] != max_seq) + has_padding = 1; + int needs_attn_mask = has_padding && !causal; + if (ids_bytes == 0 || gather_bytes == 0 || (needs_attn_mask && mask_bytes == 0)) + return NULL; + + int *padded_ids = (int *)malloc(ids_bytes); + float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; + int *gather_idx = (int *)malloc(gather_bytes); + if (!padded_ids || (needs_attn_mask && !attn_mask_data) || !gather_idx) { + free(padded_ids); + free(attn_mask_data); + free(gather_idx); + return NULL; + } + + int gpos = 0; + for (int d = 0; d < n_docs; d++) { + for (int t = 0; t < max_seq; t++) { + size_t idx = (size_t)d * max_seq + t; + if (t < n_tokens[d]) { + int id = doc_ids[d][t]; + if (id < 0 || id >= c->vocab_size) { + fprintf(stderr, "mlx late: invalid token id %d at doc %d position %d\n", id, d, + t); + free(padded_ids); + free(attn_mask_data); + free(gather_idx); + return NULL; + } + padded_ids[idx] = id; + if (needs_attn_mask) + attn_mask_data[idx] = 0.0f; + } else { + padded_ids[idx] = 0; + if (needs_attn_mask) + attn_mask_data[idx] = -1.0e9f; + } + } + /* Map each kept token to its row in the flattened projection. */ + for (int k = 0; k < n_keep[d]; k++) + gather_idx[gpos++] = d * max_seq + keep[d][k]; + } + + int ids_shape[] = {n_docs, max_seq}; + int attn_mask_shape[] = {n_docs, 1, 1, max_seq}; + int gather_shape[] = {total_keep}; + mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + mlx_array attn_mask = needs_attn_mask + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array gather = mlx_array_new_data(gather_idx, gather_shape, 1, MLX_INT32); + free(padded_ids); + free(attn_mask_data); + free(gather_idx); + if (!arr_ok(ids) || (needs_attn_mask && !arr_ok(attn_mask)) || !arr_ok(gather)) { + mlx_array_free(ids); + mlx_array_free(attn_mask); + mlx_array_free(gather); + return NULL; + } + if (needs_attn_mask && mlx_weight_dtype(base) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) { + mlx_array_free(ids); + mlx_array_free(gather); + return NULL; + } + } + + embed_mlx_late_vectors_t *result = NULL; + mlx_array x = mlx_array_new(); + mlx_array x_normed = (mlx_array){0}; + mlx_array projected = (mlx_array){0}; + mlx_array flat = (mlx_array){0}; + mlx_array gathered = (mlx_array){0}; + + mlx_take_axis(&x, base->embed_tokens, ids, 0, S); + mlx_array_free(ids); + if (!arr_ok(x)) + goto cleanup; + + x = mlx_forward_layers(base, x, n_docs, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers); + if (!arr_ok(x)) + goto cleanup; + + x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S); + mlx_array_free(x); + x = (mlx_array){0}; + if (!arr_ok(x_normed)) + goto cleanup; + + projected = linear(x_normed, &ctx->projection, S); + mlx_array_free(x_normed); + x_normed = (mlx_array){0}; + if (!arr_ok(projected)) + goto cleanup; + + /* Flatten [n_docs, max_seq, dim] -> [n_docs * max_seq, dim] and gather the + * kept tokens in document order into one packed [total_keep, dim] array. */ + int flat_shape[] = {(int)elems, ctx->token_dim}; + flat = mlx_array_new(); + mlx_reshape(&flat, projected, flat_shape, 2, S); + mlx_array_free(projected); + projected = (mlx_array){0}; + if (!arr_ok(flat)) + goto cleanup; + + gathered = mlx_array_new(); + mlx_take_axis(&gathered, flat, gather, 0, S); + mlx_array_free(flat); + flat = (mlx_array){0}; + if (!arr_ok(gathered)) + goto cleanup; + + /* Normalize after gathering: per-token L2 is row-independent, so this is + * identical to normalizing the full batch but skips the dropped and padded + * rows entirely. */ + if (normalize) { + mlx_array sq = mlx_array_new(); + mlx_square(&sq, gathered, S); + if (!arr_ok(sq)) { + mlx_array_free(sq); + goto cleanup; + } + mlx_array norm2 = mlx_array_new(); + mlx_sum_axis(&norm2, sq, 1, true, S); + mlx_array_free(sq); + if (!arr_ok(norm2)) { + mlx_array_free(norm2); + goto cleanup; + } + mlx_array inv_norm = mlx_array_new(); + mlx_rsqrt(&inv_norm, norm2, S); + mlx_array_free(norm2); + if (!arr_ok(inv_norm)) { + mlx_array_free(inv_norm); + goto cleanup; + } + mlx_array normalized = mlx_array_new(); + mlx_multiply(&normalized, gathered, inv_norm, S); + mlx_array_free(inv_norm); + mlx_array_free(gathered); + gathered = normalized; + if (!arr_ok(gathered)) + goto cleanup; + } + + result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result)); + if (!result) + goto cleanup; + result->owner = ctx; + result->vectors = gathered; + result->tokens = total_keep; + result->dim = ctx->token_dim; + gathered = (mlx_array){0}; + +cleanup: + mlx_array_free(gathered); + mlx_array_free(flat); + mlx_array_free(projected); + mlx_array_free(x_normed); + mlx_array_free(x); + mlx_array_free(gather); + mlx_array_free(attn_mask); + return result; +} + +/* Device-resident select/concat primitives. The server rerank path batches + * candidates through embed_mlx_late_encode_docs_device, but the late-interaction + * verification harness (scripts/check_late_interaction.py) drives these directly + * to validate per-document gather and concat against the CPU MaxSim reference. */ +embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( + embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) { + if (!ctx || !ctx->base || !items || count <= 0) + return NULL; + int dim = 0; + int total = 0; + for (int i = 0; i < count; i++) { + const embed_mlx_late_vectors_t *v = items[i]; + if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0) + return NULL; + if (i == 0) + dim = v->dim; + if (v->dim != dim || total > INT_MAX - v->tokens) + return NULL; + total += v->tokens; + } + + mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts)); + if (!parts) + return NULL; + for (int i = 0; i < count; i++) + parts[i] = items[i]->vectors; + + mlx_stream S = ctx->base->stream; + mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count); + free(parts); + if (!vec.ctx) + return NULL; + + mlx_array joined = mlx_array_new(); + mlx_concatenate_axis(&joined, vec, 0, S); + mlx_vector_array_free(vec); + if (!arr_ok(joined)) { + mlx_array_free(joined); + return NULL; + } + + embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + if (!out) { + mlx_array_free(joined); + return NULL; + } + out->owner = ctx; + out->vectors = joined; + out->tokens = total; + out->dim = dim; + return out; +} + +embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *vecs, + const int *token_indices, + int count) { + if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) || + !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0) + return NULL; + + for (int i = 0; i < count; i++) { + if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens) + return NULL; + } + + size_t idx_bytes; + if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0) + return NULL; + int *idx_data = (int *)malloc(idx_bytes); + if (!idx_data) + return NULL; + memcpy(idx_data, token_indices, idx_bytes); + + int idx_shape[] = {count}; + mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32); + free(idx_data); + if (!arr_ok(idx)) + return NULL; + + mlx_stream S = ctx->base->stream; + mlx_array selected = mlx_array_new(); + mlx_take_axis(&selected, vecs->vectors, idx, 0, S); + mlx_array_free(idx); + if (!arr_ok(selected)) { + mlx_array_free(selected); + return NULL; + } + + embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + if (!out) { + mlx_array_free(selected); + return NULL; + } + out->owner = ctx; + out->vectors = selected; + out->tokens = count; + out->dim = vecs->dim; + return out; +} + +int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *query, + const embed_mlx_late_vectors_t *docs, + const int *doc_offsets, + int docs_count, + float *scores) { + if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores || + query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim || + query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) || + !arr_ok(docs->vectors) || doc_offsets[0] != 0) + return -1; + + int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths)); + if (!lengths) + return -1; + int max_doc_tokens = 0; + for (int i = 0; i < docs_count; i++) { + if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] || + doc_offsets[i + 1] > docs->tokens) { + free(lengths); + return -1; + } + int n = doc_offsets[i + 1] - doc_offsets[i]; + lengths[i] = n; + if (n > max_doc_tokens) + max_doc_tokens = n; + } + if (max_doc_tokens <= 0) { + free(lengths); + return -1; + } + + mlx_stream S = ctx->base->stream; + int dim = query->dim; + int rc = -1; + mlx_array *doc_rows = (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); + if (!doc_rows) { + free(lengths); + return -1; + } + + mlx_vector_array doc_vec = (mlx_vector_array){0}; + mlx_array pad_value = mlx_array_new_float32(0.0f); + mlx_array doc_batch = (mlx_array){0}; + mlx_array query_3d = (mlx_array){0}; + mlx_array query_batched = (mlx_array){0}; + mlx_array docs_t = (mlx_array){0}; + mlx_array sims = (mlx_array){0}; + mlx_array lengths_arr = (mlx_array){0}; + mlx_array positions = (mlx_array){0}; + mlx_array lengths_2d = (mlx_array){0}; + mlx_array positions_2d = (mlx_array){0}; + mlx_array valid = (mlx_array){0}; + mlx_array valid_3d = (mlx_array){0}; + mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX); + mlx_array masked_sims = (mlx_array){0}; + mlx_array maxes = (mlx_array){0}; + mlx_array sums = (mlx_array){0}; + mlx_array sums_f32 = (mlx_array){0}; + + for (int i = 0; i < docs_count; i++) { + int n = lengths[i]; + int start[] = {doc_offsets[i], 0}; + int stop[] = {doc_offsets[i + 1], dim}; + int strides[] = {1, 1}; + + mlx_array doc = mlx_array_new(); + mlx_array padded = (mlx_array){0}; + + mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S); + if (!arr_ok(doc)) + goto cleanup; + + if (n < max_doc_tokens) { + int axes[] = {0}; + int low[] = {0}; + int high[] = {max_doc_tokens - n}; + padded = mlx_array_new(); + mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, "constant", S); + mlx_array_free(doc); + doc = padded; + padded = (mlx_array){0}; + if (!arr_ok(doc)) + goto cleanup; + } + + doc_rows[i] = mlx_array_new(); + mlx_expand_dims(&doc_rows[i], doc, 0, S); + mlx_array_free(doc); + if (!arr_ok(doc_rows[i])) + goto cleanup; + } + + doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count); + if (!doc_vec.ctx) + goto cleanup; + + doc_batch = mlx_array_new(); + mlx_concatenate_axis(&doc_batch, doc_vec, 0, S); + if (!arr_ok(doc_batch)) + goto cleanup; + + query_3d = mlx_array_new(); + mlx_expand_dims(&query_3d, query->vectors, 0, S); + if (!arr_ok(query_3d)) + goto cleanup; + + int q_shape[] = {docs_count, query->tokens, dim}; + query_batched = mlx_array_new(); + mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S); + if (!arr_ok(query_batched)) + goto cleanup; + + int perm[] = {0, 2, 1}; + docs_t = mlx_array_new(); + mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S); + if (!arr_ok(docs_t)) + goto cleanup; + + sims = mlx_array_new(); + mlx_matmul(&sims, query_batched, docs_t, S); + if (!arr_ok(sims)) + goto cleanup; + + int len_shape[] = {docs_count}; + lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32); + if (!arr_ok(lengths_arr)) + goto cleanup; + + positions = mlx_array_new(); + mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S); + if (!arr_ok(positions)) + goto cleanup; + + lengths_2d = mlx_array_new(); + mlx_expand_dims(&lengths_2d, lengths_arr, 1, S); + if (!arr_ok(lengths_2d)) + goto cleanup; + + positions_2d = mlx_array_new(); + mlx_expand_dims(&positions_2d, positions, 0, S); + if (!arr_ok(positions_2d)) + goto cleanup; + + valid = mlx_array_new(); + mlx_less(&valid, positions_2d, lengths_2d, S); + if (!arr_ok(valid)) + goto cleanup; + + valid_3d = mlx_array_new(); + mlx_expand_dims(&valid_3d, valid, 1, S); + if (!arr_ok(valid_3d)) + goto cleanup; + + masked_sims = mlx_array_new(); + mlx_where(&masked_sims, valid_3d, sims, invalid_value, S); + if (!arr_ok(masked_sims)) + goto cleanup; + + maxes = mlx_array_new(); + mlx_max_axis(&maxes, masked_sims, 2, false, S); + if (!arr_ok(maxes)) + goto cleanup; + + sums = mlx_array_new(); + mlx_sum_axis(&sums, maxes, 1, false, S); + if (!arr_ok(sums)) + goto cleanup; + + sums_f32 = mlx_array_new(); + mlx_astype(&sums_f32, sums, MLX_FLOAT32, S); + if (!arr_ok(sums_f32)) + goto cleanup; + + mlx_array_eval(sums_f32); + const float *data = mlx_array_data_float32(sums_f32); + if (!data) + goto cleanup; + memcpy(scores, data, (size_t)docs_count * sizeof(float)); + rc = 0; + +cleanup: + mlx_array_free(sums_f32); + mlx_array_free(sums); + mlx_array_free(maxes); + mlx_array_free(masked_sims); + mlx_array_free(invalid_value); + mlx_array_free(valid_3d); + mlx_array_free(valid); + mlx_array_free(positions_2d); + mlx_array_free(lengths_2d); + mlx_array_free(positions); + mlx_array_free(lengths_arr); + mlx_array_free(sims); + mlx_array_free(docs_t); + mlx_array_free(query_batched); + mlx_array_free(query_3d); + mlx_array_free(doc_batch); + if (doc_vec.ctx) + mlx_vector_array_free(doc_vec); + mlx_array_free(pad_value); + for (int i = 0; i < docs_count; i++) + mlx_array_free(doc_rows[i]); + free(doc_rows); + free(lengths); + return rc; +} + +int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { + embed_mlx_late_vectors_t *vecs = + embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize); + if (!vecs) + return -1; + int rc = embed_mlx_late_vectors_copy(vecs, out_vectors); + embed_mlx_late_vectors_free(vecs); + return rc; +} + +static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || + ctx->layer_end != ctx->config.n_layers) + return -1; + + const embed_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + + int hidden = c->hidden_size; + int max_seq = 0; + int has_padding = 0; + int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; + int cls_pool = c->pooling_mode == EMBED_POOL_CLS; + /* CLS and last-token are both single-token pooling: gather one row per + * sequence (the first for CLS, the last otherwise) rather than mean-pool. */ + int single_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN || cls_pool; + if (hidden <= 0 || c->vocab_size <= 0) + return -1; + + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) + return -1; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; + } + if (max_seq <= 0) + return -1; + + for (int b = 0; b < batch; b++) { + if (inputs[b].n_tokens != max_seq) + has_padding = 1; + } + int needs_pool_mask = has_padding && !single_token_pool; + int needs_attn_mask = has_padding && !causal; + + size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes; + size_t last_index_bytes = 0; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 || + mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 || + mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 || + mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || + mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || + mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 || + (single_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) + return -1; + if (ids_bytes == 0 || out_bytes == 0 || + (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) || + (needs_attn_mask && mask_bytes == 0) || (single_token_pool && last_index_bytes == 0)) + return -1; + int *padded_ids = (int *)malloc(ids_bytes); + float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL; + float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; + float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL; + int *last_index_data = single_token_pool ? (int *)malloc(last_index_bytes) : NULL; + if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) || + (needs_attn_mask && !attn_mask_data)) { + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); + free(last_index_data); + return -1; + } + if (single_token_pool && !last_index_data) { + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); + return -1; + } + + for (int b = 0; b < batch; b++) { + if (needs_pool_mask) + len_data[b] = (float)inputs[b].n_tokens; + if (single_token_pool) { + int index = cls_pool ? 0 : inputs[b].n_tokens - 1; + for (int d = 0; d < hidden; d++) + last_index_data[(size_t)b * hidden + d] = index; + } + for (int t = 0; t < max_seq; t++) { + size_t idx = (size_t)b * max_seq + t; + if (t < inputs[b].n_tokens) { + int id = inputs[b].ids[t]; + if (id < 0 || id >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t); + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); + free(last_index_data); + return -1; + } + padded_ids[idx] = id; + if (needs_pool_mask) + pool_mask_data[idx] = 1.0f; + if (needs_attn_mask) + attn_mask_data[idx] = 0.0f; + } else { + padded_ids[idx] = 0; + if (needs_pool_mask) + pool_mask_data[idx] = 0.0f; + if (needs_attn_mask) + attn_mask_data[idx] = -1.0e9f; + } + } + } + + int ids_shape[] = {batch, max_seq}; + int pool_mask_shape[] = {batch, max_seq, 1}; + int attn_mask_shape[] = {batch, 1, 1, max_seq}; + int len_shape[] = {batch, 1}; + int last_index_shape[] = {batch, 1, hidden}; + + mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + mlx_array pool_mask = needs_pool_mask + ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array attn_mask = needs_attn_mask + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array lengths = + needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0}; + mlx_array last_indices = + single_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) + : (mlx_array){0}; + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); + free(last_index_data); + + if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) || + (needs_attn_mask && !arr_ok(attn_mask)) || (single_token_pool && !arr_ok(last_indices))) { + mlx_array_free(ids); + mlx_array_free(pool_mask); + mlx_array_free(attn_mask); + mlx_array_free(lengths); + mlx_array_free(last_indices); + return -1; + } + if (needs_attn_mask && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) { + mlx_array_free(ids); + mlx_array_free(pool_mask); + mlx_array_free(lengths); + mlx_array_free(last_indices); + return -1; + } + } + + /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */ + mlx_array x = mlx_array_new(); + mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); + mlx_array_free(ids); + + /* 2. Family forward. BERT adds learned positions + token-type[0] and an + * embedding LayerNorm, then runs post-norm encoder layers; its states are + * already final, so pooling uses them directly. Qwen runs the decoder block + * and applies the final RMSNorm here. */ + mlx_array x_normed; + if (c->family == EMBED_FAMILY_BERT) { + int start2[] = {0, 0}, stride2[] = {1, 1}; + int pos_stop[] = {max_seq, hidden}, type_stop[] = {1, hidden}; + mlx_array pos = mlx_array_new(); + mlx_slice(&pos, ctx->position_embeddings, start2, 2, pos_stop, 2, stride2, 2, S); + mlx_array xp = mlx_array_new(); + mlx_add(&xp, x, pos, S); + mlx_array_free(x); + mlx_array_free(pos); + mlx_array tt = mlx_array_new(); + mlx_slice(&tt, ctx->token_type_embeddings, start2, 2, type_stop, 2, stride2, 2, S); + mlx_array xt = mlx_array_new(); + mlx_add(&xt, xp, tt, S); + mlx_array_free(xp); + mlx_array_free(tt); + mlx_array xe = mlx_array_new(); + mlx_fast_layer_norm(&xe, xt, ctx->embed_ln_w, ctx->embed_ln_b, c->layer_norm_eps, S); + mlx_array_free(xt); + x_normed = + mlx_forward_bert(ctx, xe, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers); + } else { + x = mlx_forward_layers(ctx, x, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers); + x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); + mlx_array_free(x); + } + + mlx_array emb = mlx_array_new(); + if (single_token_pool) { + mlx_array selected = mlx_array_new(); + mlx_take_along_axis(&selected, x_normed, last_indices, 1, S); + mlx_array_free(x_normed); + mlx_array_free(last_indices); + int emb_shape[] = {batch, hidden}; + mlx_reshape(&emb, selected, emb_shape, 2, S); + mlx_array_free(selected); + mlx_array_free(pool_mask); + mlx_array_free(attn_mask); + mlx_array_free(lengths); + } else if (has_padding) { + /* 4. Masked mean pool over T. */ + mlx_array masked = mlx_array_new(); + mlx_multiply(&masked, x_normed, pool_mask, S); + mlx_array_free(x_normed); + mlx_array_free(pool_mask); + + mlx_array emb_sum = mlx_array_new(); + mlx_sum_axis(&emb_sum, masked, 1, false, S); + mlx_array_free(masked); + + mlx_divide(&emb, emb_sum, lengths, S); + mlx_array_free(emb_sum); + mlx_array_free(lengths); + mlx_array_free(attn_mask); + } else { + /* Equal-length batches do not need masks or a separate divide. */ + mlx_mean_axis(&emb, x_normed, 1, false, S); + mlx_array_free(x_normed); + } + + /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are + * intentionally unnormalized; callers should use cosine similarity. */ + mlx_array emb_f32 = mlx_array_new(); + mlx_astype(&emb_f32, emb, MLX_FLOAT32, S); + mlx_array_free(emb); + + mlx_array_eval(emb_f32); + const float *data = mlx_array_data_float32(emb_f32); + int rc = -1; + if (data) { + memcpy(out_embeddings, data, out_bytes); + if (c->normalize_embeddings) { + rc = 0; + for (int b = 0; b < batch; b++) { + if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) { + rc = -1; + break; + } + } + } else { + rc = 0; + } + } + mlx_array_free(emb_f32); + return rc; +} + +typedef struct { + int index; + const embed_input_t *input; +} mlx_batch_item_t; + +static int mlx_batch_item_cmp(const void *a, const void *b) { + const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a; + const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b; + int da = ia->input->n_tokens; + int db = ib->input->n_tokens; + return (da > db) - (da < db); +} + +static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) { + if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5) + return 1; + return padded * 5 > tokens * 11; +} + +static int mlx_batch_should_split(const embed_input_t *inputs, + int batch, + int *out_total_tokens, + int *out_max_seq) { + int max_seq = 0; + int total_tokens = 0; + for (int b = 0; b < batch; b++) { + if (!inputs[b].ids || inputs[b].n_tokens <= 0) + return 0; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; + if (total_tokens > INT_MAX - inputs[b].n_tokens) + return 0; + total_tokens += inputs[b].n_tokens; + } + if (out_total_tokens) + *out_total_tokens = total_tokens; + if (out_max_seq) + *out_max_seq = max_seq; + if (batch < 4 || max_seq < 64 || total_tokens <= 0) + return 0; + + size_t padded; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) + return 0; + + return mlx_padding_exceeds_limit(padded, (size_t)total_tokens); +} + +static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx, + int group_count, + int group_max, + int next_tokens) { + if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0) + return 1; + + int next_count = group_count + 1; + int next_max = next_tokens > group_max ? next_tokens : group_max; + size_t merged, split_left, split_total; + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &merged) != 0 || + mlx_mul_size((size_t)group_count, (size_t)group_max, &split_left) != 0) + return 0; + /* Treat one extra MLX launch as roughly 256 padded token rows for 4B. + * This preserves small mixed groups while splitting larger padded tails. */ + if (split_left > SIZE_MAX - (size_t)next_tokens || + split_left + (size_t)next_tokens > SIZE_MAX - 256) + return 0; + split_total = split_left + (size_t)next_tokens + 256; + return merged <= split_total; +} + +int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; + + if (!mlx_batch_should_split(inputs, batch, NULL, NULL)) + return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings); + + const int hidden = ctx->config.hidden_size; + mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items)); + embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); + size_t group_values, group_bytes; + if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 || + mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { + free(items); + free(group_inputs); + return -1; + } + float *group_out = (float *)malloc(group_bytes); + if (!items || !group_inputs || !group_out) { + free(items); + free(group_inputs); + free(group_out); + return -1; + } + for (int i = 0; i < batch; i++) { + items[i].index = i; + items[i].input = &inputs[i]; + } + qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp); + + int rc = 0; + for (int start = 0; start < batch && rc == 0;) { + int group_max = 0; + int group_tokens = 0; + int end = start; + while (end < batch) { + int next_tokens = items[end].input->n_tokens; + int next_count = end - start + 1; + int next_max = next_tokens > group_max ? next_tokens : group_max; + int next_total = group_tokens + next_tokens; + if (end > start) { + size_t next_padded; + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 || + mlx_padding_exceeds_limit(next_padded, (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens)) + break; + } + group_max = next_max; + group_tokens = next_total; + end++; + } + + int group_count = end - start; + for (int i = 0; i < group_count; i++) + group_inputs[i] = *items[start + i].input; + rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out); + if (rc != 0) + break; + for (int i = 0; i < group_count; i++) { + int dst = items[start + i].index; + memcpy(out_embeddings + (size_t)dst * hidden, group_out + (size_t)i * hidden, + (size_t)hidden * sizeof(float)); + } + start = end; + } + + free(items); + free(group_inputs); + free(group_out); + return rc; +} + +int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding) { + embed_input_t input = {token_ids, n_tokens}; + return embed_mlx_encode_batch(ctx, &input, 1, out_embedding); +} + +typedef struct { + int index; + int span_offset; + const embed_context_input_t *input; +} mlx_context_batch_item_t; + +static int mlx_context_batch_item_cmp(const void *a, const void *b) { + const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a; + const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b; + int da = ia->input->input.n_tokens; + int db = ib->input->input.n_tokens; + return (da > db) - (da < db); +} + +static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) { + if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4) + return 1; + return padded * 4 > tokens * 5; +} + +static int mlx_context_batch_should_split(const embed_context_input_t *inputs, + int batch, + int *out_total_spans) { + int max_seq = 0; + int total_tokens = 0; + int total_spans = 0; + for (int b = 0; b < batch; b++) { + int n_tokens = inputs[b].input.n_tokens; + if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || inputs[b].n_spans <= 0) + return 0; + if (n_tokens > max_seq) + max_seq = n_tokens; + if (total_tokens > INT_MAX - n_tokens || total_spans > INT_MAX - inputs[b].n_spans) + return 0; + total_tokens += n_tokens; + total_spans += inputs[b].n_spans; + } + if (out_total_spans) + *out_total_spans = total_spans; + if (batch < 2 || max_seq < 64 || total_tokens <= 0) + return 0; + + size_t padded; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) + return 0; + return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens); +} + +static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || + ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) + return -1; + + const embed_config_t *c = &ctx->config; + mlx_stream S = ctx->stream; + int hidden = c->hidden_size; + int max_seq = 0; + int total_spans = 0; + int has_padding = 0; + for (int b = 0; b < batch; b++) { + const embed_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || + total_spans > INT_MAX - input->n_spans) + return -1; + total_spans += input->n_spans; + if (n_tokens > max_seq) + max_seq = n_tokens; + for (int t = 0; t < n_tokens; t++) { + if (input->input.ids[t] < 0 || input->input.ids[t] >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", + input->input.ids[t], b, t); + return -1; + } + } + for (int s = 0; s < input->n_spans; s++) { + if (input->spans[s].start < 0 || input->spans[s].start > n_tokens || + input->spans[s].n_tokens <= 0 || + input->spans[s].n_tokens > n_tokens - input->spans[s].start) + return -1; + } + } + for (int b = 0; b < batch; b++) + if (inputs[b].input.n_tokens != max_seq) + has_padding = 1; + + size_t rows, ids_bytes, mask_bytes, index_bytes; + size_t span_float_bytes, out_values, out_bytes; + if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || + mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || + mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || + mlx_mul_size((size_t)total_spans, sizeof(int), &index_bytes) != 0 || + mlx_mul_size((size_t)total_spans, sizeof(float), &span_float_bytes) != 0 || + mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 || + mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) + return -1; + if (rows > INT_MAX) + return -1; + int *padded_ids = (int *)malloc(ids_bytes); + float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; + if (!padded_ids || (has_padding && !attn_mask_data)) { + free(padded_ids); + free(attn_mask_data); + return -1; + } + for (int b = 0; b < batch; b++) { + const embed_input_t *input = &inputs[b].input; + for (int t = 0; t < max_seq; t++) { + size_t row = (size_t)b * max_seq + t; + if (t < input->n_tokens) { + padded_ids[row] = input->ids[t]; + if (has_padding) + attn_mask_data[row] = 0.0f; + } else { + padded_ids[row] = 0; + if (has_padding) + attn_mask_data[row] = -1.0e9f; + } + } + } + + mlx_array ids = (mlx_array){0}; + mlx_array attn_mask = (mlx_array){0}; + mlx_array x = (mlx_array){0}; + mlx_array x_normed = (mlx_array){0}; + mlx_array prefix = (mlx_array){0}; + mlx_array flat_prefix = (mlx_array){0}; + mlx_array end_idx = (mlx_array){0}; + mlx_array start_idx = (mlx_array){0}; + mlx_array lengths = (mlx_array){0}; + mlx_array start_mask = (mlx_array){0}; + mlx_array end_values = (mlx_array){0}; + mlx_array start_values = (mlx_array){0}; + mlx_array masked_start = (mlx_array){0}; + mlx_array span_sums = (mlx_array){0}; + mlx_array embeddings = (mlx_array){0}; + mlx_array embeddings_f32 = (mlx_array){0}; + mlx_array *pooled = NULL; + mlx_vector_array pooled_vec = (mlx_vector_array){0}; + int *end_idx_data = NULL; + int *start_idx_data = NULL; + float *lengths_data = NULL; + float *start_mask_data = NULL; + int rc = -1; + + int ids_shape[] = {batch, max_seq}; + int attn_mask_shape[] = {batch, 1, 1, max_seq}; + ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + attn_mask = has_padding ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + free(padded_ids); + free(attn_mask_data); + if (!arr_ok(ids)) + goto cleanup; + if (has_padding && !arr_ok(attn_mask)) + goto cleanup; + if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) + goto cleanup; + } + + x = mlx_array_new(); + mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); + mlx_array_free(ids); + ids = (mlx_array){0}; + if (!arr_ok(x)) + goto cleanup; + + x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers); + if (has_padding) { + mlx_array_free(attn_mask); + attn_mask = (mlx_array){0}; + } + if (!arr_ok(x)) + goto cleanup; + + x_normed = mlx_array_new(); + mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); + mlx_array_free(x); + x = (mlx_array){0}; + if (!arr_ok(x_normed)) + goto cleanup; + + if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled)); + if (!pooled) + goto cleanup; + + int pooled_count = 0; + for (int b = 0; b < batch; b++) { + for (int s = 0; s < inputs[b].n_spans; s++) { + const embed_span_t *span = &inputs[b].spans[s]; + int start[] = {b, span->start, 0}; + int stop[] = {b + 1, span->start + span->n_tokens, hidden}; + int strides[] = {1, 1, 1}; + mlx_array slice = mlx_array_new(); + mlx_slice(&slice, x_normed, start, 3, stop, 3, strides, 3, S); + if (!arr_ok(slice)) { + mlx_array_free(slice); + goto cleanup; + } + + pooled[pooled_count] = mlx_array_new(); + mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S); + mlx_array_free(slice); + if (!arr_ok(pooled[pooled_count])) + goto cleanup; + pooled_count++; + } + } + + pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans); + if (!pooled_vec.ctx) + goto cleanup; + embeddings = mlx_array_new(); + mlx_concatenate_axis(&embeddings, pooled_vec, 0, S); + if (!arr_ok(embeddings)) + goto cleanup; + } else { + end_idx_data = (int *)malloc(index_bytes); + start_idx_data = (int *)malloc(index_bytes); + lengths_data = (float *)malloc(span_float_bytes); + start_mask_data = (float *)malloc(span_float_bytes); + if (!end_idx_data || !start_idx_data || !lengths_data || !start_mask_data) + goto cleanup; + + int span_count = 0; + for (int b = 0; b < batch; b++) { + for (int s = 0; s < inputs[b].n_spans; s++) { + const embed_span_t *span = &inputs[b].spans[s]; + int base = b * max_seq; + int start = span->start; + int stop = span->start + span->n_tokens; + end_idx_data[span_count] = base + stop - 1; + start_idx_data[span_count] = start > 0 ? base + start - 1 : base; + lengths_data[span_count] = (float)span->n_tokens; + start_mask_data[span_count] = start > 0 ? 1.0f : 0.0f; + span_count++; + } + } + + int index_shape[] = {total_spans}; + int scale_shape[] = {total_spans, 1}; + end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32); + start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, MLX_INT32); + lengths = mlx_array_new_data(lengths_data, scale_shape, 2, MLX_FLOAT32); + start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, MLX_FLOAT32); + free(end_idx_data); + end_idx_data = NULL; + free(start_idx_data); + start_idx_data = NULL; + free(lengths_data); + lengths_data = NULL; + free(start_mask_data); + start_mask_data = NULL; + if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || !arr_ok(start_mask)) + goto cleanup; + + prefix = mlx_array_new(); + mlx_cumsum(&prefix, x_normed, 1, false, true, S); + mlx_array_free(x_normed); + x_normed = (mlx_array){0}; + if (!arr_ok(prefix)) + goto cleanup; + + int flat_shape[] = {(int)rows, hidden}; + flat_prefix = mlx_array_new(); + mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S); + mlx_array_free(prefix); + prefix = (mlx_array){0}; + if (!arr_ok(flat_prefix)) + goto cleanup; + + end_values = mlx_array_new(); + mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S); + if (!arr_ok(end_values)) + goto cleanup; + + start_values = mlx_array_new(); + mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S); + if (!arr_ok(start_values)) + goto cleanup; + + masked_start = mlx_array_new(); + mlx_multiply(&masked_start, start_values, start_mask, S); + if (!arr_ok(masked_start)) + goto cleanup; + + span_sums = mlx_array_new(); + mlx_subtract(&span_sums, end_values, masked_start, S); + if (!arr_ok(span_sums)) + goto cleanup; + + embeddings = mlx_array_new(); + mlx_divide(&embeddings, span_sums, lengths, S); + if (!arr_ok(embeddings)) + goto cleanup; + } + + embeddings_f32 = mlx_array_new(); + mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S); + if (!arr_ok(embeddings_f32)) + goto cleanup; + + mlx_array_eval(embeddings_f32); + const float *data = mlx_array_data_float32(embeddings_f32); + if (data) { + memcpy(out_embeddings, data, out_bytes); + rc = 0; + } + +cleanup: + free(start_mask_data); + free(lengths_data); + free(start_idx_data); + free(end_idx_data); + mlx_array_free(embeddings_f32); + mlx_array_free(embeddings); + if (pooled_vec.ctx) + mlx_vector_array_free(pooled_vec); + if (pooled) { + for (int s = 0; s < total_spans; s++) + mlx_array_free(pooled[s]); + free(pooled); + } + mlx_array_free(span_sums); + mlx_array_free(masked_start); + mlx_array_free(start_values); + mlx_array_free(end_values); + mlx_array_free(start_mask); + mlx_array_free(lengths); + mlx_array_free(start_idx); + mlx_array_free(end_idx); + mlx_array_free(flat_prefix); + mlx_array_free(prefix); + mlx_array_free(x_normed); + mlx_array_free(x); + mlx_array_free(attn_mask); + mlx_array_free(ids); + return rc; +} + +int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings) + return -1; + + int total_spans = 0; + if (!mlx_context_batch_should_split(inputs, batch, &total_spans)) + return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings); + + const int hidden = ctx->config.hidden_size; + mlx_context_batch_item_t *items = + (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items)); + embed_context_input_t *group_inputs = + (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); + size_t group_values, group_bytes; + if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 || + mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { + free(items); + free(group_inputs); + return -1; + } + float *group_out = (float *)malloc(group_bytes); + if (!items || !group_inputs || !group_out) { + free(items); + free(group_inputs); + free(group_out); + return -1; + } + + int span_offset = 0; + for (int i = 0; i < batch; i++) { + items[i].index = i; + items[i].span_offset = span_offset; + items[i].input = &inputs[i]; + span_offset += inputs[i].n_spans; + } + qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp); + + int rc = 0; + for (int start = 0; start < batch && rc == 0;) { + int group_max = 0; + int group_tokens = 0; + int end = start; + while (end < batch) { + int next_tokens = items[end].input->input.n_tokens; + int next_count = end - start + 1; + int next_max = next_tokens > group_max ? next_tokens : group_max; + int next_total = group_tokens + next_tokens; + if (end > start) { + size_t next_padded; + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 || + mlx_context_padding_exceeds_limit(next_padded, (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens)) + break; + } + group_max = next_max; + group_tokens = next_total; + end++; + } + + int group_count = end - start; + for (int i = 0; i < group_count; i++) + group_inputs[i] = *items[start + i].input; + rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out); + if (rc != 0) + break; + int src_span = 0; + for (int i = 0; i < group_count; i++) { + int n_spans = group_inputs[i].n_spans; + int dst_span = items[start + i].span_offset; + memcpy(out_embeddings + (size_t)dst_span * hidden, + group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float)); + src_span += n_spans; + } + start = end; + } + + free(items); + free(group_inputs); + free(group_out); + return rc; +} + +int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings) { + embed_context_input_t input = { + .input = {token_ids, n_tokens}, + .spans = spans, + .n_spans = n_spans, + }; + return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings); +} + +float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { + if (!ctx || !token_ids || n_tokens <= 0) + return NULL; + + int hidden = ctx->config.hidden_size; + size_t out_bytes; + if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0) + return NULL; + float *out = (float *)malloc(out_bytes); + if (!out) + return NULL; + + if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) { + free(out); + return NULL; + } + return out; +} |
