aboutsummaryrefslogtreecommitdiff
path: root/embed_mlx.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
commitd1ba1ebc06ef0d62ca932388fae83e6992594719 (patch)
treef962207b779016593258f990348216999a58766a /embed_mlx.c
parentbe20dfda0f836808dccfc14db244592e41ba28bc (diff)
downloadffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz
ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'embed_mlx.c')
-rw-r--r--embed_mlx.c2752
1 files changed, 0 insertions, 2752 deletions
diff --git a/embed_mlx.c b/embed_mlx.c
deleted file mode 100644
index 7cd9357..0000000
--- a/embed_mlx.c
+++ /dev/null
@@ -1,2752 +0,0 @@
-#include "embed_mlx.h"
-#include "embed_config.h"
-#include "embed.h"
-#include "qwen_safetensors.h"
-
-#include <mlx/c/mlx.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <limits.h>
-#include <float.h>
-
-/* ========================================================================
- * Per-layer MLX weight arrays
- * ======================================================================== */
-
-typedef struct {
- mlx_array w;
- mlx_array wt;
- mlx_array scales;
- mlx_array biases;
- int quantized;
- int bits;
- int group_size;
-} mlx_linear_t;
-
-typedef struct {
- mlx_linear_t wq, wk, wv, wo;
- mlx_array q_bias, k_bias, v_bias;
- mlx_array q_norm, k_norm;
- mlx_array input_norm, post_attn_norm;
- mlx_linear_t gate_proj, up_proj, down_proj;
- /* BERT family (unset for Qwen3; see embed_internal.h). wo/up_proj/down_proj
- * and input_norm/post_attn_norm are reused as the BERT attention output,
- * FFN matrices, and the two block LayerNorm weights. */
- mlx_array o_bias, attn_ln_bias, ffn_inter_bias, ffn_out_bias, ffn_ln_bias;
-} mlx_layer_t;
-
-struct embed_mlx_ctx {
- embed_config_t config;
- mlx_array embed_tokens;
- mlx_layer_t *layers; /* heap-allocated [n_layers] */
- mlx_array norm;
- /* BERT family (unset for Qwen3): learned positions, token-type, embed LN. */
- mlx_array position_embeddings, token_type_embeddings, embed_ln_w, embed_ln_b;
- mlx_stream stream;
- multi_safetensors_t *ms;
- int layer_start; /* loaded transformer range, inclusive */
- int layer_end; /* loaded transformer range, exclusive */
- int quantize_bits;
- int quantize_group_size;
- mlx_dtype weight_dtype;
-};
-
-struct embed_mlx_late_ctx {
- embed_mlx_ctx_t *base;
- multi_safetensors_t *dense_ms;
- mlx_linear_t projection;
- int token_dim;
-};
-
-struct embed_mlx_late_vectors {
- embed_mlx_late_ctx_t *owner;
- mlx_array vectors; /* [tokens, dim] */
- int tokens;
- int dim;
-};
-
-/* ========================================================================
- * Helpers
- * ======================================================================== */
-
-static int arr_ok(mlx_array a) { return a.ctx != NULL; }
-
-static int mlx_mul_size(size_t a, size_t b, size_t *out) {
- if (a != 0 && b > SIZE_MAX / a)
- return -1;
- *out = a * b;
- return 0;
-}
-
-static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; }
-
-static size_t safetensor_dtype_size(safetensor_dtype_t dtype) {
- switch (dtype) {
- case DTYPE_F32:
- return sizeof(float);
- case DTYPE_BF16:
- return sizeof(uint16_t);
- default:
- return 0;
- }
-}
-
-static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int *shape,
- int ndim) {
- size_t elem_size = safetensor_dtype_size(t->dtype);
- if (elem_size == 0) {
- fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
- return 0;
- }
- if (t->ndim != ndim) {
- fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
- return 0;
- }
-
- size_t numel = 1;
- for (int i = 0; i < ndim; i++) {
- if (shape[i] <= 0 || t->shape[i] != shape[i]) {
- fprintf(stderr,
- "mlx: bad shape for %s at dim %d: "
- "got %lld, expected %d\n",
- name, i, (long long)t->shape[i], shape[i]);
- return 0;
- }
- if (mlx_mul_size(numel, (size_t)shape[i], &numel) != 0) {
- fprintf(stderr, "mlx: shape too large for %s\n", name);
- return 0;
- }
- }
-
- size_t bytes;
- if (mlx_mul_size(numel, elem_size, &bytes) != 0) {
- fprintf(stderr, "mlx: tensor data too large for %s\n", name);
- return 0;
- }
- if (t->data_size != bytes) {
- fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
- bytes);
- return 0;
- }
-
- if (sf->header_size > SIZE_MAX - 8)
- return 0;
- size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
- t->data_size > sf->file_size - data_start - t->data_offset) {
- fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name);
- return 0;
- }
- return 1;
-}
-
-static mlx_array
-load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
- if (!t) {
- fprintf(stderr, "mlx: tensor not found: %s\n", name);
- return (mlx_array){0};
- }
- if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim))
- return (mlx_array){0};
- mlx_dtype dtype;
- if (t->dtype == DTYPE_F32) {
- dtype = MLX_FLOAT32;
- } else if (t->dtype == DTYPE_BF16) {
- dtype = MLX_BFLOAT16;
- } else {
- fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
- return (mlx_array){0};
- }
- return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype);
-}
-
-static mlx_linear_t
-load_linear_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
- mlx_linear_t l = {0};
- l.w = load_tensor(ms, name, shape, ndim);
- return l;
-}
-
-static int linear_ok(mlx_linear_t l) { return arr_ok(l.w) || arr_ok(l.wt); }
-
-static void free_linear(mlx_linear_t *l) {
- mlx_array_free(l->w);
- mlx_array_free(l->wt);
- mlx_array_free(l->scales);
- mlx_array_free(l->biases);
- memset(l, 0, sizeof(*l));
-}
-
-static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) {
- if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt))
- return 0;
-
- mlx_array transposed = mlx_array_new();
- mlx_transpose(&transposed, l->w, s);
- if (!arr_ok(transposed)) {
- mlx_array_free(transposed);
- fprintf(stderr, "mlx: failed to prepare transposed linear weight\n");
- return -1;
- }
-
- l->wt = mlx_array_new();
- mlx_contiguous(&l->wt, transposed, false, s);
- mlx_array_free(transposed);
- if (!arr_ok(l->wt)) {
- mlx_array_free(l->wt);
- l->wt = (mlx_array){0};
- fprintf(stderr, "mlx: failed to materialize transposed linear weight\n");
- return -1;
- }
- mlx_array_eval(l->wt);
- mlx_synchronize(s);
- mlx_array_free(l->w);
- l->w = (mlx_array){0};
- return 0;
-}
-
-static int quantize_linear(mlx_linear_t *l, int bits, int group_size, mlx_stream s) {
- if (!l || !arr_ok(l->w) || bits == 0)
- return 0;
- if (bits != 8) {
- fprintf(stderr, "mlx: quantized weights support only 8 bits\n");
- return -1;
- }
- if (group_size <= 0)
- group_size = 64;
-
- mlx_optional_int gs = {.value = group_size, .has_value = true};
- mlx_optional_int qb = {.value = bits, .has_value = true};
- mlx_array null_arr = (mlx_array){0};
- mlx_vector_array q = mlx_vector_array_new();
- mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s);
- if (!q.ctx || mlx_vector_array_size(q) < 2) {
- mlx_vector_array_free(q);
- fprintf(stderr, "mlx: quantize failed\n");
- return -1;
- }
-
- mlx_array qw = (mlx_array){0};
- mlx_array scales = (mlx_array){0};
- mlx_array biases = (mlx_array){0};
- mlx_vector_array_get(&qw, q, 0);
- mlx_vector_array_get(&scales, q, 1);
- if (mlx_vector_array_size(q) >= 3)
- mlx_vector_array_get(&biases, q, 2);
- mlx_vector_array_free(q);
-
- if (!arr_ok(qw) || !arr_ok(scales)) {
- mlx_array_free(qw);
- mlx_array_free(scales);
- mlx_array_free(biases);
- fprintf(stderr, "mlx: quantize returned invalid arrays\n");
- return -1;
- }
- mlx_array_eval(qw);
- mlx_array_eval(scales);
- if (arr_ok(biases))
- mlx_array_eval(biases);
- mlx_synchronize(s);
-
- mlx_array_free(l->w);
- mlx_array_free(l->wt);
- l->w = qw;
- l->wt = (mlx_array){0};
- l->scales = scales;
- l->biases = biases;
- l->quantized = 1;
- l->bits = bits;
- l->group_size = group_size;
- return 0;
-}
-
-/* y = x @ W^T */
-static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) {
- if (W->quantized) {
- if (!arr_ok(W->w))
- return (mlx_array){0};
- mlx_array res = mlx_array_new();
- mlx_optional_int gs = {.value = W->group_size, .has_value = true};
- mlx_optional_int bits = {.value = W->bits, .has_value = true};
- mlx_array null_arr = (mlx_array){0};
- mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr,
- true, gs, bits, "affine", s);
- return res;
- }
- mlx_array Wt;
- int free_wt = 0;
- if (arr_ok(W->wt)) {
- Wt = W->wt;
- } else if (arr_ok(W->w)) {
- Wt = mlx_array_new();
- mlx_transpose(&Wt, W->w, s);
- free_wt = 1;
- } else {
- return (mlx_array){0};
- }
- mlx_array res = mlx_array_new();
- if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && mlx_array_dtype(x) == MLX_FLOAT32) {
- /* F32 activations with BF16 weights: upcast the weight operand and run
- * the matmul in F32 (so accumulation is F32). Only the weights are
- * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers),
- * matching the CPU/CUDA accuracy while keeping BF16 weight storage. */
- mlx_array Wf = mlx_array_new();
- mlx_astype(&Wf, Wt, MLX_FLOAT32, s);
- mlx_matmul(&res, x, Wf, s);
- mlx_array_free(Wf);
- } else {
- mlx_matmul(&res, x, Wt, s);
- }
- if (free_wt)
- mlx_array_free(Wt);
- return res;
-}
-
-const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) {
- return ctx ? &ctx->config : NULL;
-}
-
-/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the
- * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the
- * residual stream in F32 while keeping the large matmul weights in BF16. */
-static void upcast_f32_inplace(mlx_array *a, mlx_stream s) {
- if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32)
- return;
- mlx_array f = mlx_array_new();
- mlx_astype(&f, *a, MLX_FLOAT32, s);
- mlx_array_free(*a);
- *a = f;
-}
-
-/* ========================================================================
- * Load
- * ======================================================================== */
-
-static int mlx_model_dir_has_file(const char *model_dir, const char *rel) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
- if (n < 0 || (size_t)n >= sizeof(path))
- return 0;
- FILE *f = fopen(path, "rb");
- if (!f)
- return 0;
- fclose(f);
- return 1;
-}
-
-static int mlx_model_dir_has_late_projection(const char *model_dir) {
- return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") &&
- mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors");
-}
-
-static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) {
- embed_mlx_options_t out = {0};
- if (opts)
- out = *opts;
- if (out.quantize_bits != 0 && out.quantize_bits != 8) {
- fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n");
- out.quantize_bits = -1;
- }
- if (out.quantize_bits && out.quantize_group_size <= 0)
- out.quantize_group_size = 64;
- return out;
-}
-
-static int quantize_layer(mlx_layer_t *l, int bits, int group_size, mlx_stream s) {
- if (bits == 0)
- return 0;
- return quantize_linear(&l->wq, bits, group_size, s) ||
- quantize_linear(&l->wk, bits, group_size, s) ||
- quantize_linear(&l->wv, bits, group_size, s) ||
- quantize_linear(&l->wo, bits, group_size, s) ||
- quantize_linear(&l->gate_proj, bits, group_size, s) ||
- quantize_linear(&l->up_proj, bits, group_size, s) ||
- quantize_linear(&l->down_proj, bits, group_size, s);
-}
-
-static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) {
- return prepare_linear_transpose(&l->wq, s) || prepare_linear_transpose(&l->wk, s) ||
- prepare_linear_transpose(&l->wv, s) || prepare_linear_transpose(&l->wo, s) ||
- prepare_linear_transpose(&l->gate_proj, s) || prepare_linear_transpose(&l->up_proj, s) ||
- prepare_linear_transpose(&l->down_proj, s);
-}
-
-/* BERT-family MLX loader, mirroring load_bert_weights on the CPU side. The
- * shared post-load below (stream, transposes, optional BF16 upcast) runs after
- * either family fills ctx->layers. */
-static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int layer_end) {
- const embed_config_t *c = &ctx->config;
- int h = c->hidden_size, inter = c->intermediate_size;
- char name[256];
-
- const char *p;
- if (multi_safetensors_find(ctx->ms, "embeddings.word_embeddings.weight", NULL))
- p = "";
- else if (multi_safetensors_find(ctx->ms, "bert.embeddings.word_embeddings.weight", NULL))
- p = "bert.";
- else {
- fprintf(stderr, "mlx: BERT word embeddings not found\n");
- return -1;
- }
-
- if (layer_start == 0) {
- int emb_shape[] = {c->vocab_size, h};
- snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p);
- ctx->embed_tokens = load_tensor(ctx->ms, name, emb_shape, 2);
- int pos_shape[] = {c->max_position_embeddings, h};
- snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p);
- ctx->position_embeddings = load_tensor(ctx->ms, name, pos_shape, 2);
- int type_shape[] = {2, h};
- snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p);
- ctx->token_type_embeddings = load_tensor(ctx->ms, name, type_shape, 2);
- int ln_shape[] = {h};
- snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p);
- ctx->embed_ln_w = load_tensor(ctx->ms, name, ln_shape, 1);
- snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p);
- ctx->embed_ln_b = load_tensor(ctx->ms, name, ln_shape, 1);
- if (!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->position_embeddings) ||
- !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->embed_ln_w) ||
- !arr_ok(ctx->embed_ln_b))
- return -1;
- }
-
- ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
- if (!ctx->layers)
- return -1;
-
- for (int i = layer_start; i < layer_end; i++) {
- mlx_layer_t *l = &ctx->layers[i];
-#define LDL(fld, suffix, ...) \
- do { \
- snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!linear_ok(l->fld)) \
- return -1; \
- } while (0)
-#define LDV(fld, suffix, ...) \
- do { \
- snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!arr_ok(l->fld)) \
- return -1; \
- } while (0)
-
- LDL(wq, "attention.self.query.weight", h, h);
- LDV(q_bias, "attention.self.query.bias", h);
- LDL(wk, "attention.self.key.weight", h, h);
- LDV(k_bias, "attention.self.key.bias", h);
- LDL(wv, "attention.self.value.weight", h, h);
- LDV(v_bias, "attention.self.value.bias", h);
- LDL(wo, "attention.output.dense.weight", h, h);
- LDV(o_bias, "attention.output.dense.bias", h);
- LDV(input_norm, "attention.output.LayerNorm.weight", h);
- LDV(attn_ln_bias, "attention.output.LayerNorm.bias", h);
- LDL(up_proj, "intermediate.dense.weight", inter, h);
- LDV(ffn_inter_bias, "intermediate.dense.bias", inter);
- LDL(down_proj, "output.dense.weight", h, inter);
- LDV(ffn_out_bias, "output.dense.bias", h);
- LDV(post_attn_norm, "output.LayerNorm.weight", h);
- LDV(ffn_ln_bias, "output.LayerNorm.bias", h);
-#undef LDL
-#undef LDV
- }
- return 0;
-}
-
-static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
- int layer_start,
- int layer_end,
- const embed_mlx_options_t *opts,
- int allow_late) {
- embed_mlx_options_t options = normalize_mlx_options(opts);
- if (options.quantize_bits < 0)
- return NULL;
- if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "mlx: late-interaction models require token-level "
- "MaxSim support and are not valid pooled embedding models "
- "yet\n");
- return NULL;
- }
-
- embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t));
- if (!ctx)
- return NULL;
- ctx->quantize_bits = options.quantize_bits;
- ctx->quantize_group_size = options.quantize_group_size;
-
- if (embed_config_parse(&ctx->config, model_dir) != 0) {
- free(ctx);
- return NULL;
- }
-
- const embed_config_t *c = &ctx->config;
- if (layer_end == -1)
- layer_end = c->n_layers;
- if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) {
- fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", layer_start,
- layer_end, c->n_layers);
- free(ctx);
- return NULL;
- }
- ctx->layer_start = layer_start;
- ctx->layer_end = layer_end;
-
- ctx->ms = multi_safetensors_open(model_dir);
- if (!ctx->ms) {
- fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
- free(ctx);
- return NULL;
- }
-
- if (c->family == EMBED_FAMILY_BERT) {
- if (load_mlx_bert_weights(ctx, layer_start, layer_end) != 0)
- goto fail;
- goto loaded;
- }
-
- const char *weight_prefix = multi_safetensors_weight_prefix(ctx->ms, "embed_tokens.weight");
- if (!weight_prefix) {
- fprintf(stderr, "mlx: neither embed_tokens.weight nor "
- "model.embed_tokens.weight was found\n");
- goto fail;
- }
-
- int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
- int inter = c->intermediate_size, hd = c->head_dim;
- char name[256];
-
- /* Only the first stage needs token embeddings. */
- if (layer_start == 0) {
- int emb_shape[] = {c->vocab_size, h};
- snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix);
- ctx->embed_tokens = load_tensor(ctx->ms, name, emb_shape, 2);
- if (!arr_ok(ctx->embed_tokens))
- goto fail;
- }
-
- /* Layers */
- ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
- if (!ctx->layers)
- goto fail;
-
- for (int i = layer_start; i < layer_end; i++) {
- mlx_layer_t *l = &ctx->layers[i];
-
-#define LD(fld, fmt, ...) \
- do { \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!linear_ok(l->fld)) \
- goto fail; \
- } while (0)
-
- LD(wq, "%slayers.%d.self_attn.q_proj.weight", qd, h);
- LD(wk, "%slayers.%d.self_attn.k_proj.weight", kvd, h);
- LD(wv, "%slayers.%d.self_attn.v_proj.weight", kvd, h);
- LD(wo, "%slayers.%d.self_attn.o_proj.weight", h, qd);
-#undef LD
-
-#define LD(fld, fmt, ...) \
- do { \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!arr_ok(l->fld)) \
- goto fail; \
- } while (0)
-
- if (c->qkv_bias) {
- LD(q_bias, "%slayers.%d.self_attn.q_proj.bias", qd);
- LD(k_bias, "%slayers.%d.self_attn.k_proj.bias", kvd);
- LD(v_bias, "%slayers.%d.self_attn.v_proj.bias", kvd);
- }
- if (c->qk_norm) {
- LD(q_norm, "%slayers.%d.self_attn.q_norm.weight", hd);
- LD(k_norm, "%slayers.%d.self_attn.k_norm.weight", hd);
- }
- LD(input_norm, "%slayers.%d.input_layernorm.weight", h);
- LD(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", h);
-#undef LD
-
-#define LD(fld, fmt, ...) \
- do { \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!linear_ok(l->fld)) \
- goto fail; \
- } while (0)
-
- LD(gate_proj, "%slayers.%d.mlp.gate_proj.weight", inter, h);
- LD(up_proj, "%slayers.%d.mlp.up_proj.weight", inter, h);
- LD(down_proj, "%slayers.%d.mlp.down_proj.weight", h, inter);
-#undef LD
-
- if (embed_verbose >= 2)
- fprintf(stderr, " mlx layer %d loaded\n", i);
- }
-
- if (layer_end == c->n_layers) {
- int ns[] = {h};
- snprintf(name, sizeof(name), "%snorm.weight", weight_prefix);
- ctx->norm = load_tensor(ctx->ms, name, ns, 1);
- if (!arr_ok(ctx->norm))
- goto fail;
- }
-
-loaded:
- ctx->stream = mlx_default_gpu_stream_new();
- ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w);
- /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the
- * tiny norm vectors so they match the F32 activations in mlx_forward_layers;
- * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */
- if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) {
- for (int i = layer_start; i < layer_end; i++) {
- mlx_layer_t *l = &ctx->layers[i];
- upcast_f32_inplace(&l->q_bias, ctx->stream);
- upcast_f32_inplace(&l->k_bias, ctx->stream);
- upcast_f32_inplace(&l->v_bias, ctx->stream);
- upcast_f32_inplace(&l->q_norm, ctx->stream);
- upcast_f32_inplace(&l->k_norm, ctx->stream);
- upcast_f32_inplace(&l->input_norm, ctx->stream);
- upcast_f32_inplace(&l->post_attn_norm, ctx->stream);
- /* BERT block biases (no-ops for Qwen, whose arrays are unset); the
- * two BERT LayerNorm weights reuse input_norm/post_attn_norm above. */
- upcast_f32_inplace(&l->o_bias, ctx->stream);
- upcast_f32_inplace(&l->attn_ln_bias, ctx->stream);
- upcast_f32_inplace(&l->ffn_inter_bias, ctx->stream);
- upcast_f32_inplace(&l->ffn_out_bias, ctx->stream);
- upcast_f32_inplace(&l->ffn_ln_bias, ctx->stream);
- }
- upcast_f32_inplace(&ctx->norm, ctx->stream);
- /* BERT embedding stage (no-ops for Qwen): learned position and token-type
- * embeddings and the embedding LayerNorm all feed the F32 residual. */
- upcast_f32_inplace(&ctx->position_embeddings, ctx->stream);
- upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream);
- upcast_f32_inplace(&ctx->embed_ln_w, ctx->stream);
- upcast_f32_inplace(&ctx->embed_ln_b, ctx->stream);
- }
- if (ctx->quantize_bits) {
- for (int i = layer_start; i < layer_end; i++) {
- if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, ctx->quantize_group_size,
- ctx->stream) != 0)
- goto fail;
- }
- mlx_clear_cache();
- if (embed_verbose >= 1)
- fprintf(stderr,
- "mlx: quantized linear weights to %d-bit "
- "(group_size=%d)\n",
- ctx->quantize_bits, ctx->quantize_group_size);
- } else {
- for (int i = layer_start; i < layer_end; i++) {
- if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0)
- goto fail;
- }
- }
-
- if (embed_verbose >= 2) {
- size_t active = 0, cache = 0, peak = 0;
- mlx_get_active_memory(&active);
- mlx_get_cache_memory(&cache);
- mlx_get_peak_memory(&peak);
- fprintf(stderr,
- "mlx: memory active=%.2f GiB cache=%.2f GiB "
- "peak=%.2f GiB\n",
- (double)active / (1024.0 * 1024.0 * 1024.0),
- (double)cache / (1024.0 * 1024.0 * 1024.0),
- (double)peak / (1024.0 * 1024.0 * 1024.0));
- }
-
- if (embed_verbose >= 1)
- fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h);
-
- return ctx;
-
-fail:
- embed_mlx_free(ctx);
- return NULL;
-}
-
-embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) {
- return mlx_load_range_ex(model_dir, 0, -1, NULL, 0);
-}
-
-embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts) {
- return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
-}
-
-/* ========================================================================
- * Free
- * ======================================================================== */
-
-static void free_mlx_layer(mlx_layer_t *l) {
- free_linear(&l->wq);
- free_linear(&l->wk);
- free_linear(&l->wv);
- free_linear(&l->wo);
- mlx_array_free(l->q_bias);
- mlx_array_free(l->k_bias);
- mlx_array_free(l->v_bias);
- mlx_array_free(l->q_norm);
- mlx_array_free(l->k_norm);
- mlx_array_free(l->input_norm);
- mlx_array_free(l->post_attn_norm);
- free_linear(&l->gate_proj);
- free_linear(&l->up_proj);
- free_linear(&l->down_proj);
- mlx_array_free(l->o_bias);
- mlx_array_free(l->attn_ln_bias);
- mlx_array_free(l->ffn_inter_bias);
- mlx_array_free(l->ffn_out_bias);
- mlx_array_free(l->ffn_ln_bias);
-}
-
-void embed_mlx_free(embed_mlx_ctx_t *ctx) {
- if (!ctx)
- return;
- mlx_array_free(ctx->embed_tokens);
- if (ctx->layers) {
- for (int i = 0; i < ctx->config.n_layers; i++)
- free_mlx_layer(&ctx->layers[i]);
- free(ctx->layers);
- }
- mlx_array_free(ctx->norm);
- mlx_array_free(ctx->position_embeddings);
- mlx_array_free(ctx->token_type_embeddings);
- mlx_array_free(ctx->embed_ln_w);
- mlx_array_free(ctx->embed_ln_b);
- if (ctx->stream.ctx)
- mlx_stream_free(ctx->stream);
- if (ctx->ms)
- multi_safetensors_close(ctx->ms);
- free(ctx);
-}
-
-embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts) {
- if (!mlx_model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "mlx late: missing 1_Dense projection\n");
- return NULL;
- }
-
- embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late));
- if (!late)
- return NULL;
-
- late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1);
- if (!late->base)
- goto fail;
-
- char dense_dir[1024];
- int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir);
- if (n < 0 || (size_t)n >= sizeof(dense_dir)) {
- fprintf(stderr, "mlx late: model path too long\n");
- goto fail;
- }
-
- late->dense_ms = multi_safetensors_open(dense_dir);
- if (!late->dense_ms) {
- fprintf(stderr, "mlx late: failed to open safetensors in %s\n", dense_dir);
- goto fail;
- }
-
- const safetensor_t *t = multi_safetensors_find(late->dense_ms, "linear.weight", NULL);
- if (!t) {
- fprintf(stderr, "mlx late: missing linear.weight\n");
- goto fail;
- }
- if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX ||
- t->shape[1] != late->base->config.hidden_size) {
- fprintf(stderr, "mlx late: bad projection shape\n");
- goto fail;
- }
-
- int shape[] = {(int)t->shape[0], late->base->config.hidden_size};
- late->projection = load_linear_tensor(late->dense_ms, "linear.weight", shape, 2);
- if (!linear_ok(late->projection))
- goto fail;
- if (prepare_linear_transpose(&late->projection, late->base->stream) != 0)
- goto fail;
- late->token_dim = shape[0];
-
- if (embed_verbose >= 1)
- fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim);
- return late;
-
-fail:
- embed_mlx_late_free(late);
- return NULL;
-}
-
-embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) {
- return embed_mlx_late_load_with_options(model_dir, NULL);
-}
-
-void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) {
- if (!ctx)
- return;
- free_linear(&ctx->projection);
- if (ctx->dense_ms)
- multi_safetensors_close(ctx->dense_ms);
- embed_mlx_free(ctx->base);
- free(ctx);
-}
-
-const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) {
- return ctx && ctx->base ? &ctx->base->config : NULL;
-}
-
-int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
-
-/* ========================================================================
- * Forward pass
- * ======================================================================== */
-
-/* Exact erf GeLU: 0.5 * x * (1 + erf(x / sqrt(2))). Frees nothing it is given. */
-static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) {
- mlx_array c = mlx_array_new_float32(0.70710678118654752f);
- mlx_array scaled = mlx_array_new();
- mlx_multiply(&scaled, x, c, S);
- mlx_array_free(c);
- mlx_array e = mlx_array_new();
- mlx_erf(&e, scaled, S);
- mlx_array_free(scaled);
- mlx_array one = mlx_array_new_float32(1.0f);
- mlx_array e1 = mlx_array_new();
- mlx_add(&e1, e, one, S);
- mlx_array_free(e);
- mlx_array_free(one);
- mlx_array half = mlx_array_new_float32(0.5f);
- mlx_array xh = mlx_array_new();
- mlx_multiply(&xh, x, half, S);
- mlx_array_free(half);
- mlx_array g = mlx_array_new();
- mlx_multiply(&g, xh, e1, S);
- mlx_array_free(xh);
- mlx_array_free(e1);
- return g;
-}
-
-/* Tanh-approximation GeLU (gelu_new / gelu_pytorch_tanh):
- * 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). Frees nothing given. */
-static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) {
- mlx_array x2 = mlx_array_new();
- mlx_multiply(&x2, x, x, S);
- mlx_array x3 = mlx_array_new();
- mlx_multiply(&x3, x2, x, S);
- mlx_array_free(x2);
- mlx_array c = mlx_array_new_float32(0.044715f);
- mlx_array cx3 = mlx_array_new();
- mlx_multiply(&cx3, x3, c, S);
- mlx_array_free(x3);
- mlx_array_free(c);
- mlx_array sum = mlx_array_new();
- mlx_add(&sum, x, cx3, S);
- mlx_array_free(cx3);
- mlx_array s2pi = mlx_array_new_float32(0.79788456080286536f);
- mlx_array inner = mlx_array_new();
- mlx_multiply(&inner, sum, s2pi, S);
- mlx_array_free(sum);
- mlx_array_free(s2pi);
- mlx_array t = mlx_array_new();
- mlx_tanh(&t, inner, S);
- mlx_array_free(inner);
- mlx_array one = mlx_array_new_float32(1.0f);
- mlx_array t1 = mlx_array_new();
- mlx_add(&t1, t, one, S);
- mlx_array_free(t);
- mlx_array_free(one);
- mlx_array half = mlx_array_new_float32(0.5f);
- mlx_array xh = mlx_array_new();
- mlx_multiply(&xh, x, half, S);
- mlx_array_free(half);
- mlx_array g = mlx_array_new();
- mlx_multiply(&g, xh, t1, S);
- mlx_array_free(xh);
- mlx_array_free(t1);
- return g;
-}
-
-/* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU
- * feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the
- * x it is given and returns the new hidden states. */
-static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
- mlx_array x,
- int batch,
- int max_seq,
- int has_padding,
- mlx_array attn_mask,
- int layer_start,
- int layer_end) {
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
- int n_heads = c->n_heads, head_dim = c->head_dim, q_dim = c->q_dim;
- float scale = 1.0f / sqrtf((float)head_dim);
- float eps = c->layer_norm_eps;
- mlx_array null_arr = (mlx_array){0};
-
- for (int layer = layer_start; layer < layer_end; layer++) {
- mlx_layer_t *l = &ctx->layers[layer];
-
- /* Self-attention reads x directly (post-norm). */
- mlx_array q_flat = linear(x, &l->wq, S);
- mlx_array k_flat = linear(x, &l->wk, S);
- mlx_array v_flat = linear(x, &l->wv, S);
- mlx_array t = mlx_array_new();
- mlx_add(&t, q_flat, l->q_bias, S);
- mlx_array_free(q_flat);
- q_flat = t;
- t = mlx_array_new();
- mlx_add(&t, k_flat, l->k_bias, S);
- mlx_array_free(k_flat);
- k_flat = t;
- t = mlx_array_new();
- mlx_add(&t, v_flat, l->v_bias, S);
- mlx_array_free(v_flat);
- v_flat = t;
-
- int s4[] = {batch, max_seq, n_heads, head_dim};
- mlx_array q = mlx_array_new(), k = mlx_array_new(), v = mlx_array_new();
- mlx_reshape(&q, q_flat, s4, 4, S);
- mlx_reshape(&k, k_flat, s4, 4, S);
- mlx_reshape(&v, v_flat, s4, 4, S);
- mlx_array_free(q_flat);
- mlx_array_free(k_flat);
- mlx_array_free(v_flat);
-
- int perm[] = {0, 2, 1, 3};
- mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
- mlx_transpose_axes(&qt, q, perm, 4, S);
- mlx_transpose_axes(&kt, k, perm, 4, S);
- mlx_transpose_axes(&vt, v, perm, 4, S);
- mlx_array_free(q);
- mlx_array_free(k);
- mlx_array_free(v);
-
- mlx_array attn = mlx_array_new();
- mlx_fast_scaled_dot_product_attention(&attn, qt, kt, vt, scale, "",
- has_padding ? attn_mask : null_arr, null_arr, S);
- mlx_array_free(qt);
- mlx_array_free(kt);
- mlx_array_free(vt);
-
- mlx_array attn_t = mlx_array_new();
- mlx_transpose_axes(&attn_t, attn, perm, 4, S);
- mlx_array_free(attn);
- int s3[] = {batch, max_seq, q_dim};
- mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_t, s3, 3, S);
- mlx_array_free(attn_t);
-
- mlx_array proj = linear(attn_flat, &l->wo, S);
- mlx_array_free(attn_flat);
- mlx_array proj_b = mlx_array_new();
- mlx_add(&proj_b, proj, l->o_bias, S);
- mlx_array_free(proj);
- mlx_array res = mlx_array_new();
- mlx_add(&res, x, proj_b, S);
- mlx_array_free(x);
- mlx_array_free(proj_b);
- x = mlx_array_new();
- mlx_fast_layer_norm(&x, res, l->input_norm, l->attn_ln_bias, eps, S);
- mlx_array_free(res);
-
- /* Feed-forward: dense -> GeLU -> dense. */
- mlx_array mid = linear(x, &l->up_proj, S);
- mlx_array mid_b = mlx_array_new();
- mlx_add(&mid_b, mid, l->ffn_inter_bias, S);
- mlx_array_free(mid);
- mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S)
- : mlx_gelu_erf(mid_b, S);
- mlx_array_free(mid_b);
- mlx_array out = linear(g, &l->down_proj, S);
- mlx_array_free(g);
- mlx_array out_b = mlx_array_new();
- mlx_add(&out_b, out, l->ffn_out_bias, S);
- mlx_array_free(out);
- mlx_array res2 = mlx_array_new();
- mlx_add(&res2, x, out_b, S);
- mlx_array_free(x);
- mlx_array_free(out_b);
- x = mlx_array_new();
- mlx_fast_layer_norm(&x, res2, l->post_attn_norm, l->ffn_ln_bias, eps, S);
- mlx_array_free(res2);
- }
- return x;
-}
-
-static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
- mlx_array x,
- int batch,
- int max_seq,
- int has_padding,
- mlx_array attn_mask,
- int layer_start,
- int layer_end) {
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
- int n_heads = c->n_heads;
- int n_kv_heads = c->n_kv_heads;
- int head_dim = c->head_dim;
- int q_dim = c->q_dim;
- float scale = 1.0f / sqrtf((float)head_dim);
- mlx_array null_arr = (mlx_array){0};
- const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
-
- /* For non-quantized BF16 weights, run the residual stream, norms, and
- * attention in F32; matmuls downcast their operands to BF16 internally
- * (see linear). This keeps weight storage at BF16 while avoiding the
- * cross-layer accumulation error of a fully BF16 forward. F32 and quantized
- * models are unaffected. */
- mlx_array local_mask = attn_mask;
- int free_mask = 0;
- int mixed_bf16 =
- mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized;
- if (mixed_bf16) {
- if (mlx_array_dtype(x) != MLX_FLOAT32) {
- mlx_array xf = mlx_array_new();
- mlx_astype(&xf, x, MLX_FLOAT32, S);
- mlx_array_free(x);
- x = xf;
- }
- if (has_padding && arr_ok(attn_mask) && mlx_array_dtype(attn_mask) != MLX_FLOAT32) {
- local_mask = mlx_array_new();
- mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S);
- free_mask = 1;
- }
- }
-
- for (int layer = layer_start; layer < layer_end; layer++) {
- mlx_layer_t *l = &ctx->layers[layer];
-
- mlx_array xn = mlx_array_new();
- mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S);
-
- mlx_array q_flat = linear(xn, &l->wq, S);
- mlx_array k_flat = linear(xn, &l->wk, S);
- mlx_array v_flat = linear(xn, &l->wv, S);
- mlx_array_free(xn);
- if (c->qkv_bias) {
- mlx_array biased = mlx_array_new();
- mlx_add(&biased, q_flat, l->q_bias, S);
- mlx_array_free(q_flat);
- q_flat = biased;
- biased = mlx_array_new();
- mlx_add(&biased, k_flat, l->k_bias, S);
- mlx_array_free(k_flat);
- k_flat = biased;
- biased = mlx_array_new();
- mlx_add(&biased, v_flat, l->v_bias, S);
- mlx_array_free(v_flat);
- v_flat = biased;
- }
-
- int q_shape[] = {batch, max_seq, n_heads, head_dim};
- int k_shape[] = {batch, max_seq, n_kv_heads, head_dim};
- mlx_array q = mlx_array_new();
- mlx_array k = mlx_array_new();
- mlx_array v = mlx_array_new();
- mlx_reshape(&q, q_flat, q_shape, 4, S);
- mlx_reshape(&k, k_flat, k_shape, 4, S);
- mlx_reshape(&v, v_flat, k_shape, 4, S);
- mlx_array_free(q_flat);
- mlx_array_free(k_flat);
- mlx_array_free(v_flat);
-
- mlx_array qn = q;
- mlx_array kn = k;
- if (c->qk_norm) {
- qn = mlx_array_new();
- kn = mlx_array_new();
- mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
- mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
- mlx_array_free(q);
- mlx_array_free(k);
- }
-
- /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */
- int perm[] = {0, 2, 1, 3};
- mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
- mlx_transpose_axes(&qt, qn, perm, 4, S);
- mlx_transpose_axes(&kt, kn, perm, 4, S);
- mlx_transpose_axes(&vt, v, perm, 4, S);
- mlx_array_free(qn);
- mlx_array_free(kn);
- mlx_array_free(v);
-
- mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
- mlx_array qr = mlx_array_new(), kr = mlx_array_new();
- mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_array_free(qt);
- mlx_array_free(kt);
-
- mlx_array attn = mlx_array_new();
- mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, mask_mode,
- has_padding ? local_mask : null_arr, null_arr, S);
- mlx_array_free(qr);
- mlx_array_free(kr);
- mlx_array_free(vt);
-
- mlx_array attn_t = mlx_array_new();
- mlx_transpose_axes(&attn_t, attn, perm, 4, S);
- mlx_array_free(attn);
-
- int flat_shape[] = {batch, max_seq, q_dim};
- mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S);
- mlx_array_free(attn_t);
-
- mlx_array proj = linear(attn_flat, &l->wo, S);
- mlx_array_free(attn_flat);
- mlx_array x2 = mlx_array_new();
- mlx_add(&x2, x, proj, S);
- mlx_array_free(x);
- mlx_array_free(proj);
- x = x2;
-
- mlx_array xn2 = mlx_array_new();
- mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
-
- mlx_array gate = linear(xn2, &l->gate_proj, S);
- mlx_array up = linear(xn2, &l->up_proj, S);
- mlx_array_free(xn2);
-
- mlx_array gate_sig = mlx_array_new();
- mlx_sigmoid(&gate_sig, gate, S);
- mlx_array silu_gate = mlx_array_new();
- mlx_multiply(&silu_gate, gate, gate_sig, S);
- mlx_array_free(gate);
- mlx_array_free(gate_sig);
-
- mlx_array mid = mlx_array_new();
- mlx_multiply(&mid, silu_gate, up, S);
- mlx_array_free(silu_gate);
- mlx_array_free(up);
-
- mlx_array ffn = linear(mid, &l->down_proj, S);
- mlx_array_free(mid);
-
- mlx_array x3 = mlx_array_new();
- mlx_add(&x3, x, ffn, S);
- mlx_array_free(x);
- mlx_array_free(ffn);
- x = x3;
- }
- if (free_mask)
- mlx_array_free(local_mask);
- return x;
-}
-
-embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize) {
- if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 ||
- !linear_ok(ctx->projection))
- return NULL;
-
- embed_mlx_ctx_t *base = ctx->base;
- if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
- !arr_ok(base->embed_tokens) || !arr_ok(base->norm))
- return NULL;
-
- const embed_config_t *c = &base->config;
- mlx_stream S = base->stream;
- for (int i = 0; i < n_tokens; i++) {
- if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
- fprintf(stderr, "mlx late: invalid token id %d at position %d\n", token_ids[i], i);
- return NULL;
- }
- }
-
- size_t id_bytes;
- if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0)
- return NULL;
-
- int *ids_data = (int *)malloc(id_bytes);
- if (!ids_data)
- return NULL;
- memcpy(ids_data, token_ids, id_bytes);
-
- int ids_shape[] = {1, n_tokens};
- mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32);
- free(ids_data);
- if (!arr_ok(ids))
- return NULL;
-
- embed_mlx_late_vectors_t *result = NULL;
- mlx_array x = mlx_array_new();
- mlx_array x_normed = (mlx_array){0};
- mlx_array projected = (mlx_array){0};
- mlx_array reshaped = (mlx_array){0};
-
- mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
- if (!arr_ok(x))
- goto cleanup;
-
- x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, 0, c->n_layers);
- if (!arr_ok(x))
- goto cleanup;
-
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- x = (mlx_array){0};
- if (!arr_ok(x_normed))
- goto cleanup;
-
- projected = linear(x_normed, &ctx->projection, S);
- mlx_array_free(x_normed);
- x_normed = (mlx_array){0};
- if (!arr_ok(projected))
- goto cleanup;
-
- if (normalize) {
- mlx_array sq = mlx_array_new();
- mlx_square(&sq, projected, S);
- if (!arr_ok(sq)) {
- mlx_array_free(sq);
- goto cleanup;
- }
-
- mlx_array norm2 = mlx_array_new();
- mlx_sum_axis(&norm2, sq, 2, true, S);
- mlx_array_free(sq);
- if (!arr_ok(norm2)) {
- mlx_array_free(norm2);
- goto cleanup;
- }
-
- mlx_array inv_norm = mlx_array_new();
- mlx_rsqrt(&inv_norm, norm2, S);
- mlx_array_free(norm2);
- if (!arr_ok(inv_norm)) {
- mlx_array_free(inv_norm);
- goto cleanup;
- }
-
- mlx_array normalized = mlx_array_new();
- mlx_multiply(&normalized, projected, inv_norm, S);
- mlx_array_free(inv_norm);
- mlx_array_free(projected);
- projected = normalized;
- if (!arr_ok(projected))
- goto cleanup;
- }
-
- int vec_shape[] = {n_tokens, ctx->token_dim};
- reshaped = mlx_array_new();
- mlx_reshape(&reshaped, projected, vec_shape, 2, S);
- if (!arr_ok(reshaped))
- goto cleanup;
-
- result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
- if (!result)
- goto cleanup;
- result->owner = ctx;
- result->vectors = reshaped;
- result->tokens = n_tokens;
- result->dim = ctx->token_dim;
- reshaped = (mlx_array){0};
-
-cleanup:
- mlx_array_free(reshaped);
- mlx_array_free(projected);
- mlx_array_free(x_normed);
- mlx_array_free(x);
- return result;
-}
-
-void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) {
- if (!vecs)
- return;
- mlx_array_free(vecs->vectors);
- free(vecs);
-}
-
-int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) {
- return vecs ? vecs->tokens : 0;
-}
-
-int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) {
- return vecs ? vecs->dim : 0;
-}
-
-int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) {
- if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0)
- return -1;
-
- size_t values, bytes;
- if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 ||
- mlx_mul_size(values, sizeof(float), &bytes) != 0)
- return -1;
-
- mlx_stream S = vecs->owner->base->stream;
- mlx_array out_f32 = mlx_array_new();
- mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S);
- if (!arr_ok(out_f32)) {
- mlx_array_free(out_f32);
- return -1;
- }
-
- int rc = -1;
- mlx_array_eval(out_f32);
- const float *data = mlx_array_data_float32(out_f32);
- if (data) {
- memcpy(out_vectors, data, bytes);
- rc = 0;
- }
- mlx_array_free(out_f32);
- return rc;
-}
-
-/*
- * Batched late-interaction document encoder.
- *
- * Encodes every candidate in one padded transformer pass, then gathers each
- * document's kept token vectors into a single packed [total_keep, token_dim]
- * array in document order. out_offsets receives the prefix sum of kept counts
- * (out_offsets[i + 1] - out_offsets[i] == n_keep[i]), the exact layout
- * embed_mlx_late_maxsim_batch_device consumes. This collapses N per-candidate
- * transformer calls into one forward.
- *
- * Right-padding keeps real tokens at positions 0..n_tokens[d]-1 so RoPE
- * positions are unchanged; a bidirectional model gets a padding attention mask
- * so real tokens never attend to pad slots (the same masking the dense pooled
- * batch path uses). Kept-token vectors therefore match per-document encoding to
- * within MLX's batched-GEMM rounding.
- */
-embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- int *out_offsets) {
- if (!ctx || !ctx->base || !doc_ids || !n_tokens || !keep || !n_keep || !out_offsets ||
- n_docs <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection))
- return NULL;
-
- embed_mlx_ctx_t *base = ctx->base;
- if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
- !arr_ok(base->embed_tokens) || !arr_ok(base->norm))
- return NULL;
-
- const embed_config_t *c = &base->config;
- mlx_stream S = base->stream;
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
-
- /* Validate inputs and size the batch: longest sequence and total kept
- * tokens. out_offsets is the running prefix sum of kept counts. */
- int max_seq = 0;
- int total_keep = 0;
- out_offsets[0] = 0;
- for (int d = 0; d < n_docs; d++) {
- if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
- n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d])
- return NULL;
- if (n_tokens[d] > max_seq)
- max_seq = n_tokens[d];
- for (int k = 0; k < n_keep[d]; k++) {
- if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d])
- return NULL;
- }
- total_keep += n_keep[d];
- out_offsets[d + 1] = total_keep;
- }
-
- /* The gather index lives in an MLX int32 array, so the flattened
- * [n_docs * max_seq, dim] row count must stay within int range. */
- size_t elems, ids_bytes, mask_bytes, gather_bytes;
- if (mlx_mul_size((size_t)n_docs, (size_t)max_seq, &elems) != 0 || elems > INT_MAX ||
- mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 ||
- mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 ||
- mlx_mul_size((size_t)total_keep, sizeof(int), &gather_bytes) != 0)
- return NULL;
-
- int has_padding = 0;
- for (int d = 0; d < n_docs; d++)
- if (n_tokens[d] != max_seq)
- has_padding = 1;
- int needs_attn_mask = has_padding && !causal;
- if (ids_bytes == 0 || gather_bytes == 0 || (needs_attn_mask && mask_bytes == 0))
- return NULL;
-
- int *padded_ids = (int *)malloc(ids_bytes);
- float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
- int *gather_idx = (int *)malloc(gather_bytes);
- if (!padded_ids || (needs_attn_mask && !attn_mask_data) || !gather_idx) {
- free(padded_ids);
- free(attn_mask_data);
- free(gather_idx);
- return NULL;
- }
-
- int gpos = 0;
- for (int d = 0; d < n_docs; d++) {
- for (int t = 0; t < max_seq; t++) {
- size_t idx = (size_t)d * max_seq + t;
- if (t < n_tokens[d]) {
- int id = doc_ids[d][t];
- if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx late: invalid token id %d at doc %d position %d\n", id, d,
- t);
- free(padded_ids);
- free(attn_mask_data);
- free(gather_idx);
- return NULL;
- }
- padded_ids[idx] = id;
- if (needs_attn_mask)
- attn_mask_data[idx] = 0.0f;
- } else {
- padded_ids[idx] = 0;
- if (needs_attn_mask)
- attn_mask_data[idx] = -1.0e9f;
- }
- }
- /* Map each kept token to its row in the flattened projection. */
- for (int k = 0; k < n_keep[d]; k++)
- gather_idx[gpos++] = d * max_seq + keep[d][k];
- }
-
- int ids_shape[] = {n_docs, max_seq};
- int attn_mask_shape[] = {n_docs, 1, 1, max_seq};
- int gather_shape[] = {total_keep};
- mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- mlx_array attn_mask = needs_attn_mask
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array gather = mlx_array_new_data(gather_idx, gather_shape, 1, MLX_INT32);
- free(padded_ids);
- free(attn_mask_data);
- free(gather_idx);
- if (!arr_ok(ids) || (needs_attn_mask && !arr_ok(attn_mask)) || !arr_ok(gather)) {
- mlx_array_free(ids);
- mlx_array_free(attn_mask);
- mlx_array_free(gather);
- return NULL;
- }
- if (needs_attn_mask && mlx_weight_dtype(base) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) {
- mlx_array_free(ids);
- mlx_array_free(gather);
- return NULL;
- }
- }
-
- embed_mlx_late_vectors_t *result = NULL;
- mlx_array x = mlx_array_new();
- mlx_array x_normed = (mlx_array){0};
- mlx_array projected = (mlx_array){0};
- mlx_array flat = (mlx_array){0};
- mlx_array gathered = (mlx_array){0};
-
- mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
- if (!arr_ok(x))
- goto cleanup;
-
- x = mlx_forward_layers(base, x, n_docs, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
- if (!arr_ok(x))
- goto cleanup;
-
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- x = (mlx_array){0};
- if (!arr_ok(x_normed))
- goto cleanup;
-
- projected = linear(x_normed, &ctx->projection, S);
- mlx_array_free(x_normed);
- x_normed = (mlx_array){0};
- if (!arr_ok(projected))
- goto cleanup;
-
- /* Flatten [n_docs, max_seq, dim] -> [n_docs * max_seq, dim] and gather the
- * kept tokens in document order into one packed [total_keep, dim] array. */
- int flat_shape[] = {(int)elems, ctx->token_dim};
- flat = mlx_array_new();
- mlx_reshape(&flat, projected, flat_shape, 2, S);
- mlx_array_free(projected);
- projected = (mlx_array){0};
- if (!arr_ok(flat))
- goto cleanup;
-
- gathered = mlx_array_new();
- mlx_take_axis(&gathered, flat, gather, 0, S);
- mlx_array_free(flat);
- flat = (mlx_array){0};
- if (!arr_ok(gathered))
- goto cleanup;
-
- /* Normalize after gathering: per-token L2 is row-independent, so this is
- * identical to normalizing the full batch but skips the dropped and padded
- * rows entirely. */
- if (normalize) {
- mlx_array sq = mlx_array_new();
- mlx_square(&sq, gathered, S);
- if (!arr_ok(sq)) {
- mlx_array_free(sq);
- goto cleanup;
- }
- mlx_array norm2 = mlx_array_new();
- mlx_sum_axis(&norm2, sq, 1, true, S);
- mlx_array_free(sq);
- if (!arr_ok(norm2)) {
- mlx_array_free(norm2);
- goto cleanup;
- }
- mlx_array inv_norm = mlx_array_new();
- mlx_rsqrt(&inv_norm, norm2, S);
- mlx_array_free(norm2);
- if (!arr_ok(inv_norm)) {
- mlx_array_free(inv_norm);
- goto cleanup;
- }
- mlx_array normalized = mlx_array_new();
- mlx_multiply(&normalized, gathered, inv_norm, S);
- mlx_array_free(inv_norm);
- mlx_array_free(gathered);
- gathered = normalized;
- if (!arr_ok(gathered))
- goto cleanup;
- }
-
- result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
- if (!result)
- goto cleanup;
- result->owner = ctx;
- result->vectors = gathered;
- result->tokens = total_keep;
- result->dim = ctx->token_dim;
- gathered = (mlx_array){0};
-
-cleanup:
- mlx_array_free(gathered);
- mlx_array_free(flat);
- mlx_array_free(projected);
- mlx_array_free(x_normed);
- mlx_array_free(x);
- mlx_array_free(gather);
- mlx_array_free(attn_mask);
- return result;
-}
-
-/* Device-resident select/concat primitives. The server rerank path batches
- * candidates through embed_mlx_late_encode_docs_device, but the late-interaction
- * verification harness (scripts/check_late_interaction.py) drives these directly
- * to validate per-document gather and concat against the CPU MaxSim reference. */
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) {
- if (!ctx || !ctx->base || !items || count <= 0)
- return NULL;
- int dim = 0;
- int total = 0;
- for (int i = 0; i < count; i++) {
- const embed_mlx_late_vectors_t *v = items[i];
- if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0)
- return NULL;
- if (i == 0)
- dim = v->dim;
- if (v->dim != dim || total > INT_MAX - v->tokens)
- return NULL;
- total += v->tokens;
- }
-
- mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts));
- if (!parts)
- return NULL;
- for (int i = 0; i < count; i++)
- parts[i] = items[i]->vectors;
-
- mlx_stream S = ctx->base->stream;
- mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count);
- free(parts);
- if (!vec.ctx)
- return NULL;
-
- mlx_array joined = mlx_array_new();
- mlx_concatenate_axis(&joined, vec, 0, S);
- mlx_vector_array_free(vec);
- if (!arr_ok(joined)) {
- mlx_array_free(joined);
- return NULL;
- }
-
- embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
- if (!out) {
- mlx_array_free(joined);
- return NULL;
- }
- out->owner = ctx;
- out->vectors = joined;
- out->tokens = total;
- out->dim = dim;
- return out;
-}
-
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *vecs,
- const int *token_indices,
- int count) {
- if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) ||
- !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0)
- return NULL;
-
- for (int i = 0; i < count; i++) {
- if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens)
- return NULL;
- }
-
- size_t idx_bytes;
- if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0)
- return NULL;
- int *idx_data = (int *)malloc(idx_bytes);
- if (!idx_data)
- return NULL;
- memcpy(idx_data, token_indices, idx_bytes);
-
- int idx_shape[] = {count};
- mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32);
- free(idx_data);
- if (!arr_ok(idx))
- return NULL;
-
- mlx_stream S = ctx->base->stream;
- mlx_array selected = mlx_array_new();
- mlx_take_axis(&selected, vecs->vectors, idx, 0, S);
- mlx_array_free(idx);
- if (!arr_ok(selected)) {
- mlx_array_free(selected);
- return NULL;
- }
-
- embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
- if (!out) {
- mlx_array_free(selected);
- return NULL;
- }
- out->owner = ctx;
- out->vectors = selected;
- out->tokens = count;
- out->dim = vecs->dim;
- return out;
-}
-
-int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs,
- const int *doc_offsets,
- int docs_count,
- float *scores) {
- if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores ||
- query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim ||
- query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) ||
- !arr_ok(docs->vectors) || doc_offsets[0] != 0)
- return -1;
-
- int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths));
- if (!lengths)
- return -1;
- int max_doc_tokens = 0;
- for (int i = 0; i < docs_count; i++) {
- if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] ||
- doc_offsets[i + 1] > docs->tokens) {
- free(lengths);
- return -1;
- }
- int n = doc_offsets[i + 1] - doc_offsets[i];
- lengths[i] = n;
- if (n > max_doc_tokens)
- max_doc_tokens = n;
- }
- if (max_doc_tokens <= 0) {
- free(lengths);
- return -1;
- }
-
- mlx_stream S = ctx->base->stream;
- int dim = query->dim;
- int rc = -1;
- mlx_array *doc_rows = (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
- if (!doc_rows) {
- free(lengths);
- return -1;
- }
-
- mlx_vector_array doc_vec = (mlx_vector_array){0};
- mlx_array pad_value = mlx_array_new_float32(0.0f);
- mlx_array doc_batch = (mlx_array){0};
- mlx_array query_3d = (mlx_array){0};
- mlx_array query_batched = (mlx_array){0};
- mlx_array docs_t = (mlx_array){0};
- mlx_array sims = (mlx_array){0};
- mlx_array lengths_arr = (mlx_array){0};
- mlx_array positions = (mlx_array){0};
- mlx_array lengths_2d = (mlx_array){0};
- mlx_array positions_2d = (mlx_array){0};
- mlx_array valid = (mlx_array){0};
- mlx_array valid_3d = (mlx_array){0};
- mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX);
- mlx_array masked_sims = (mlx_array){0};
- mlx_array maxes = (mlx_array){0};
- mlx_array sums = (mlx_array){0};
- mlx_array sums_f32 = (mlx_array){0};
-
- for (int i = 0; i < docs_count; i++) {
- int n = lengths[i];
- int start[] = {doc_offsets[i], 0};
- int stop[] = {doc_offsets[i + 1], dim};
- int strides[] = {1, 1};
-
- mlx_array doc = mlx_array_new();
- mlx_array padded = (mlx_array){0};
-
- mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S);
- if (!arr_ok(doc))
- goto cleanup;
-
- if (n < max_doc_tokens) {
- int axes[] = {0};
- int low[] = {0};
- int high[] = {max_doc_tokens - n};
- padded = mlx_array_new();
- mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, "constant", S);
- mlx_array_free(doc);
- doc = padded;
- padded = (mlx_array){0};
- if (!arr_ok(doc))
- goto cleanup;
- }
-
- doc_rows[i] = mlx_array_new();
- mlx_expand_dims(&doc_rows[i], doc, 0, S);
- mlx_array_free(doc);
- if (!arr_ok(doc_rows[i]))
- goto cleanup;
- }
-
- doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count);
- if (!doc_vec.ctx)
- goto cleanup;
-
- doc_batch = mlx_array_new();
- mlx_concatenate_axis(&doc_batch, doc_vec, 0, S);
- if (!arr_ok(doc_batch))
- goto cleanup;
-
- query_3d = mlx_array_new();
- mlx_expand_dims(&query_3d, query->vectors, 0, S);
- if (!arr_ok(query_3d))
- goto cleanup;
-
- int q_shape[] = {docs_count, query->tokens, dim};
- query_batched = mlx_array_new();
- mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S);
- if (!arr_ok(query_batched))
- goto cleanup;
-
- int perm[] = {0, 2, 1};
- docs_t = mlx_array_new();
- mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S);
- if (!arr_ok(docs_t))
- goto cleanup;
-
- sims = mlx_array_new();
- mlx_matmul(&sims, query_batched, docs_t, S);
- if (!arr_ok(sims))
- goto cleanup;
-
- int len_shape[] = {docs_count};
- lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32);
- if (!arr_ok(lengths_arr))
- goto cleanup;
-
- positions = mlx_array_new();
- mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S);
- if (!arr_ok(positions))
- goto cleanup;
-
- lengths_2d = mlx_array_new();
- mlx_expand_dims(&lengths_2d, lengths_arr, 1, S);
- if (!arr_ok(lengths_2d))
- goto cleanup;
-
- positions_2d = mlx_array_new();
- mlx_expand_dims(&positions_2d, positions, 0, S);
- if (!arr_ok(positions_2d))
- goto cleanup;
-
- valid = mlx_array_new();
- mlx_less(&valid, positions_2d, lengths_2d, S);
- if (!arr_ok(valid))
- goto cleanup;
-
- valid_3d = mlx_array_new();
- mlx_expand_dims(&valid_3d, valid, 1, S);
- if (!arr_ok(valid_3d))
- goto cleanup;
-
- masked_sims = mlx_array_new();
- mlx_where(&masked_sims, valid_3d, sims, invalid_value, S);
- if (!arr_ok(masked_sims))
- goto cleanup;
-
- maxes = mlx_array_new();
- mlx_max_axis(&maxes, masked_sims, 2, false, S);
- if (!arr_ok(maxes))
- goto cleanup;
-
- sums = mlx_array_new();
- mlx_sum_axis(&sums, maxes, 1, false, S);
- if (!arr_ok(sums))
- goto cleanup;
-
- sums_f32 = mlx_array_new();
- mlx_astype(&sums_f32, sums, MLX_FLOAT32, S);
- if (!arr_ok(sums_f32))
- goto cleanup;
-
- mlx_array_eval(sums_f32);
- const float *data = mlx_array_data_float32(sums_f32);
- if (!data)
- goto cleanup;
- memcpy(scores, data, (size_t)docs_count * sizeof(float));
- rc = 0;
-
-cleanup:
- mlx_array_free(sums_f32);
- mlx_array_free(sums);
- mlx_array_free(maxes);
- mlx_array_free(masked_sims);
- mlx_array_free(invalid_value);
- mlx_array_free(valid_3d);
- mlx_array_free(valid);
- mlx_array_free(positions_2d);
- mlx_array_free(lengths_2d);
- mlx_array_free(positions);
- mlx_array_free(lengths_arr);
- mlx_array_free(sims);
- mlx_array_free(docs_t);
- mlx_array_free(query_batched);
- mlx_array_free(query_3d);
- mlx_array_free(doc_batch);
- if (doc_vec.ctx)
- mlx_vector_array_free(doc_vec);
- mlx_array_free(pad_value);
- for (int i = 0; i < docs_count; i++)
- mlx_array_free(doc_rows[i]);
- free(doc_rows);
- free(lengths);
- return rc;
-}
-
-int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- embed_mlx_late_vectors_t *vecs =
- embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize);
- if (!vecs)
- return -1;
- int rc = embed_mlx_late_vectors_copy(vecs, out_vectors);
- embed_mlx_late_vectors_free(vecs);
- return rc;
-}
-
-static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
- ctx->layer_end != ctx->config.n_layers)
- return -1;
-
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
-
- int hidden = c->hidden_size;
- int max_seq = 0;
- int has_padding = 0;
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
- int cls_pool = c->pooling_mode == EMBED_POOL_CLS;
- /* CLS and last-token are both single-token pooling: gather one row per
- * sequence (the first for CLS, the last otherwise) rather than mean-pool. */
- int single_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN || cls_pool;
- if (hidden <= 0 || c->vocab_size <= 0)
- return -1;
-
- for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0)
- return -1;
- if (inputs[b].n_tokens > max_seq)
- max_seq = inputs[b].n_tokens;
- }
- if (max_seq <= 0)
- return -1;
-
- for (int b = 0; b < batch; b++) {
- if (inputs[b].n_tokens != max_seq)
- has_padding = 1;
- }
- int needs_pool_mask = has_padding && !single_token_pool;
- int needs_attn_mask = has_padding && !causal;
-
- size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes;
- size_t last_index_bytes = 0;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 ||
- mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 ||
- mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 ||
- mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 ||
- mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 ||
- mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 ||
- (single_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
- return -1;
- if (ids_bytes == 0 || out_bytes == 0 ||
- (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) ||
- (needs_attn_mask && mask_bytes == 0) || (single_token_pool && last_index_bytes == 0))
- return -1;
- int *padded_ids = (int *)malloc(ids_bytes);
- float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
- float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
- float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL;
- int *last_index_data = single_token_pool ? (int *)malloc(last_index_bytes) : NULL;
- if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) ||
- (needs_attn_mask && !attn_mask_data)) {
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- free(last_index_data);
- return -1;
- }
- if (single_token_pool && !last_index_data) {
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- return -1;
- }
-
- for (int b = 0; b < batch; b++) {
- if (needs_pool_mask)
- len_data[b] = (float)inputs[b].n_tokens;
- if (single_token_pool) {
- int index = cls_pool ? 0 : inputs[b].n_tokens - 1;
- for (int d = 0; d < hidden; d++)
- last_index_data[(size_t)b * hidden + d] = index;
- }
- for (int t = 0; t < max_seq; t++) {
- size_t idx = (size_t)b * max_seq + t;
- if (t < inputs[b].n_tokens) {
- int id = inputs[b].ids[t];
- if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t);
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- free(last_index_data);
- return -1;
- }
- padded_ids[idx] = id;
- if (needs_pool_mask)
- pool_mask_data[idx] = 1.0f;
- if (needs_attn_mask)
- attn_mask_data[idx] = 0.0f;
- } else {
- padded_ids[idx] = 0;
- if (needs_pool_mask)
- pool_mask_data[idx] = 0.0f;
- if (needs_attn_mask)
- attn_mask_data[idx] = -1.0e9f;
- }
- }
- }
-
- int ids_shape[] = {batch, max_seq};
- int pool_mask_shape[] = {batch, max_seq, 1};
- int attn_mask_shape[] = {batch, 1, 1, max_seq};
- int len_shape[] = {batch, 1};
- int last_index_shape[] = {batch, 1, hidden};
-
- mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- mlx_array pool_mask = needs_pool_mask
- ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array attn_mask = needs_attn_mask
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array lengths =
- needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0};
- mlx_array last_indices =
- single_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
- : (mlx_array){0};
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- free(last_index_data);
-
- if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
- (needs_attn_mask && !arr_ok(attn_mask)) || (single_token_pool && !arr_ok(last_indices))) {
- mlx_array_free(ids);
- mlx_array_free(pool_mask);
- mlx_array_free(attn_mask);
- mlx_array_free(lengths);
- mlx_array_free(last_indices);
- return -1;
- }
- if (needs_attn_mask && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) {
- mlx_array_free(ids);
- mlx_array_free(pool_mask);
- mlx_array_free(lengths);
- mlx_array_free(last_indices);
- return -1;
- }
- }
-
- /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */
- mlx_array x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
-
- /* 2. Family forward. BERT adds learned positions + token-type[0] and an
- * embedding LayerNorm, then runs post-norm encoder layers; its states are
- * already final, so pooling uses them directly. Qwen runs the decoder block
- * and applies the final RMSNorm here. */
- mlx_array x_normed;
- if (c->family == EMBED_FAMILY_BERT) {
- int start2[] = {0, 0}, stride2[] = {1, 1};
- int pos_stop[] = {max_seq, hidden}, type_stop[] = {1, hidden};
- mlx_array pos = mlx_array_new();
- mlx_slice(&pos, ctx->position_embeddings, start2, 2, pos_stop, 2, stride2, 2, S);
- mlx_array xp = mlx_array_new();
- mlx_add(&xp, x, pos, S);
- mlx_array_free(x);
- mlx_array_free(pos);
- mlx_array tt = mlx_array_new();
- mlx_slice(&tt, ctx->token_type_embeddings, start2, 2, type_stop, 2, stride2, 2, S);
- mlx_array xt = mlx_array_new();
- mlx_add(&xt, xp, tt, S);
- mlx_array_free(xp);
- mlx_array_free(tt);
- mlx_array xe = mlx_array_new();
- mlx_fast_layer_norm(&xe, xt, ctx->embed_ln_w, ctx->embed_ln_b, c->layer_norm_eps, S);
- mlx_array_free(xt);
- x_normed =
- mlx_forward_bert(ctx, xe, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
- } else {
- x = mlx_forward_layers(ctx, x, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- }
-
- mlx_array emb = mlx_array_new();
- if (single_token_pool) {
- mlx_array selected = mlx_array_new();
- mlx_take_along_axis(&selected, x_normed, last_indices, 1, S);
- mlx_array_free(x_normed);
- mlx_array_free(last_indices);
- int emb_shape[] = {batch, hidden};
- mlx_reshape(&emb, selected, emb_shape, 2, S);
- mlx_array_free(selected);
- mlx_array_free(pool_mask);
- mlx_array_free(attn_mask);
- mlx_array_free(lengths);
- } else if (has_padding) {
- /* 4. Masked mean pool over T. */
- mlx_array masked = mlx_array_new();
- mlx_multiply(&masked, x_normed, pool_mask, S);
- mlx_array_free(x_normed);
- mlx_array_free(pool_mask);
-
- mlx_array emb_sum = mlx_array_new();
- mlx_sum_axis(&emb_sum, masked, 1, false, S);
- mlx_array_free(masked);
-
- mlx_divide(&emb, emb_sum, lengths, S);
- mlx_array_free(emb_sum);
- mlx_array_free(lengths);
- mlx_array_free(attn_mask);
- } else {
- /* Equal-length batches do not need masks or a separate divide. */
- mlx_mean_axis(&emb, x_normed, 1, false, S);
- mlx_array_free(x_normed);
- }
-
- /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are
- * intentionally unnormalized; callers should use cosine similarity. */
- mlx_array emb_f32 = mlx_array_new();
- mlx_astype(&emb_f32, emb, MLX_FLOAT32, S);
- mlx_array_free(emb);
-
- mlx_array_eval(emb_f32);
- const float *data = mlx_array_data_float32(emb_f32);
- int rc = -1;
- if (data) {
- memcpy(out_embeddings, data, out_bytes);
- if (c->normalize_embeddings) {
- rc = 0;
- for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) {
- rc = -1;
- break;
- }
- }
- } else {
- rc = 0;
- }
- }
- mlx_array_free(emb_f32);
- return rc;
-}
-
-typedef struct {
- int index;
- const embed_input_t *input;
-} mlx_batch_item_t;
-
-static int mlx_batch_item_cmp(const void *a, const void *b) {
- const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a;
- const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b;
- int da = ia->input->n_tokens;
- int db = ib->input->n_tokens;
- return (da > db) - (da < db);
-}
-
-static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) {
- if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5)
- return 1;
- return padded * 5 > tokens * 11;
-}
-
-static int mlx_batch_should_split(const embed_input_t *inputs,
- int batch,
- int *out_total_tokens,
- int *out_max_seq) {
- int max_seq = 0;
- int total_tokens = 0;
- for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0)
- return 0;
- if (inputs[b].n_tokens > max_seq)
- max_seq = inputs[b].n_tokens;
- if (total_tokens > INT_MAX - inputs[b].n_tokens)
- return 0;
- total_tokens += inputs[b].n_tokens;
- }
- if (out_total_tokens)
- *out_total_tokens = total_tokens;
- if (out_max_seq)
- *out_max_seq = max_seq;
- if (batch < 4 || max_seq < 64 || total_tokens <= 0)
- return 0;
-
- size_t padded;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
- return 0;
-
- return mlx_padding_exceeds_limit(padded, (size_t)total_tokens);
-}
-
-static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
- int group_count,
- int group_max,
- int next_tokens) {
- if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0)
- return 1;
-
- int next_count = group_count + 1;
- int next_max = next_tokens > group_max ? next_tokens : group_max;
- size_t merged, split_left, split_total;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max, &merged) != 0 ||
- mlx_mul_size((size_t)group_count, (size_t)group_max, &split_left) != 0)
- return 0;
- /* Treat one extra MLX launch as roughly 256 padded token rows for 4B.
- * This preserves small mixed groups while splitting larger padded tails. */
- if (split_left > SIZE_MAX - (size_t)next_tokens ||
- split_left + (size_t)next_tokens > SIZE_MAX - 256)
- return 0;
- split_total = split_left + (size_t)next_tokens + 256;
- return merged <= split_total;
-}
-
-int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings)
- return -1;
-
- if (!mlx_batch_should_split(inputs, batch, NULL, NULL))
- return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings);
-
- const int hidden = ctx->config.hidden_size;
- mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items));
- embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
- size_t group_values, group_bytes;
- if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 ||
- mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items);
- free(group_inputs);
- return -1;
- }
- float *group_out = (float *)malloc(group_bytes);
- if (!items || !group_inputs || !group_out) {
- free(items);
- free(group_inputs);
- free(group_out);
- return -1;
- }
- for (int i = 0; i < batch; i++) {
- items[i].index = i;
- items[i].input = &inputs[i];
- }
- qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp);
-
- int rc = 0;
- for (int start = 0; start < batch && rc == 0;) {
- int group_max = 0;
- int group_tokens = 0;
- int end = start;
- while (end < batch) {
- int next_tokens = items[end].input->n_tokens;
- int next_count = end - start + 1;
- int next_max = next_tokens > group_max ? next_tokens : group_max;
- int next_total = group_tokens + next_tokens;
- if (end > start) {
- size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
- mlx_padding_exceeds_limit(next_padded, (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
- break;
- }
- group_max = next_max;
- group_tokens = next_total;
- end++;
- }
-
- int group_count = end - start;
- for (int i = 0; i < group_count; i++)
- group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out);
- if (rc != 0)
- break;
- for (int i = 0; i < group_count; i++) {
- int dst = items[start + i].index;
- memcpy(out_embeddings + (size_t)dst * hidden, group_out + (size_t)i * hidden,
- (size_t)hidden * sizeof(float));
- }
- start = end;
- }
-
- free(items);
- free(group_inputs);
- free(group_out);
- return rc;
-}
-
-int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- embed_input_t input = {token_ids, n_tokens};
- return embed_mlx_encode_batch(ctx, &input, 1, out_embedding);
-}
-
-typedef struct {
- int index;
- int span_offset;
- const embed_context_input_t *input;
-} mlx_context_batch_item_t;
-
-static int mlx_context_batch_item_cmp(const void *a, const void *b) {
- const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a;
- const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b;
- int da = ia->input->input.n_tokens;
- int db = ib->input->input.n_tokens;
- return (da > db) - (da < db);
-}
-
-static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) {
- if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4)
- return 1;
- return padded * 4 > tokens * 5;
-}
-
-static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
- int batch,
- int *out_total_spans) {
- int max_seq = 0;
- int total_tokens = 0;
- int total_spans = 0;
- for (int b = 0; b < batch; b++) {
- int n_tokens = inputs[b].input.n_tokens;
- if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || inputs[b].n_spans <= 0)
- return 0;
- if (n_tokens > max_seq)
- max_seq = n_tokens;
- if (total_tokens > INT_MAX - n_tokens || total_spans > INT_MAX - inputs[b].n_spans)
- return 0;
- total_tokens += n_tokens;
- total_spans += inputs[b].n_spans;
- }
- if (out_total_spans)
- *out_total_spans = total_spans;
- if (batch < 2 || max_seq < 64 || total_tokens <= 0)
- return 0;
-
- size_t padded;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
- return 0;
- return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens);
-}
-
-static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
- ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
- return -1;
-
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
- int hidden = c->hidden_size;
- int max_seq = 0;
- int total_spans = 0;
- int has_padding = 0;
- for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
- int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
- total_spans > INT_MAX - input->n_spans)
- return -1;
- total_spans += input->n_spans;
- if (n_tokens > max_seq)
- max_seq = n_tokens;
- for (int t = 0; t < n_tokens; t++) {
- if (input->input.ids[t] < 0 || input->input.ids[t] >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
- input->input.ids[t], b, t);
- return -1;
- }
- }
- for (int s = 0; s < input->n_spans; s++) {
- if (input->spans[s].start < 0 || input->spans[s].start > n_tokens ||
- input->spans[s].n_tokens <= 0 ||
- input->spans[s].n_tokens > n_tokens - input->spans[s].start)
- return -1;
- }
- }
- for (int b = 0; b < batch; b++)
- if (inputs[b].input.n_tokens != max_seq)
- has_padding = 1;
-
- size_t rows, ids_bytes, mask_bytes, index_bytes;
- size_t span_float_bytes, out_values, out_bytes;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 ||
- mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 ||
- mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 ||
- mlx_mul_size((size_t)total_spans, sizeof(int), &index_bytes) != 0 ||
- mlx_mul_size((size_t)total_spans, sizeof(float), &span_float_bytes) != 0 ||
- mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 ||
- mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
- return -1;
- if (rows > INT_MAX)
- return -1;
- int *padded_ids = (int *)malloc(ids_bytes);
- float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
- if (!padded_ids || (has_padding && !attn_mask_data)) {
- free(padded_ids);
- free(attn_mask_data);
- return -1;
- }
- for (int b = 0; b < batch; b++) {
- const embed_input_t *input = &inputs[b].input;
- for (int t = 0; t < max_seq; t++) {
- size_t row = (size_t)b * max_seq + t;
- if (t < input->n_tokens) {
- padded_ids[row] = input->ids[t];
- if (has_padding)
- attn_mask_data[row] = 0.0f;
- } else {
- padded_ids[row] = 0;
- if (has_padding)
- attn_mask_data[row] = -1.0e9f;
- }
- }
- }
-
- mlx_array ids = (mlx_array){0};
- mlx_array attn_mask = (mlx_array){0};
- mlx_array x = (mlx_array){0};
- mlx_array x_normed = (mlx_array){0};
- mlx_array prefix = (mlx_array){0};
- mlx_array flat_prefix = (mlx_array){0};
- mlx_array end_idx = (mlx_array){0};
- mlx_array start_idx = (mlx_array){0};
- mlx_array lengths = (mlx_array){0};
- mlx_array start_mask = (mlx_array){0};
- mlx_array end_values = (mlx_array){0};
- mlx_array start_values = (mlx_array){0};
- mlx_array masked_start = (mlx_array){0};
- mlx_array span_sums = (mlx_array){0};
- mlx_array embeddings = (mlx_array){0};
- mlx_array embeddings_f32 = (mlx_array){0};
- mlx_array *pooled = NULL;
- mlx_vector_array pooled_vec = (mlx_vector_array){0};
- int *end_idx_data = NULL;
- int *start_idx_data = NULL;
- float *lengths_data = NULL;
- float *start_mask_data = NULL;
- int rc = -1;
-
- int ids_shape[] = {batch, max_seq};
- int attn_mask_shape[] = {batch, 1, 1, max_seq};
- ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- attn_mask = has_padding ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- free(padded_ids);
- free(attn_mask_data);
- if (!arr_ok(ids))
- goto cleanup;
- if (has_padding && !arr_ok(attn_mask))
- goto cleanup;
- if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask))
- goto cleanup;
- }
-
- x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
- ids = (mlx_array){0};
- if (!arr_ok(x))
- goto cleanup;
-
- x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers);
- if (has_padding) {
- mlx_array_free(attn_mask);
- attn_mask = (mlx_array){0};
- }
- if (!arr_ok(x))
- goto cleanup;
-
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- x = (mlx_array){0};
- if (!arr_ok(x_normed))
- goto cleanup;
-
- if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled));
- if (!pooled)
- goto cleanup;
-
- int pooled_count = 0;
- for (int b = 0; b < batch; b++) {
- for (int s = 0; s < inputs[b].n_spans; s++) {
- const embed_span_t *span = &inputs[b].spans[s];
- int start[] = {b, span->start, 0};
- int stop[] = {b + 1, span->start + span->n_tokens, hidden};
- int strides[] = {1, 1, 1};
- mlx_array slice = mlx_array_new();
- mlx_slice(&slice, x_normed, start, 3, stop, 3, strides, 3, S);
- if (!arr_ok(slice)) {
- mlx_array_free(slice);
- goto cleanup;
- }
-
- pooled[pooled_count] = mlx_array_new();
- mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S);
- mlx_array_free(slice);
- if (!arr_ok(pooled[pooled_count]))
- goto cleanup;
- pooled_count++;
- }
- }
-
- pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans);
- if (!pooled_vec.ctx)
- goto cleanup;
- embeddings = mlx_array_new();
- mlx_concatenate_axis(&embeddings, pooled_vec, 0, S);
- if (!arr_ok(embeddings))
- goto cleanup;
- } else {
- end_idx_data = (int *)malloc(index_bytes);
- start_idx_data = (int *)malloc(index_bytes);
- lengths_data = (float *)malloc(span_float_bytes);
- start_mask_data = (float *)malloc(span_float_bytes);
- if (!end_idx_data || !start_idx_data || !lengths_data || !start_mask_data)
- goto cleanup;
-
- int span_count = 0;
- for (int b = 0; b < batch; b++) {
- for (int s = 0; s < inputs[b].n_spans; s++) {
- const embed_span_t *span = &inputs[b].spans[s];
- int base = b * max_seq;
- int start = span->start;
- int stop = span->start + span->n_tokens;
- end_idx_data[span_count] = base + stop - 1;
- start_idx_data[span_count] = start > 0 ? base + start - 1 : base;
- lengths_data[span_count] = (float)span->n_tokens;
- start_mask_data[span_count] = start > 0 ? 1.0f : 0.0f;
- span_count++;
- }
- }
-
- int index_shape[] = {total_spans};
- int scale_shape[] = {total_spans, 1};
- end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32);
- start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, MLX_INT32);
- lengths = mlx_array_new_data(lengths_data, scale_shape, 2, MLX_FLOAT32);
- start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, MLX_FLOAT32);
- free(end_idx_data);
- end_idx_data = NULL;
- free(start_idx_data);
- start_idx_data = NULL;
- free(lengths_data);
- lengths_data = NULL;
- free(start_mask_data);
- start_mask_data = NULL;
- if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || !arr_ok(start_mask))
- goto cleanup;
-
- prefix = mlx_array_new();
- mlx_cumsum(&prefix, x_normed, 1, false, true, S);
- mlx_array_free(x_normed);
- x_normed = (mlx_array){0};
- if (!arr_ok(prefix))
- goto cleanup;
-
- int flat_shape[] = {(int)rows, hidden};
- flat_prefix = mlx_array_new();
- mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S);
- mlx_array_free(prefix);
- prefix = (mlx_array){0};
- if (!arr_ok(flat_prefix))
- goto cleanup;
-
- end_values = mlx_array_new();
- mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S);
- if (!arr_ok(end_values))
- goto cleanup;
-
- start_values = mlx_array_new();
- mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S);
- if (!arr_ok(start_values))
- goto cleanup;
-
- masked_start = mlx_array_new();
- mlx_multiply(&masked_start, start_values, start_mask, S);
- if (!arr_ok(masked_start))
- goto cleanup;
-
- span_sums = mlx_array_new();
- mlx_subtract(&span_sums, end_values, masked_start, S);
- if (!arr_ok(span_sums))
- goto cleanup;
-
- embeddings = mlx_array_new();
- mlx_divide(&embeddings, span_sums, lengths, S);
- if (!arr_ok(embeddings))
- goto cleanup;
- }
-
- embeddings_f32 = mlx_array_new();
- mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S);
- if (!arr_ok(embeddings_f32))
- goto cleanup;
-
- mlx_array_eval(embeddings_f32);
- const float *data = mlx_array_data_float32(embeddings_f32);
- if (data) {
- memcpy(out_embeddings, data, out_bytes);
- rc = 0;
- }
-
-cleanup:
- free(start_mask_data);
- free(lengths_data);
- free(start_idx_data);
- free(end_idx_data);
- mlx_array_free(embeddings_f32);
- mlx_array_free(embeddings);
- if (pooled_vec.ctx)
- mlx_vector_array_free(pooled_vec);
- if (pooled) {
- for (int s = 0; s < total_spans; s++)
- mlx_array_free(pooled[s]);
- free(pooled);
- }
- mlx_array_free(span_sums);
- mlx_array_free(masked_start);
- mlx_array_free(start_values);
- mlx_array_free(end_values);
- mlx_array_free(start_mask);
- mlx_array_free(lengths);
- mlx_array_free(start_idx);
- mlx_array_free(end_idx);
- mlx_array_free(flat_prefix);
- mlx_array_free(prefix);
- mlx_array_free(x_normed);
- mlx_array_free(x);
- mlx_array_free(attn_mask);
- mlx_array_free(ids);
- return rc;
-}
-
-int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings)
- return -1;
-
- int total_spans = 0;
- if (!mlx_context_batch_should_split(inputs, batch, &total_spans))
- return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings);
-
- const int hidden = ctx->config.hidden_size;
- mlx_context_batch_item_t *items =
- (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items));
- embed_context_input_t *group_inputs =
- (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
- size_t group_values, group_bytes;
- if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 ||
- mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items);
- free(group_inputs);
- return -1;
- }
- float *group_out = (float *)malloc(group_bytes);
- if (!items || !group_inputs || !group_out) {
- free(items);
- free(group_inputs);
- free(group_out);
- return -1;
- }
-
- int span_offset = 0;
- for (int i = 0; i < batch; i++) {
- items[i].index = i;
- items[i].span_offset = span_offset;
- items[i].input = &inputs[i];
- span_offset += inputs[i].n_spans;
- }
- qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp);
-
- int rc = 0;
- for (int start = 0; start < batch && rc == 0;) {
- int group_max = 0;
- int group_tokens = 0;
- int end = start;
- while (end < batch) {
- int next_tokens = items[end].input->input.n_tokens;
- int next_count = end - start + 1;
- int next_max = next_tokens > group_max ? next_tokens : group_max;
- int next_total = group_tokens + next_tokens;
- if (end > start) {
- size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
- mlx_context_padding_exceeds_limit(next_padded, (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
- break;
- }
- group_max = next_max;
- group_tokens = next_total;
- end++;
- }
-
- int group_count = end - start;
- for (int i = 0; i < group_count; i++)
- group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out);
- if (rc != 0)
- break;
- int src_span = 0;
- for (int i = 0; i < group_count; i++) {
- int n_spans = group_inputs[i].n_spans;
- int dst_span = items[start + i].span_offset;
- memcpy(out_embeddings + (size_t)dst_span * hidden,
- group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float));
- src_span += n_spans;
- }
- start = end;
- }
-
- free(items);
- free(group_inputs);
- free(group_out);
- return rc;
-}
-
-int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings) {
- embed_context_input_t input = {
- .input = {token_ids, n_tokens},
- .spans = spans,
- .n_spans = n_spans,
- };
- return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings);
-}
-
-float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) {
- if (!ctx || !token_ids || n_tokens <= 0)
- return NULL;
-
- int hidden = ctx->config.hidden_size;
- size_t out_bytes;
- if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0)
- return NULL;
- float *out = (float *)malloc(out_bytes);
- if (!out)
- return NULL;
-
- if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) {
- free(out);
- return NULL;
- }
- return out;
-}