aboutsummaryrefslogtreecommitdiff
path: root/embed_mlx.c
diff options
context:
space:
mode:
Diffstat (limited to 'embed_mlx.c')
-rw-r--r--embed_mlx.c1153
1 files changed, 610 insertions, 543 deletions
diff --git a/embed_mlx.c b/embed_mlx.c
index cb3b737..9f20026 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -35,12 +35,12 @@ typedef struct {
struct embed_mlx_ctx {
embed_config_t config;
mlx_array embed_tokens;
- mlx_layer_t *layers; /* heap-allocated [n_layers] */
+ mlx_layer_t *layers; /* heap-allocated [n_layers] */
mlx_array norm;
mlx_stream stream;
multi_safetensors_t *ms;
- int layer_start; /* loaded transformer range, inclusive */
- int layer_end; /* loaded transformer range, exclusive */
+ int layer_start; /* loaded transformer range, inclusive */
+ int layer_end; /* loaded transformer range, exclusive */
int quantize_bits;
int quantize_group_size;
mlx_dtype weight_dtype;
@@ -55,7 +55,7 @@ struct embed_mlx_late_ctx {
struct embed_mlx_late_vectors {
embed_mlx_late_ctx_t *owner;
- mlx_array vectors; /* [tokens, dim] */
+ mlx_array vectors; /* [tokens, dim] */
int tokens;
int dim;
};
@@ -66,47 +66,46 @@ struct embed_mlx_late_vectors {
static int arr_ok(mlx_array a) { return a.ctx != NULL; }
-static int mlx_mul_size(size_t a, size_t b, size_t *out)
-{
- if (a != 0 && b > SIZE_MAX / a) return -1;
+static int mlx_mul_size(size_t a, size_t b, size_t *out) {
+ if (a != 0 && b > SIZE_MAX / a)
+ return -1;
*out = a * b;
return 0;
}
-static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx)
-{
- return ctx->weight_dtype;
-}
+static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; }
-static size_t safetensor_dtype_size(safetensor_dtype_t dtype)
-{
+static size_t safetensor_dtype_size(safetensor_dtype_t dtype) {
switch (dtype) {
- case DTYPE_F32: return sizeof(float);
- case DTYPE_BF16: return sizeof(uint16_t);
- default: return 0;
+ case DTYPE_F32:
+ return sizeof(float);
+ case DTYPE_BF16:
+ return sizeof(uint16_t);
+ default:
+ return 0;
}
}
static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
const safetensor_t *t,
- const char *name, const int *shape,
- int ndim)
-{
+ const char *name,
+ const int *shape,
+ int ndim) {
size_t elem_size = safetensor_dtype_size(t->dtype);
if (elem_size == 0) {
fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
return 0;
}
if (t->ndim != ndim) {
- fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n",
- name, t->ndim, ndim);
+ fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
return 0;
}
size_t numel = 1;
for (int i = 0; i < ndim; i++) {
if (shape[i] <= 0 || t->shape[i] != shape[i]) {
- fprintf(stderr, "mlx: bad shape for %s at dim %d: "
+ fprintf(stderr,
+ "mlx: bad shape for %s at dim %d: "
"got %lld, expected %d\n",
name, i, (long long)t->shape[i], shape[i]);
return 0;
@@ -123,15 +122,15 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
return 0;
}
if (t->data_size != bytes) {
- fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n",
- name, t->data_size, bytes);
+ fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
+ bytes);
return 0;
}
- if (sf->header_size > SIZE_MAX - 8) return 0;
+ if (sf->header_size > SIZE_MAX - 8)
+ return 0;
size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size ||
- t->data_offset > sf->file_size - data_start ||
+ if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
t->data_size > sf->file_size - data_start - t->data_offset) {
fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name);
return 0;
@@ -139,11 +138,10 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
- const int *shape, int ndim)
-{
+static mlx_array
+load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
fprintf(stderr, "mlx: tensor not found: %s\n", name);
return (mlx_array){0};
@@ -162,22 +160,16 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype);
}
-static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms,
- const char *name,
- const int *shape, int ndim)
-{
+static mlx_linear_t
+load_linear_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
mlx_linear_t l = {0};
l.w = load_tensor(ms, name, shape, ndim);
return l;
}
-static int linear_ok(mlx_linear_t l)
-{
- return arr_ok(l.w) || arr_ok(l.wt);
-}
+static int linear_ok(mlx_linear_t l) { return arr_ok(l.w) || arr_ok(l.wt); }
-static void free_linear(mlx_linear_t *l)
-{
+static void free_linear(mlx_linear_t *l) {
mlx_array_free(l->w);
mlx_array_free(l->wt);
mlx_array_free(l->scales);
@@ -185,8 +177,7 @@ static void free_linear(mlx_linear_t *l)
memset(l, 0, sizeof(*l));
}
-static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s)
-{
+static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) {
if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt))
return 0;
@@ -214,15 +205,15 @@ static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s)
return 0;
}
-static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
- mlx_stream s)
-{
- if (!l || !arr_ok(l->w) || bits == 0) return 0;
+static int quantize_linear(mlx_linear_t *l, int bits, int group_size, mlx_stream s) {
+ if (!l || !arr_ok(l->w) || bits == 0)
+ return 0;
if (bits != 8) {
fprintf(stderr, "mlx: quantized weights support only 8 bits\n");
return -1;
}
- if (group_size <= 0) group_size = 64;
+ if (group_size <= 0)
+ group_size = 64;
mlx_optional_int gs = {.value = group_size, .has_value = true};
mlx_optional_int qb = {.value = bits, .has_value = true};
@@ -253,7 +244,8 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
}
mlx_array_eval(qw);
mlx_array_eval(scales);
- if (arr_ok(biases)) mlx_array_eval(biases);
+ if (arr_ok(biases))
+ mlx_array_eval(biases);
mlx_synchronize(s);
mlx_array_free(l->w);
@@ -269,16 +261,15 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size,
}
/* y = x @ W^T */
-static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
-{
+static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) {
if (W->quantized) {
- if (!arr_ok(W->w)) return (mlx_array){0};
+ if (!arr_ok(W->w))
+ return (mlx_array){0};
mlx_array res = mlx_array_new();
mlx_optional_int gs = {.value = W->group_size, .has_value = true};
mlx_optional_int bits = {.value = W->bits, .has_value = true};
mlx_array null_arr = (mlx_array){0};
- mlx_quantized_matmul(&res, x, W->w, W->scales,
- arr_ok(W->biases) ? W->biases : null_arr,
+ mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr,
true, gs, bits, "affine", s);
return res;
}
@@ -294,8 +285,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
return (mlx_array){0};
}
mlx_array res = mlx_array_new();
- if (mlx_array_dtype(Wt) == MLX_BFLOAT16 &&
- mlx_array_dtype(x) == MLX_FLOAT32) {
+ if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && mlx_array_dtype(x) == MLX_FLOAT32) {
/* F32 activations with BF16 weights: upcast the weight operand and run
* the matmul in F32 (so accumulation is F32). Only the weights are
* BF16-rounded; the residual stream stays F32 (see mlx_forward_layers),
@@ -307,21 +297,21 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s)
} else {
mlx_matmul(&res, x, Wt, s);
}
- if (free_wt) mlx_array_free(Wt);
+ if (free_wt)
+ mlx_array_free(Wt);
return res;
}
-const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx)
-{
+const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) {
return ctx ? &ctx->config : NULL;
}
/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the
* tiny RMSNorm vectors so non-quantized BF16 models can run norms and the
* residual stream in F32 while keeping the large matmul weights in BF16. */
-static void upcast_f32_inplace(mlx_array *a, mlx_stream s)
-{
- if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return;
+static void upcast_f32_inplace(mlx_array *a, mlx_stream s) {
+ if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32)
+ return;
mlx_array f = mlx_array_new();
mlx_astype(&f, *a, MLX_FLOAT32, s);
mlx_array_free(*a);
@@ -335,14 +325,13 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s)
/* We duplicate a minimal config parser here so MLX can load standalone.
* (Same logic as embed.c:parse_config, kept inline for simplicity.) */
-static const char *skip_ws_m(const char *p)
-{
- while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+static const char *skip_ws_m(const char *p) {
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')
+ p++;
return p;
}
-static const char *json_find_m(const char *json, const char *key)
-{
+static const char *json_find_m(const char *json, const char *key) {
size_t klen = strlen(key);
const char *p = json;
while ((p = strstr(p, key)) != NULL) {
@@ -350,7 +339,8 @@ static const char *json_find_m(const char *json, const char *key)
const char *a = p + klen;
if (*a == '"') {
a = skip_ws_m(a + 1);
- if (*a == ':') return skip_ws_m(a + 1);
+ if (*a == ':')
+ return skip_ws_m(a + 1);
}
}
p += klen;
@@ -358,18 +348,16 @@ static const char *json_find_m(const char *json, const char *key)
return NULL;
}
-static int json_string_equals_m(const char *json, const char *key,
- const char *expected)
-{
+static int json_string_equals_m(const char *json, const char *key, const char *expected) {
const char *v = json_find_m(json, key);
- if (!v || *v != '"') return 0;
+ if (!v || *v != '"')
+ return 0;
v++;
size_t len = strlen(expected);
return strncmp(v, expected, len) == 0 && v[len] == '"';
}
-static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
-{
+static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) {
char path[1024];
int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir);
if (path_len < 0 || (size_t)path_len >= sizeof(path)) {
@@ -377,8 +365,14 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
return -1;
}
FILE *f = fopen(path, "rb");
- if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; }
- if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; }
+ if (!f) {
+ fprintf(stderr, "mlx: cannot open %s\n", path);
+ return -1;
+ }
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
long sz = ftell(f);
if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
fclose(f);
@@ -386,22 +380,32 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
}
char *buf = (char *)malloc((size_t)sz + 1);
if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f); free(buf); return -1;
+ fclose(f);
+ free(buf);
+ return -1;
}
fclose(f);
buf[sz] = '\0';
-#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0)
-#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0)
- GI("hidden_size", hidden_size, 0);
- GI("num_hidden_layers", n_layers, 0);
- GI("num_attention_heads", n_heads, 0);
- GI("num_key_value_heads", n_kv_heads, 0);
- GI("head_dim", head_dim, EMBED_HEAD_DIM);
- GI("intermediate_size", intermediate_size, 0);
- GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE);
- GF("rms_norm_eps", rms_norm_eps, 1e-6f);
- GF("rope_theta", rope_theta, 1000000.0f);
+#define GI(k, fb, def) \
+ do { \
+ const char *v = json_find_m(buf, k); \
+ cfg->fb = v ? atoi(v) : def; \
+ } while (0)
+#define GF(k, fb, def) \
+ do { \
+ const char *v = json_find_m(buf, k); \
+ cfg->fb = v ? (float)atof(v) : def; \
+ } while (0)
+ GI("hidden_size", hidden_size, 0);
+ GI("num_hidden_layers", n_layers, 0);
+ GI("num_attention_heads", n_heads, 0);
+ GI("num_key_value_heads", n_kv_heads, 0);
+ GI("head_dim", head_dim, EMBED_HEAD_DIM);
+ GI("intermediate_size", intermediate_size, 0);
+ GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE);
+ GF("rms_norm_eps", rms_norm_eps, 1e-6f);
+ GF("rope_theta", rope_theta, 1000000.0f);
#undef GI
#undef GF
cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
@@ -417,58 +421,53 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir)
cfg->q_dim = 0;
cfg->kv_dim = 0;
- if (cfg->n_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim)
cfg->q_dim = cfg->n_heads * cfg->head_dim;
- if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
free(buf);
- if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 ||
- cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
- cfg->head_dim <= 0 || cfg->intermediate_size <= 0 ||
- cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
- (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
- !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
- fprintf(stderr, "mlx: invalid config in %s "
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
+ cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 ||
+ cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) ||
+ cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) ||
+ cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
+ fprintf(stderr,
+ "mlx: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
- path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
- cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size);
+ path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim,
+ cfg->intermediate_size);
return -1;
}
if (cfg->n_layers > EMBED_MAX_LAYERS) {
- fprintf(stderr, "mlx: too many layers (%d > %d)\n",
- cfg->n_layers, EMBED_MAX_LAYERS);
+ fprintf(stderr, "mlx: too many layers (%d > %d)\n", cfg->n_layers, EMBED_MAX_LAYERS);
return -1;
}
return 0;
}
-static int mlx_model_dir_has_file(const char *model_dir, const char *rel)
-{
+static int mlx_model_dir_has_file(const char *model_dir, const char *rel) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
- if (n < 0 || (size_t)n >= sizeof(path)) return 0;
+ if (n < 0 || (size_t)n >= sizeof(path))
+ return 0;
FILE *f = fopen(path, "rb");
- if (!f) return 0;
+ if (!f)
+ return 0;
fclose(f);
return 1;
}
-static int mlx_model_dir_has_late_projection(const char *model_dir)
-{
+static int mlx_model_dir_has_late_projection(const char *model_dir) {
return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") &&
mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors");
}
-static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts)
-{
+static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) {
embed_mlx_options_t out = {0};
- if (opts) out = *opts;
- if (out.quantize_bits != 0 &&
- out.quantize_bits != 8) {
+ if (opts)
+ out = *opts;
+ if (out.quantize_bits != 0 && out.quantize_bits != 8) {
fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n");
out.quantize_bits = -1;
}
@@ -477,10 +476,9 @@ static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts
return out;
}
-static int quantize_layer(mlx_layer_t *l, int bits, int group_size,
- mlx_stream s)
-{
- if (bits == 0) return 0;
+static int quantize_layer(mlx_layer_t *l, int bits, int group_size, mlx_stream s) {
+ if (bits == 0)
+ return 0;
return quantize_linear(&l->wq, bits, group_size, s) ||
quantize_linear(&l->wk, bits, group_size, s) ||
quantize_linear(&l->wv, bits, group_size, s) ||
@@ -490,45 +488,45 @@ static int quantize_layer(mlx_layer_t *l, int bits, int group_size,
quantize_linear(&l->down_proj, bits, group_size, s);
}
-static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s)
-{
- return prepare_linear_transpose(&l->wq, s) ||
- prepare_linear_transpose(&l->wk, s) ||
- prepare_linear_transpose(&l->wv, s) ||
- prepare_linear_transpose(&l->wo, s) ||
- prepare_linear_transpose(&l->gate_proj, s) ||
- prepare_linear_transpose(&l->up_proj, s) ||
+static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) {
+ return prepare_linear_transpose(&l->wq, s) || prepare_linear_transpose(&l->wk, s) ||
+ prepare_linear_transpose(&l->wv, s) || prepare_linear_transpose(&l->wo, s) ||
+ prepare_linear_transpose(&l->gate_proj, s) || prepare_linear_transpose(&l->up_proj, s) ||
prepare_linear_transpose(&l->down_proj, s);
}
static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
- int layer_start, int layer_end,
- const embed_mlx_options_t *opts,
- int allow_late)
-{
+ int layer_start,
+ int layer_end,
+ const embed_mlx_options_t *opts,
+ int allow_late) {
embed_mlx_options_t options = normalize_mlx_options(opts);
- if (options.quantize_bits < 0) return NULL;
+ if (options.quantize_bits < 0)
+ return NULL;
if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "mlx: late-interaction models require token-level "
- "MaxSim support and are not valid pooled embedding models "
- "yet\n");
+ "MaxSim support and are not valid pooled embedding models "
+ "yet\n");
return NULL;
}
embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t));
- if (!ctx) return NULL;
+ if (!ctx)
+ return NULL;
ctx->quantize_bits = options.quantize_bits;
ctx->quantize_group_size = options.quantize_group_size;
- if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; }
+ if (mlx_parse_config(&ctx->config, model_dir) != 0) {
+ free(ctx);
+ return NULL;
+ }
const embed_config_t *c = &ctx->config;
if (layer_end == -1)
layer_end = c->n_layers;
- if (layer_start < 0 || layer_start >= layer_end ||
- layer_end > c->n_layers) {
- fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n",
- layer_start, layer_end, c->n_layers);
+ if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) {
+ fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", layer_start,
+ layer_end, c->n_layers);
free(ctx);
return NULL;
}
@@ -538,7 +536,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
ctx->ms = multi_safetensors_open(model_dir);
if (!ctx->ms) {
fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
- free(ctx); return NULL;
+ free(ctx);
+ return NULL;
}
int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
@@ -547,64 +546,73 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
/* Only the first stage needs token embeddings. */
if (layer_start == 0) {
int emb_shape[] = {c->vocab_size, h};
- ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight",
- emb_shape, 2);
- if (!arr_ok(ctx->embed_tokens)) goto fail;
+ ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2);
+ if (!arr_ok(ctx->embed_tokens))
+ goto fail;
}
/* Layers */
ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
- if (!ctx->layers) goto fail;
+ if (!ctx->layers)
+ goto fail;
char name[256];
for (int i = layer_start; i < layer_end; i++) {
mlx_layer_t *l = &ctx->layers[i];
-#define LD(fld, fmt, ...) do { \
- snprintf(name, sizeof(name), fmt, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
- if (!linear_ok(l->fld)) goto fail; \
-} while (0)
+#define LD(fld, fmt, ...) \
+ do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
+ if (!linear_ok(l->fld)) \
+ goto fail; \
+ } while (0)
- LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h);
- LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h);
- LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h);
- LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd);
+ LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h);
+ LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h);
+ LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h);
+ LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd);
#undef LD
-#define LD(fld, fmt, ...) do { \
- snprintf(name, sizeof(name), fmt, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
- if (!arr_ok(l->fld)) goto fail; \
-} while (0)
+#define LD(fld, fmt, ...) \
+ do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
+ if (!arr_ok(l->fld)) \
+ goto fail; \
+ } while (0)
- LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd);
- LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd);
- LD(input_norm, "layers.%d.input_layernorm.weight", h);
+ LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd);
+ LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd);
+ LD(input_norm, "layers.%d.input_layernorm.weight", h);
LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h);
#undef LD
-#define LD(fld, fmt, ...) do { \
- snprintf(name, sizeof(name), fmt, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
- if (!linear_ok(l->fld)) goto fail; \
-} while (0)
+#define LD(fld, fmt, ...) \
+ do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
+ if (!linear_ok(l->fld)) \
+ goto fail; \
+ } while (0)
- LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h);
- LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h);
- LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter);
+ LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h);
+ LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h);
+ LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter);
#undef LD
- if (embed_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i);
+ if (embed_verbose >= 2)
+ fprintf(stderr, " mlx layer %d loaded\n", i);
}
if (layer_end == c->n_layers) {
int ns[] = {h};
ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1);
- if (!arr_ok(ctx->norm)) goto fail;
+ if (!arr_ok(ctx->norm))
+ goto fail;
}
ctx->stream = mlx_default_gpu_stream_new();
@@ -624,13 +632,14 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
}
if (ctx->quantize_bits) {
for (int i = layer_start; i < layer_end; i++) {
- if (quantize_layer(&ctx->layers[i], ctx->quantize_bits,
- ctx->quantize_group_size, ctx->stream) != 0)
+ if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, ctx->quantize_group_size,
+ ctx->stream) != 0)
goto fail;
}
mlx_clear_cache();
if (embed_verbose >= 1)
- fprintf(stderr, "mlx: quantized linear weights to %d-bit "
+ fprintf(stderr,
+ "mlx: quantized linear weights to %d-bit "
"(group_size=%d)\n",
ctx->quantize_bits, ctx->quantize_group_size);
} else {
@@ -645,7 +654,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
mlx_get_active_memory(&active);
mlx_get_cache_memory(&cache);
mlx_get_peak_memory(&peak);
- fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB "
+ fprintf(stderr,
+ "mlx: memory active=%.2f GiB cache=%.2f GiB "
"peak=%.2f GiB\n",
(double)active / (1024.0 * 1024.0 * 1024.0),
(double)cache / (1024.0 * 1024.0 * 1024.0),
@@ -653,8 +663,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
}
if (embed_verbose >= 1)
- fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n",
- layer_start, layer_end, h);
+ fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h);
return ctx;
@@ -663,14 +672,12 @@ fail:
return NULL;
}
-embed_mlx_ctx_t *embed_mlx_load(const char *model_dir)
-{
+embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) {
return mlx_load_range_ex(model_dir, 0, -1, NULL, 0);
}
embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts)
-{
+ const embed_mlx_options_t *opts) {
return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
}
@@ -678,19 +685,23 @@ embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
* Free
* ======================================================================== */
-static void free_mlx_layer(mlx_layer_t *l)
-{
- free_linear(&l->wq); free_linear(&l->wk);
- free_linear(&l->wv); free_linear(&l->wo);
- mlx_array_free(l->q_norm); mlx_array_free(l->k_norm);
- mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm);
- free_linear(&l->gate_proj); free_linear(&l->up_proj);
+static void free_mlx_layer(mlx_layer_t *l) {
+ free_linear(&l->wq);
+ free_linear(&l->wk);
+ free_linear(&l->wv);
+ free_linear(&l->wo);
+ mlx_array_free(l->q_norm);
+ mlx_array_free(l->k_norm);
+ mlx_array_free(l->input_norm);
+ mlx_array_free(l->post_attn_norm);
+ free_linear(&l->gate_proj);
+ free_linear(&l->up_proj);
free_linear(&l->down_proj);
}
-void embed_mlx_free(embed_mlx_ctx_t *ctx)
-{
- if (!ctx) return;
+void embed_mlx_free(embed_mlx_ctx_t *ctx) {
+ if (!ctx)
+ return;
mlx_array_free(ctx->embed_tokens);
if (ctx->layers) {
for (int i = 0; i < ctx->config.n_layers; i++)
@@ -698,24 +709,27 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx)
free(ctx->layers);
}
mlx_array_free(ctx->norm);
- if (ctx->stream.ctx) mlx_stream_free(ctx->stream);
- if (ctx->ms) multi_safetensors_close(ctx->ms);
+ if (ctx->stream.ctx)
+ mlx_stream_free(ctx->stream);
+ if (ctx->ms)
+ multi_safetensors_close(ctx->ms);
free(ctx);
}
-embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(
- const char *model_dir, const embed_mlx_options_t *opts)
-{
+embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
+ const embed_mlx_options_t *opts) {
if (!mlx_model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "mlx late: missing 1_Dense projection\n");
return NULL;
}
embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late));
- if (!late) return NULL;
+ if (!late)
+ return NULL;
late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1);
- if (!late->base) goto fail;
+ if (!late->base)
+ goto fail;
char dense_dir[1024];
int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir);
@@ -726,13 +740,11 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(
late->dense_ms = multi_safetensors_open(dense_dir);
if (!late->dense_ms) {
- fprintf(stderr, "mlx late: failed to open safetensors in %s\n",
- dense_dir);
+ fprintf(stderr, "mlx late: failed to open safetensors in %s\n", dense_dir);
goto fail;
}
- const safetensor_t *t = multi_safetensors_find(late->dense_ms,
- "linear.weight", NULL);
+ const safetensor_t *t = multi_safetensors_find(late->dense_ms, "linear.weight", NULL);
if (!t) {
fprintf(stderr, "mlx late: missing linear.weight\n");
goto fail;
@@ -743,17 +755,16 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(
goto fail;
}
- int shape[] = { (int)t->shape[0], late->base->config.hidden_size };
- late->projection = load_linear_tensor(late->dense_ms, "linear.weight",
- shape, 2);
- if (!linear_ok(late->projection)) goto fail;
+ int shape[] = {(int)t->shape[0], late->base->config.hidden_size};
+ late->projection = load_linear_tensor(late->dense_ms, "linear.weight", shape, 2);
+ if (!linear_ok(late->projection))
+ goto fail;
if (prepare_linear_transpose(&late->projection, late->base->stream) != 0)
goto fail;
late->token_dim = shape[0];
if (embed_verbose >= 1)
- fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n",
- late->token_dim);
+ fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim);
return late;
fail:
@@ -761,39 +772,38 @@ fail:
return NULL;
}
-embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir)
-{
+embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) {
return embed_mlx_late_load_with_options(model_dir, NULL);
}
-void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx)
-{
- if (!ctx) return;
+void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) {
+ if (!ctx)
+ return;
free_linear(&ctx->projection);
- if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms);
+ if (ctx->dense_ms)
+ multi_safetensors_close(ctx->dense_ms);
embed_mlx_free(ctx->base);
free(ctx);
}
-const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx)
-{
+const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) {
return ctx && ctx->base ? &ctx->base->config : NULL;
}
-int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx)
-{
- return ctx ? ctx->token_dim : 0;
-}
+int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
/* ========================================================================
* Forward pass
* ======================================================================== */
-static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
- int batch, int max_seq, int has_padding,
+static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
+ mlx_array x,
+ int batch,
+ int max_seq,
+ int has_padding,
mlx_array attn_mask,
- int layer_start, int layer_end)
-{
+ int layer_start,
+ int layer_end) {
const embed_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
int n_heads = c->n_heads;
@@ -802,8 +812,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
int q_dim = c->q_dim;
float scale = 1.0f / sqrtf((float)head_dim);
mlx_array null_arr = (mlx_array){0};
- const char *mask_mode =
- c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
+ const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
/* For non-quantized BF16 weights, run the residual stream, norms, and
* attention in F32; matmuls downcast their operands to BF16 internally
@@ -812,8 +821,8 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
* models are unaffected. */
mlx_array local_mask = attn_mask;
int free_mask = 0;
- int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 &&
- !ctx->layers[layer_start].wq.quantized;
+ int mixed_bf16 =
+ mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized;
if (mixed_bf16) {
if (mlx_array_dtype(x) != MLX_FLOAT32) {
mlx_array xf = mlx_array_new();
@@ -821,8 +830,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_array_free(x);
x = xf;
}
- if (has_padding && arr_ok(attn_mask) &&
- mlx_array_dtype(attn_mask) != MLX_FLOAT32) {
+ if (has_padding && arr_ok(attn_mask) && mlx_array_dtype(attn_mask) != MLX_FLOAT32) {
local_mask = mlx_array_new();
mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S);
free_mask = 1;
@@ -840,7 +848,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_array v_flat = linear(xn, &l->wv, S);
mlx_array_free(xn);
- int q_shape[] = {batch, max_seq, n_heads, head_dim};
+ int q_shape[] = {batch, max_seq, n_heads, head_dim};
int k_shape[] = {batch, max_seq, n_kv_heads, head_dim};
mlx_array q = mlx_array_new();
mlx_array k = mlx_array_new();
@@ -848,33 +856,40 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_reshape(&q, q_flat, q_shape, 4, S);
mlx_reshape(&k, k_flat, k_shape, 4, S);
mlx_reshape(&v, v_flat, k_shape, 4, S);
- mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat);
+ mlx_array_free(q_flat);
+ mlx_array_free(k_flat);
+ mlx_array_free(v_flat);
mlx_array qn = mlx_array_new();
mlx_array kn = mlx_array_new();
mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
- mlx_array_free(q); mlx_array_free(k);
+ mlx_array_free(q);
+ mlx_array_free(k);
/* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */
int perm[] = {0, 2, 1, 3};
mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
mlx_transpose_axes(&qt, qn, perm, 4, S);
mlx_transpose_axes(&kt, kn, perm, 4, S);
- mlx_transpose_axes(&vt, v, perm, 4, S);
- mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v);
+ mlx_transpose_axes(&vt, v, perm, 4, S);
+ mlx_array_free(qn);
+ mlx_array_free(kn);
+ mlx_array_free(v);
mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
mlx_array qr = mlx_array_new(), kr = mlx_array_new();
mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_array_free(qt); mlx_array_free(kt);
+ mlx_array_free(qt);
+ mlx_array_free(kt);
mlx_array attn = mlx_array_new();
- mlx_fast_scaled_dot_product_attention(
- &attn, qr, kr, vt, scale, mask_mode,
- has_padding ? local_mask : null_arr, null_arr, S);
- mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
+ mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, mask_mode,
+ has_padding ? local_mask : null_arr, null_arr, S);
+ mlx_array_free(qr);
+ mlx_array_free(kr);
+ mlx_array_free(vt);
mlx_array attn_t = mlx_array_new();
mlx_transpose_axes(&attn_t, attn, perm, 4, S);
@@ -889,44 +904,49 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x,
mlx_array_free(attn_flat);
mlx_array x2 = mlx_array_new();
mlx_add(&x2, x, proj, S);
- mlx_array_free(x); mlx_array_free(proj);
+ mlx_array_free(x);
+ mlx_array_free(proj);
x = x2;
mlx_array xn2 = mlx_array_new();
mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
mlx_array gate = linear(xn2, &l->gate_proj, S);
- mlx_array up = linear(xn2, &l->up_proj, S);
+ mlx_array up = linear(xn2, &l->up_proj, S);
mlx_array_free(xn2);
mlx_array gate_sig = mlx_array_new();
mlx_sigmoid(&gate_sig, gate, S);
mlx_array silu_gate = mlx_array_new();
mlx_multiply(&silu_gate, gate, gate_sig, S);
- mlx_array_free(gate); mlx_array_free(gate_sig);
+ mlx_array_free(gate);
+ mlx_array_free(gate_sig);
mlx_array mid = mlx_array_new();
mlx_multiply(&mid, silu_gate, up, S);
- mlx_array_free(silu_gate); mlx_array_free(up);
+ mlx_array_free(silu_gate);
+ mlx_array_free(up);
mlx_array ffn = linear(mid, &l->down_proj, S);
mlx_array_free(mid);
mlx_array x3 = mlx_array_new();
mlx_add(&x3, x, ffn, S);
- mlx_array_free(x); mlx_array_free(ffn);
+ mlx_array_free(x);
+ mlx_array_free(ffn);
x = x3;
}
- if (free_mask) mlx_array_free(local_mask);
+ if (free_mask)
+ mlx_array_free(local_mask);
return x;
}
-embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
- embed_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens,
- int normalize)
-{
- if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 ||
- ctx->token_dim <= 0 || !linear_ok(ctx->projection))
+embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize) {
+ if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 ||
+ !linear_ok(ctx->projection))
return NULL;
embed_mlx_ctx_t *base = ctx->base;
@@ -938,8 +958,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
mlx_stream S = base->stream;
for (int i = 0; i < n_tokens; i++) {
if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
- fprintf(stderr, "mlx late: invalid token id %d at position %d\n",
- token_ids[i], i);
+ fprintf(stderr, "mlx late: invalid token id %d at position %d\n", token_ids[i], i);
return NULL;
}
}
@@ -949,13 +968,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
return NULL;
int *ids_data = (int *)malloc(id_bytes);
- if (!ids_data) return NULL;
+ if (!ids_data)
+ return NULL;
memcpy(ids_data, token_ids, id_bytes);
int ids_shape[] = {1, n_tokens};
mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32);
free(ids_data);
- if (!arr_ok(ids)) return NULL;
+ if (!arr_ok(ids))
+ return NULL;
embed_mlx_late_vectors_t *result = NULL;
mlx_array x = mlx_array_new();
@@ -965,22 +986,25 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
mlx_array_free(ids);
- if (!arr_ok(x)) goto cleanup;
+ if (!arr_ok(x))
+ goto cleanup;
- x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0},
- 0, c->n_layers);
- if (!arr_ok(x)) goto cleanup;
+ x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, 0, c->n_layers);
+ if (!arr_ok(x))
+ goto cleanup;
x_normed = mlx_array_new();
mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S);
mlx_array_free(x);
x = (mlx_array){0};
- if (!arr_ok(x_normed)) goto cleanup;
+ if (!arr_ok(x_normed))
+ goto cleanup;
projected = linear(x_normed, &ctx->projection, S);
mlx_array_free(x_normed);
x_normed = (mlx_array){0};
- if (!arr_ok(projected)) goto cleanup;
+ if (!arr_ok(projected))
+ goto cleanup;
if (normalize) {
mlx_array sq = mlx_array_new();
@@ -1011,16 +1035,19 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(
mlx_array_free(inv_norm);
mlx_array_free(projected);
projected = normalized;
- if (!arr_ok(projected)) goto cleanup;
+ if (!arr_ok(projected))
+ goto cleanup;
}
int vec_shape[] = {n_tokens, ctx->token_dim};
reshaped = mlx_array_new();
mlx_reshape(&reshaped, projected, vec_shape, 2, S);
- if (!arr_ok(reshaped)) goto cleanup;
+ if (!arr_ok(reshaped))
+ goto cleanup;
result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
- if (!result) goto cleanup;
+ if (!result)
+ goto cleanup;
result->owner = ctx;
result->vectors = reshaped;
result->tokens = n_tokens;
@@ -1035,28 +1062,23 @@ cleanup:
return result;
}
-void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs)
-{
- if (!vecs) return;
+void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) {
+ if (!vecs)
+ return;
mlx_array_free(vecs->vectors);
free(vecs);
}
-int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs)
-{
+int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) {
return vecs ? vecs->tokens : 0;
}
-int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs)
-{
+int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) {
return vecs ? vecs->dim : 0;
}
-int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs,
- float *out_vectors)
-{
- if (!vecs || !out_vectors || !arr_ok(vecs->vectors) ||
- vecs->tokens <= 0 || vecs->dim <= 0)
+int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) {
+ if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0)
return -1;
size_t values, bytes;
@@ -1084,32 +1106,33 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs,
}
embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *const *items, int count)
-{
- if (!ctx || !ctx->base || !items || count <= 0) return NULL;
+ embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) {
+ if (!ctx || !ctx->base || !items || count <= 0)
+ return NULL;
int dim = 0;
int total = 0;
for (int i = 0; i < count; i++) {
const embed_mlx_late_vectors_t *v = items[i];
- if (!v || v->owner != ctx || !arr_ok(v->vectors) ||
- v->tokens <= 0 || v->dim <= 0)
+ if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0)
return NULL;
- if (i == 0) dim = v->dim;
+ if (i == 0)
+ dim = v->dim;
if (v->dim != dim || total > INT_MAX - v->tokens)
return NULL;
total += v->tokens;
}
mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts));
- if (!parts) return NULL;
+ if (!parts)
+ return NULL;
for (int i = 0; i < count; i++)
parts[i] = items[i]->vectors;
mlx_stream S = ctx->base->stream;
mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count);
free(parts);
- if (!vec.ctx) return NULL;
+ if (!vec.ctx)
+ return NULL;
mlx_array joined = mlx_array_new();
mlx_concatenate_axis(&joined, vec, 0, S);
@@ -1119,8 +1142,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
return NULL;
}
- embed_mlx_late_vectors_t *out =
- (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
if (!out) {
mlx_array_free(joined);
return NULL;
@@ -1132,13 +1154,12 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
return out;
}
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *vecs,
- const int *token_indices, int count)
-{
- if (!ctx || !ctx->base || !vecs || vecs->owner != ctx ||
- !arr_ok(vecs->vectors) || !token_indices || count <= 0 ||
- vecs->tokens <= 0 || vecs->dim <= 0)
+embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
+ const embed_mlx_late_vectors_t *vecs,
+ const int *token_indices,
+ int count) {
+ if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) ||
+ !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0)
return NULL;
for (int i = 0; i < count; i++) {
@@ -1150,13 +1171,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0)
return NULL;
int *idx_data = (int *)malloc(idx_bytes);
- if (!idx_data) return NULL;
+ if (!idx_data)
+ return NULL;
memcpy(idx_data, token_indices, idx_bytes);
int idx_shape[] = {count};
mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32);
free(idx_data);
- if (!arr_ok(idx)) return NULL;
+ if (!arr_ok(idx))
+ return NULL;
mlx_stream S = ctx->base->stream;
mlx_array selected = mlx_array_new();
@@ -1167,8 +1190,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
return NULL;
}
- embed_mlx_late_vectors_t *out =
- (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
if (!out) {
mlx_array_free(selected);
return NULL;
@@ -1180,21 +1202,21 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(
return out;
}
-int embed_mlx_late_maxsim_batch_device(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs, const int *doc_offsets,
- int docs_count, float *scores)
-{
- if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 ||
- !scores || query->owner != ctx || docs->owner != ctx ||
- query->dim <= 0 || query->dim != docs->dim ||
- query->tokens <= 0 || docs->tokens <= 0 ||
- !arr_ok(query->vectors) || !arr_ok(docs->vectors) ||
- doc_offsets[0] != 0)
+int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
+ const embed_mlx_late_vectors_t *query,
+ const embed_mlx_late_vectors_t *docs,
+ const int *doc_offsets,
+ int docs_count,
+ float *scores) {
+ if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores ||
+ query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim ||
+ query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) ||
+ !arr_ok(docs->vectors) || doc_offsets[0] != 0)
return -1;
int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths));
- if (!lengths) return -1;
+ if (!lengths)
+ return -1;
int max_doc_tokens = 0;
for (int i = 0; i < docs_count; i++) {
if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] ||
@@ -1204,7 +1226,8 @@ int embed_mlx_late_maxsim_batch_device(
}
int n = doc_offsets[i + 1] - doc_offsets[i];
lengths[i] = n;
- if (n > max_doc_tokens) max_doc_tokens = n;
+ if (n > max_doc_tokens)
+ max_doc_tokens = n;
}
if (max_doc_tokens <= 0) {
free(lengths);
@@ -1214,8 +1237,7 @@ int embed_mlx_late_maxsim_batch_device(
mlx_stream S = ctx->base->stream;
int dim = query->dim;
int rc = -1;
- mlx_array *doc_rows =
- (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
+ mlx_array *doc_rows = (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
if (!doc_rows) {
free(lengths);
return -1;
@@ -1250,95 +1272,114 @@ int embed_mlx_late_maxsim_batch_device(
mlx_array padded = (mlx_array){0};
mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S);
- if (!arr_ok(doc)) goto cleanup;
+ if (!arr_ok(doc))
+ goto cleanup;
if (n < max_doc_tokens) {
int axes[] = {0};
int low[] = {0};
int high[] = {max_doc_tokens - n};
padded = mlx_array_new();
- mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value,
- "constant", S);
+ mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, "constant", S);
mlx_array_free(doc);
doc = padded;
padded = (mlx_array){0};
- if (!arr_ok(doc)) goto cleanup;
+ if (!arr_ok(doc))
+ goto cleanup;
}
doc_rows[i] = mlx_array_new();
mlx_expand_dims(&doc_rows[i], doc, 0, S);
mlx_array_free(doc);
- if (!arr_ok(doc_rows[i])) goto cleanup;
+ if (!arr_ok(doc_rows[i]))
+ goto cleanup;
}
doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count);
- if (!doc_vec.ctx) goto cleanup;
+ if (!doc_vec.ctx)
+ goto cleanup;
doc_batch = mlx_array_new();
mlx_concatenate_axis(&doc_batch, doc_vec, 0, S);
- if (!arr_ok(doc_batch)) goto cleanup;
+ if (!arr_ok(doc_batch))
+ goto cleanup;
query_3d = mlx_array_new();
mlx_expand_dims(&query_3d, query->vectors, 0, S);
- if (!arr_ok(query_3d)) goto cleanup;
+ if (!arr_ok(query_3d))
+ goto cleanup;
int q_shape[] = {docs_count, query->tokens, dim};
query_batched = mlx_array_new();
mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S);
- if (!arr_ok(query_batched)) goto cleanup;
+ if (!arr_ok(query_batched))
+ goto cleanup;
int perm[] = {0, 2, 1};
docs_t = mlx_array_new();
mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S);
- if (!arr_ok(docs_t)) goto cleanup;
+ if (!arr_ok(docs_t))
+ goto cleanup;
sims = mlx_array_new();
mlx_matmul(&sims, query_batched, docs_t, S);
- if (!arr_ok(sims)) goto cleanup;
+ if (!arr_ok(sims))
+ goto cleanup;
int len_shape[] = {docs_count};
lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32);
- if (!arr_ok(lengths_arr)) goto cleanup;
+ if (!arr_ok(lengths_arr))
+ goto cleanup;
positions = mlx_array_new();
mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S);
- if (!arr_ok(positions)) goto cleanup;
+ if (!arr_ok(positions))
+ goto cleanup;
lengths_2d = mlx_array_new();
mlx_expand_dims(&lengths_2d, lengths_arr, 1, S);
- if (!arr_ok(lengths_2d)) goto cleanup;
+ if (!arr_ok(lengths_2d))
+ goto cleanup;
positions_2d = mlx_array_new();
mlx_expand_dims(&positions_2d, positions, 0, S);
- if (!arr_ok(positions_2d)) goto cleanup;
+ if (!arr_ok(positions_2d))
+ goto cleanup;
valid = mlx_array_new();
mlx_less(&valid, positions_2d, lengths_2d, S);
- if (!arr_ok(valid)) goto cleanup;
+ if (!arr_ok(valid))
+ goto cleanup;
valid_3d = mlx_array_new();
mlx_expand_dims(&valid_3d, valid, 1, S);
- if (!arr_ok(valid_3d)) goto cleanup;
+ if (!arr_ok(valid_3d))
+ goto cleanup;
masked_sims = mlx_array_new();
mlx_where(&masked_sims, valid_3d, sims, invalid_value, S);
- if (!arr_ok(masked_sims)) goto cleanup;
+ if (!arr_ok(masked_sims))
+ goto cleanup;
maxes = mlx_array_new();
mlx_max_axis(&maxes, masked_sims, 2, false, S);
- if (!arr_ok(maxes)) goto cleanup;
+ if (!arr_ok(maxes))
+ goto cleanup;
sums = mlx_array_new();
mlx_sum_axis(&sums, maxes, 1, false, S);
- if (!arr_ok(sums)) goto cleanup;
+ if (!arr_ok(sums))
+ goto cleanup;
sums_f32 = mlx_array_new();
mlx_astype(&sums_f32, sums, MLX_FLOAT32, S);
- if (!arr_ok(sums_f32)) goto cleanup;
+ if (!arr_ok(sums_f32))
+ goto cleanup;
mlx_array_eval(sums_f32);
const float *data = mlx_array_data_float32(sums_f32);
- if (!data) goto cleanup;
+ if (!data)
+ goto cleanup;
memcpy(scores, data, (size_t)docs_count * sizeof(float));
rc = 0;
@@ -1359,7 +1400,8 @@ cleanup:
mlx_array_free(query_batched);
mlx_array_free(query_3d);
mlx_array_free(doc_batch);
- if (doc_vec.ctx) mlx_vector_array_free(doc_vec);
+ if (doc_vec.ctx)
+ mlx_vector_array_free(doc_vec);
mlx_array_free(pad_value);
for (int i = 0; i < docs_count; i++)
mlx_array_free(doc_rows[i]);
@@ -1369,31 +1411,32 @@ cleanup:
}
int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids, int n_tokens,
- int normalize, float *out_vectors)
-{
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
embed_mlx_late_vectors_t *vecs =
- embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens,
- normalize);
- if (!vecs) return -1;
+ embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize);
+ if (!vecs)
+ return -1;
int rc = embed_mlx_late_vectors_copy(vecs, out_vectors);
embed_mlx_late_vectors_free(vecs);
return rc;
}
static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs, int batch,
- float *out_embeddings)
-{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
- ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers)
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
+ ctx->layer_end != ctx->config.n_layers)
return -1;
const embed_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
- int hidden = c->hidden_size;
- int max_seq = 0;
+ int hidden = c->hidden_size;
+ int max_seq = 0;
int has_padding = 0;
int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN;
@@ -1401,13 +1444,17 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
return -1;
for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1;
- if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0)
+ return -1;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
}
- if (max_seq <= 0) return -1;
+ if (max_seq <= 0)
+ return -1;
for (int b = 0; b < batch; b++) {
- if (inputs[b].n_tokens != max_seq) has_padding = 1;
+ if (inputs[b].n_tokens != max_seq)
+ has_padding = 1;
}
int needs_pool_mask = has_padding && !last_token_pool;
int needs_attn_mask = has_padding && !causal;
@@ -1420,24 +1467,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 ||
mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 ||
mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 ||
- (last_token_pool &&
- mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
+ (last_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
return -1;
if (ids_bytes == 0 || out_bytes == 0 ||
(needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) ||
- (needs_attn_mask && mask_bytes == 0) ||
- (last_token_pool && last_index_bytes == 0))
+ (needs_attn_mask && mask_bytes == 0) || (last_token_pool && last_index_bytes == 0))
return -1;
int *padded_ids = (int *)malloc(ids_bytes);
- float *pool_mask_data =
- needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
- float *attn_mask_data =
- needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
+ float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
+ float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL;
- int *last_index_data =
- last_token_pool ? (int *)malloc(last_index_bytes) : NULL;
- if (!padded_ids ||
- (needs_pool_mask && (!pool_mask_data || !len_data)) ||
+ int *last_index_data = last_token_pool ? (int *)malloc(last_index_bytes) : NULL;
+ if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) ||
(needs_attn_mask && !attn_mask_data)) {
free(padded_ids);
free(pool_mask_data);
@@ -1455,7 +1496,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
}
for (int b = 0; b < batch; b++) {
- if (needs_pool_mask) len_data[b] = (float)inputs[b].n_tokens;
+ if (needs_pool_mask)
+ len_data[b] = (float)inputs[b].n_tokens;
if (last_token_pool) {
int index = inputs[b].n_tokens - 1;
for (int d = 0; d < hidden; d++)
@@ -1466,8 +1508,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
if (t < inputs[b].n_tokens) {
int id = inputs[b].ids[t];
if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
- id, b, t);
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t);
free(padded_ids);
free(pool_mask_data);
free(attn_mask_data);
@@ -1498,25 +1539,26 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
mlx_array pool_mask = needs_pool_mask
- ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
- : (mlx_array){0};
+ ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
+ : (mlx_array){0};
mlx_array attn_mask = needs_attn_mask
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array lengths = needs_pool_mask
- ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array last_indices = last_token_pool
- ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
- : (mlx_array){0};
- free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ mlx_array lengths =
+ needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0};
+ mlx_array last_indices =
+ last_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
+ : (mlx_array){0};
+ free(padded_ids);
+ free(pool_mask_data);
+ free(attn_mask_data);
+ free(len_data);
free(last_index_data);
- if (!arr_ok(ids) ||
- (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
- (needs_attn_mask && !arr_ok(attn_mask)) ||
- (last_token_pool && !arr_ok(last_indices))) {
- mlx_array_free(ids); mlx_array_free(pool_mask);
+ if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
+ (needs_attn_mask && !arr_ok(attn_mask)) || (last_token_pool && !arr_ok(last_indices))) {
+ mlx_array_free(ids);
+ mlx_array_free(pool_mask);
mlx_array_free(attn_mask);
mlx_array_free(lengths);
mlx_array_free(last_indices);
@@ -1528,7 +1570,9 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(attn_mask);
attn_mask = attn_mask_bf16;
if (!arr_ok(attn_mask)) {
- mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths);
+ mlx_array_free(ids);
+ mlx_array_free(pool_mask);
+ mlx_array_free(lengths);
mlx_array_free(last_indices);
return -1;
}
@@ -1540,9 +1584,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(ids);
/* 2. Transformer layers. */
- x = mlx_forward_layers(ctx, x, batch, max_seq,
- needs_attn_mask, attn_mask,
- 0, c->n_layers);
+ x = mlx_forward_layers(ctx, x, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
/* 3. Final RMSNorm. */
mlx_array x_normed = mlx_array_new();
@@ -1573,7 +1615,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(masked);
mlx_divide(&emb, emb_sum, lengths, S);
- mlx_array_free(emb_sum); mlx_array_free(lengths);
+ mlx_array_free(emb_sum);
+ mlx_array_free(lengths);
mlx_array_free(attn_mask);
} else {
/* Equal-length batches do not need masks or a separate divide. */
@@ -1595,8 +1638,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
if (c->normalize_embeddings) {
rc = 0;
for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(
- out_embeddings + (size_t)b * hidden, hidden) != 0) {
+ if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) {
rc = -1;
break;
}
@@ -1614,8 +1656,7 @@ typedef struct {
const embed_input_t *input;
} mlx_batch_item_t;
-static int mlx_batch_item_cmp(const void *a, const void *b)
-{
+static int mlx_batch_item_cmp(const void *a, const void *b) {
const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a;
const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b;
int da = ia->input->n_tokens;
@@ -1623,27 +1664,33 @@ static int mlx_batch_item_cmp(const void *a, const void *b)
return (da > db) - (da < db);
}
-static int mlx_padding_exceeds_limit(size_t padded, size_t tokens)
-{
+static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) {
if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5)
return 1;
return padded * 5 > tokens * 11;
}
-static int mlx_batch_should_split(const embed_input_t *inputs, int batch,
- int *out_total_tokens, int *out_max_seq)
-{
+static int mlx_batch_should_split(const embed_input_t *inputs,
+ int batch,
+ int *out_total_tokens,
+ int *out_max_seq) {
int max_seq = 0;
int total_tokens = 0;
for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0;
- if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
- if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0;
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0)
+ return 0;
+ if (inputs[b].n_tokens > max_seq)
+ max_seq = inputs[b].n_tokens;
+ if (total_tokens > INT_MAX - inputs[b].n_tokens)
+ return 0;
total_tokens += inputs[b].n_tokens;
}
- if (out_total_tokens) *out_total_tokens = total_tokens;
- if (out_max_seq) *out_max_seq = max_seq;
- if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0;
+ if (out_total_tokens)
+ *out_total_tokens = total_tokens;
+ if (out_max_seq)
+ *out_max_seq = max_seq;
+ if (batch < 4 || max_seq < 64 || total_tokens <= 0)
+ return 0;
size_t padded;
if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
@@ -1653,9 +1700,9 @@ static int mlx_batch_should_split(const embed_input_t *inputs, int batch,
}
static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
- int group_count, int group_max,
- int next_tokens)
-{
+ int group_count,
+ int group_max,
+ int next_tokens) {
if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0)
return 1;
@@ -1674,9 +1721,10 @@ static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
return merged <= split_total;
}
-int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
- int batch, float *out_embeddings)
-{
+int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
@@ -1684,19 +1732,20 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings);
const int hidden = ctx->config.hidden_size;
- mlx_batch_item_t *items = (mlx_batch_item_t *)malloc(
- (size_t)batch * sizeof(*items));
- embed_input_t *group_inputs = (embed_input_t *)malloc(
- (size_t)batch * sizeof(*group_inputs));
+ mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items));
+ embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
size_t group_values, group_bytes;
if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 ||
mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items); free(group_inputs);
+ free(items);
+ free(group_inputs);
return -1;
}
float *group_out = (float *)malloc(group_bytes);
if (!items || !group_inputs || !group_out) {
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return -1;
}
for (int i = 0; i < batch; i++) {
@@ -1706,7 +1755,7 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp);
int rc = 0;
- for (int start = 0; start < batch && rc == 0; ) {
+ for (int start = 0; start < batch && rc == 0;) {
int group_max = 0;
int group_tokens = 0;
int end = start;
@@ -1717,12 +1766,9 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
int next_total = group_tokens + next_tokens;
if (end > start) {
size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max,
- &next_padded) != 0 ||
- mlx_padding_exceeds_limit(next_padded,
- (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start,
- group_max, next_tokens))
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
+ mlx_padding_exceeds_limit(next_padded, (size_t)next_total) ||
+ !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
break;
}
group_max = next_max;
@@ -1733,26 +1779,28 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs,
int group_count = end - start;
for (int i = 0; i < group_count; i++)
group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count,
- group_out);
- if (rc != 0) break;
+ rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out);
+ if (rc != 0)
+ break;
for (int i = 0; i < group_count; i++) {
int dst = items[start + i].index;
- memcpy(out_embeddings + (size_t)dst * hidden,
- group_out + (size_t)i * hidden,
+ memcpy(out_embeddings + (size_t)dst * hidden, group_out + (size_t)i * hidden,
(size_t)hidden * sizeof(float));
}
start = end;
}
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return rc;
}
-int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, float *out_embedding)
-{
- embed_input_t input = { token_ids, n_tokens };
+int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ embed_input_t input = {token_ids, n_tokens};
return embed_mlx_encode_batch(ctx, &input, 1, out_embedding);
}
@@ -1762,8 +1810,7 @@ typedef struct {
const embed_context_input_t *input;
} mlx_context_batch_item_t;
-static int mlx_context_batch_item_cmp(const void *a, const void *b)
-{
+static int mlx_context_batch_item_cmp(const void *a, const void *b) {
const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a;
const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b;
int da = ia->input->input.n_tokens;
@@ -1771,33 +1818,33 @@ static int mlx_context_batch_item_cmp(const void *a, const void *b)
return (da > db) - (da < db);
}
-static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens)
-{
+static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) {
if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4)
return 1;
return padded * 4 > tokens * 5;
}
static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
- int batch, int *out_total_spans)
-{
+ int batch,
+ int *out_total_spans) {
int max_seq = 0;
int total_tokens = 0;
int total_spans = 0;
for (int b = 0; b < batch; b++) {
int n_tokens = inputs[b].input.n_tokens;
- if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans ||
- inputs[b].n_spans <= 0)
+ if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || inputs[b].n_spans <= 0)
return 0;
- if (n_tokens > max_seq) max_seq = n_tokens;
- if (total_tokens > INT_MAX - n_tokens ||
- total_spans > INT_MAX - inputs[b].n_spans)
+ if (n_tokens > max_seq)
+ max_seq = n_tokens;
+ if (total_tokens > INT_MAX - n_tokens || total_spans > INT_MAX - inputs[b].n_spans)
return 0;
total_tokens += n_tokens;
total_spans += inputs[b].n_spans;
}
- if (out_total_spans) *out_total_spans = total_spans;
- if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0;
+ if (out_total_spans)
+ *out_total_spans = total_spans;
+ if (batch < 2 || max_seq < 64 || total_tokens <= 0)
+ return 0;
size_t padded;
if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
@@ -1806,13 +1853,11 @@ static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
}
static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch, float *out_embeddings)
-{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
- ctx->layer_start != 0 ||
- ctx->layer_end != ctx->config.n_layers ||
- !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
+ ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
return -1;
const embed_config_t *c = &ctx->config;
@@ -1824,29 +1869,29 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
for (int b = 0; b < batch; b++) {
const embed_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans ||
- input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans)
+ if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
+ total_spans > INT_MAX - input->n_spans)
return -1;
total_spans += input->n_spans;
- if (n_tokens > max_seq) max_seq = n_tokens;
+ if (n_tokens > max_seq)
+ max_seq = n_tokens;
for (int t = 0; t < n_tokens; t++) {
- if (input->input.ids[t] < 0 ||
- input->input.ids[t] >= c->vocab_size) {
+ if (input->input.ids[t] < 0 || input->input.ids[t] >= c->vocab_size) {
fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
input->input.ids[t], b, t);
return -1;
}
}
for (int s = 0; s < input->n_spans; s++) {
- if (input->spans[s].start < 0 ||
- input->spans[s].start > n_tokens ||
+ if (input->spans[s].start < 0 || input->spans[s].start > n_tokens ||
input->spans[s].n_tokens <= 0 ||
input->spans[s].n_tokens > n_tokens - input->spans[s].start)
return -1;
}
}
for (int b = 0; b < batch; b++)
- if (inputs[b].input.n_tokens != max_seq) has_padding = 1;
+ if (inputs[b].input.n_tokens != max_seq)
+ has_padding = 1;
size_t rows, ids_bytes, mask_bytes, index_bytes;
size_t span_float_bytes, out_values, out_bytes;
@@ -1858,10 +1903,10 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 ||
mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
return -1;
- if (rows > INT_MAX) return -1;
+ if (rows > INT_MAX)
+ return -1;
int *padded_ids = (int *)malloc(ids_bytes);
- float *attn_mask_data = has_padding
- ? (float *)malloc(mask_bytes) : NULL;
+ float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
if (!padded_ids || (has_padding && !attn_mask_data)) {
free(padded_ids);
free(attn_mask_data);
@@ -1873,10 +1918,12 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
size_t row = (size_t)b * max_seq + t;
if (t < input->n_tokens) {
padded_ids[row] = input->ids[t];
- if (has_padding) attn_mask_data[row] = 0.0f;
+ if (has_padding)
+ attn_mask_data[row] = 0.0f;
} else {
padded_ids[row] = 0;
- if (has_padding) attn_mask_data[row] = -1.0e9f;
+ if (has_padding)
+ attn_mask_data[row] = -1.0e9f;
}
}
}
@@ -1908,44 +1955,49 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
int ids_shape[] = {batch, max_seq};
int attn_mask_shape[] = {batch, 1, 1, max_seq};
ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- attn_mask = has_padding
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
+ attn_mask = has_padding ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
free(padded_ids);
free(attn_mask_data);
- if (!arr_ok(ids)) goto cleanup;
- if (has_padding && !arr_ok(attn_mask)) goto cleanup;
+ if (!arr_ok(ids))
+ goto cleanup;
+ if (has_padding && !arr_ok(attn_mask))
+ goto cleanup;
if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
mlx_array attn_mask_bf16 = mlx_array_new();
mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
mlx_array_free(attn_mask);
attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) goto cleanup;
+ if (!arr_ok(attn_mask))
+ goto cleanup;
}
x = mlx_array_new();
mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
mlx_array_free(ids);
ids = (mlx_array){0};
- if (!arr_ok(x)) goto cleanup;
+ if (!arr_ok(x))
+ goto cleanup;
- x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
- 0, c->n_layers);
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers);
if (has_padding) {
mlx_array_free(attn_mask);
attn_mask = (mlx_array){0};
}
- if (!arr_ok(x)) goto cleanup;
+ if (!arr_ok(x))
+ goto cleanup;
x_normed = mlx_array_new();
mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
mlx_array_free(x);
x = (mlx_array){0};
- if (!arr_ok(x_normed)) goto cleanup;
+ if (!arr_ok(x_normed))
+ goto cleanup;
if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled));
- if (!pooled) goto cleanup;
+ if (!pooled)
+ goto cleanup;
int pooled_count = 0;
for (int b = 0; b < batch; b++) {
@@ -1964,23 +2016,25 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
pooled[pooled_count] = mlx_array_new();
mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S);
mlx_array_free(slice);
- if (!arr_ok(pooled[pooled_count])) goto cleanup;
+ if (!arr_ok(pooled[pooled_count]))
+ goto cleanup;
pooled_count++;
}
}
pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans);
- if (!pooled_vec.ctx) goto cleanup;
+ if (!pooled_vec.ctx)
+ goto cleanup;
embeddings = mlx_array_new();
mlx_concatenate_axis(&embeddings, pooled_vec, 0, S);
- if (!arr_ok(embeddings)) goto cleanup;
+ if (!arr_ok(embeddings))
+ goto cleanup;
} else {
end_idx_data = (int *)malloc(index_bytes);
start_idx_data = (int *)malloc(index_bytes);
lengths_data = (float *)malloc(span_float_bytes);
start_mask_data = (float *)malloc(span_float_bytes);
- if (!end_idx_data || !start_idx_data || !lengths_data ||
- !start_mask_data)
+ if (!end_idx_data || !start_idx_data || !lengths_data || !start_mask_data)
goto cleanup;
int span_count = 0;
@@ -2001,57 +2055,65 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
int index_shape[] = {total_spans};
int scale_shape[] = {total_spans, 1};
end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32);
- start_idx = mlx_array_new_data(start_idx_data, index_shape, 1,
- MLX_INT32);
- lengths = mlx_array_new_data(lengths_data, scale_shape, 2,
- MLX_FLOAT32);
- start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2,
- MLX_FLOAT32);
- free(end_idx_data); end_idx_data = NULL;
- free(start_idx_data); start_idx_data = NULL;
- free(lengths_data); lengths_data = NULL;
- free(start_mask_data); start_mask_data = NULL;
- if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) ||
- !arr_ok(start_mask))
+ start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, MLX_INT32);
+ lengths = mlx_array_new_data(lengths_data, scale_shape, 2, MLX_FLOAT32);
+ start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, MLX_FLOAT32);
+ free(end_idx_data);
+ end_idx_data = NULL;
+ free(start_idx_data);
+ start_idx_data = NULL;
+ free(lengths_data);
+ lengths_data = NULL;
+ free(start_mask_data);
+ start_mask_data = NULL;
+ if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || !arr_ok(start_mask))
goto cleanup;
prefix = mlx_array_new();
mlx_cumsum(&prefix, x_normed, 1, false, true, S);
mlx_array_free(x_normed);
x_normed = (mlx_array){0};
- if (!arr_ok(prefix)) goto cleanup;
+ if (!arr_ok(prefix))
+ goto cleanup;
int flat_shape[] = {(int)rows, hidden};
flat_prefix = mlx_array_new();
mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S);
mlx_array_free(prefix);
prefix = (mlx_array){0};
- if (!arr_ok(flat_prefix)) goto cleanup;
+ if (!arr_ok(flat_prefix))
+ goto cleanup;
end_values = mlx_array_new();
mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S);
- if (!arr_ok(end_values)) goto cleanup;
+ if (!arr_ok(end_values))
+ goto cleanup;
start_values = mlx_array_new();
mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S);
- if (!arr_ok(start_values)) goto cleanup;
+ if (!arr_ok(start_values))
+ goto cleanup;
masked_start = mlx_array_new();
mlx_multiply(&masked_start, start_values, start_mask, S);
- if (!arr_ok(masked_start)) goto cleanup;
+ if (!arr_ok(masked_start))
+ goto cleanup;
span_sums = mlx_array_new();
mlx_subtract(&span_sums, end_values, masked_start, S);
- if (!arr_ok(span_sums)) goto cleanup;
+ if (!arr_ok(span_sums))
+ goto cleanup;
embeddings = mlx_array_new();
mlx_divide(&embeddings, span_sums, lengths, S);
- if (!arr_ok(embeddings)) goto cleanup;
+ if (!arr_ok(embeddings))
+ goto cleanup;
}
embeddings_f32 = mlx_array_new();
mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S);
- if (!arr_ok(embeddings_f32)) goto cleanup;
+ if (!arr_ok(embeddings_f32))
+ goto cleanup;
mlx_array_eval(embeddings_f32);
const float *data = mlx_array_data_float32(embeddings_f32);
@@ -2067,9 +2129,11 @@ cleanup:
free(end_idx_data);
mlx_array_free(embeddings_f32);
mlx_array_free(embeddings);
- if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec);
+ if (pooled_vec.ctx)
+ mlx_vector_array_free(pooled_vec);
if (pooled) {
- for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]);
+ for (int s = 0; s < total_spans; s++)
+ mlx_array_free(pooled[s]);
free(pooled);
}
mlx_array_free(span_sums);
@@ -2090,31 +2154,33 @@ cleanup:
}
int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs, int batch,
- float *out_embeddings)
-{
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
int total_spans = 0;
if (!mlx_context_batch_should_split(inputs, batch, &total_spans))
- return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch,
- out_embeddings);
+ return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings);
const int hidden = ctx->config.hidden_size;
- mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc(
- (size_t)batch * sizeof(*items));
- embed_context_input_t *group_inputs = (embed_context_input_t *)malloc(
- (size_t)batch * sizeof(*group_inputs));
+ mlx_context_batch_item_t *items =
+ (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items));
+ embed_context_input_t *group_inputs =
+ (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
size_t group_values, group_bytes;
if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 ||
mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items); free(group_inputs);
+ free(items);
+ free(group_inputs);
return -1;
}
float *group_out = (float *)malloc(group_bytes);
if (!items || !group_inputs || !group_out) {
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return -1;
}
@@ -2128,7 +2194,7 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp);
int rc = 0;
- for (int start = 0; start < batch && rc == 0; ) {
+ for (int start = 0; start < batch && rc == 0;) {
int group_max = 0;
int group_tokens = 0;
int end = start;
@@ -2139,12 +2205,9 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
int next_total = group_tokens + next_tokens;
if (end > start) {
size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max,
- &next_padded) != 0 ||
- mlx_context_padding_exceeds_limit(next_padded,
- (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start,
- group_max, next_tokens))
+ if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
+ mlx_context_padding_exceeds_limit(next_padded, (size_t)next_total) ||
+ !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
break;
}
group_max = next_max;
@@ -2155,47 +2218,51 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
int group_count = end - start;
for (int i = 0; i < group_count; i++)
group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs,
- group_count, group_out);
- if (rc != 0) break;
+ rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out);
+ if (rc != 0)
+ break;
int src_span = 0;
for (int i = 0; i < group_count; i++) {
int n_spans = group_inputs[i].n_spans;
int dst_span = items[start + i].span_offset;
memcpy(out_embeddings + (size_t)dst_span * hidden,
- group_out + (size_t)src_span * hidden,
- (size_t)n_spans * hidden * sizeof(float));
+ group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float));
src_span += n_spans;
}
start = end;
}
- free(items); free(group_inputs); free(group_out);
+ free(items);
+ free(group_inputs);
+ free(group_out);
return rc;
}
-int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, const embed_span_t *spans,
- int n_spans, float *out_embeddings)
-{
+int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
embed_context_input_t input = {
- .input = { token_ids, n_tokens },
+ .input = {token_ids, n_tokens},
.spans = spans,
.n_spans = n_spans,
};
return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings);
}
-float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
-{
- if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) {
+ if (!ctx || !token_ids || n_tokens <= 0)
+ return NULL;
int hidden = ctx->config.hidden_size;
size_t out_bytes;
if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0)
return NULL;
float *out = (float *)malloc(out_bytes);
- if (!out) return NULL;
+ if (!out)
+ return NULL;
if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) {
free(out);