diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
| commit | c4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch) | |
| tree | 81ce5675ce886aad0309a41f4d6a696826e0a061 /embed_mlx.c | |
| parent | b377ba8656bfb807ed8dda6c9f26942b50ce626d (diff) | |
| download | ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip | |
chore: run clang-format
Diffstat (limited to 'embed_mlx.c')
| -rw-r--r-- | embed_mlx.c | 1153 |
1 files changed, 610 insertions, 543 deletions
diff --git a/embed_mlx.c b/embed_mlx.c index cb3b737..9f20026 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -35,12 +35,12 @@ typedef struct { struct embed_mlx_ctx { embed_config_t config; mlx_array embed_tokens; - mlx_layer_t *layers; /* heap-allocated [n_layers] */ + mlx_layer_t *layers; /* heap-allocated [n_layers] */ mlx_array norm; mlx_stream stream; multi_safetensors_t *ms; - int layer_start; /* loaded transformer range, inclusive */ - int layer_end; /* loaded transformer range, exclusive */ + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ int quantize_bits; int quantize_group_size; mlx_dtype weight_dtype; @@ -55,7 +55,7 @@ struct embed_mlx_late_ctx { struct embed_mlx_late_vectors { embed_mlx_late_ctx_t *owner; - mlx_array vectors; /* [tokens, dim] */ + mlx_array vectors; /* [tokens, dim] */ int tokens; int dim; }; @@ -66,47 +66,46 @@ struct embed_mlx_late_vectors { static int arr_ok(mlx_array a) { return a.ctx != NULL; } -static int mlx_mul_size(size_t a, size_t b, size_t *out) -{ - if (a != 0 && b > SIZE_MAX / a) return -1; +static int mlx_mul_size(size_t a, size_t b, size_t *out) { + if (a != 0 && b > SIZE_MAX / a) + return -1; *out = a * b; return 0; } -static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) -{ - return ctx->weight_dtype; -} +static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; } -static size_t safetensor_dtype_size(safetensor_dtype_t dtype) -{ +static size_t safetensor_dtype_size(safetensor_dtype_t dtype) { switch (dtype) { - case DTYPE_F32: return sizeof(float); - case DTYPE_BF16: return sizeof(uint16_t); - default: return 0; + case DTYPE_F32: + return sizeof(float); + case DTYPE_BF16: + return sizeof(uint16_t); + default: + return 0; } } static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, const safetensor_t *t, - const char *name, const int *shape, - int ndim) -{ + const char *name, + const int *shape, + int ndim) { size_t elem_size = safetensor_dtype_size(t->dtype); if (elem_size == 0) { fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); return 0; } if (t->ndim != ndim) { - fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", - name, t->ndim, ndim); + fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); return 0; } size_t numel = 1; for (int i = 0; i < ndim; i++) { if (shape[i] <= 0 || t->shape[i] != shape[i]) { - fprintf(stderr, "mlx: bad shape for %s at dim %d: " + fprintf(stderr, + "mlx: bad shape for %s at dim %d: " "got %lld, expected %d\n", name, i, (long long)t->shape[i], shape[i]); return 0; @@ -123,15 +122,15 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, return 0; } if (t->data_size != bytes) { - fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", - name, t->data_size, bytes); + fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, + bytes); return 0; } - if (sf->header_size > SIZE_MAX - 8) return 0; + if (sf->header_size > SIZE_MAX - 8) + return 0; size_t data_start = 8 + sf->header_size; - if (data_start > sf->file_size || - t->data_offset > sf->file_size - data_start || + if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || t->data_size > sf->file_size - data_start - t->data_offset) { fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name); return 0; @@ -139,11 +138,10 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } -static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, - const int *shape, int ndim) -{ +static mlx_array +load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { fprintf(stderr, "mlx: tensor not found: %s\n", name); return (mlx_array){0}; @@ -162,22 +160,16 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); } -static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms, - const char *name, - const int *shape, int ndim) -{ +static mlx_linear_t +load_linear_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { mlx_linear_t l = {0}; l.w = load_tensor(ms, name, shape, ndim); return l; } -static int linear_ok(mlx_linear_t l) -{ - return arr_ok(l.w) || arr_ok(l.wt); -} +static int linear_ok(mlx_linear_t l) { return arr_ok(l.w) || arr_ok(l.wt); } -static void free_linear(mlx_linear_t *l) -{ +static void free_linear(mlx_linear_t *l) { mlx_array_free(l->w); mlx_array_free(l->wt); mlx_array_free(l->scales); @@ -185,8 +177,7 @@ static void free_linear(mlx_linear_t *l) memset(l, 0, sizeof(*l)); } -static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) -{ +static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) { if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt)) return 0; @@ -214,15 +205,15 @@ static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) return 0; } -static int quantize_linear(mlx_linear_t *l, int bits, int group_size, - mlx_stream s) -{ - if (!l || !arr_ok(l->w) || bits == 0) return 0; +static int quantize_linear(mlx_linear_t *l, int bits, int group_size, mlx_stream s) { + if (!l || !arr_ok(l->w) || bits == 0) + return 0; if (bits != 8) { fprintf(stderr, "mlx: quantized weights support only 8 bits\n"); return -1; } - if (group_size <= 0) group_size = 64; + if (group_size <= 0) + group_size = 64; mlx_optional_int gs = {.value = group_size, .has_value = true}; mlx_optional_int qb = {.value = bits, .has_value = true}; @@ -253,7 +244,8 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size, } mlx_array_eval(qw); mlx_array_eval(scales); - if (arr_ok(biases)) mlx_array_eval(biases); + if (arr_ok(biases)) + mlx_array_eval(biases); mlx_synchronize(s); mlx_array_free(l->w); @@ -269,16 +261,15 @@ static int quantize_linear(mlx_linear_t *l, int bits, int group_size, } /* y = x @ W^T */ -static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) -{ +static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) { if (W->quantized) { - if (!arr_ok(W->w)) return (mlx_array){0}; + if (!arr_ok(W->w)) + return (mlx_array){0}; mlx_array res = mlx_array_new(); mlx_optional_int gs = {.value = W->group_size, .has_value = true}; mlx_optional_int bits = {.value = W->bits, .has_value = true}; mlx_array null_arr = (mlx_array){0}; - mlx_quantized_matmul(&res, x, W->w, W->scales, - arr_ok(W->biases) ? W->biases : null_arr, + mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr, true, gs, bits, "affine", s); return res; } @@ -294,8 +285,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) return (mlx_array){0}; } mlx_array res = mlx_array_new(); - if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && - mlx_array_dtype(x) == MLX_FLOAT32) { + if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && mlx_array_dtype(x) == MLX_FLOAT32) { /* F32 activations with BF16 weights: upcast the weight operand and run * the matmul in F32 (so accumulation is F32). Only the weights are * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers), @@ -307,21 +297,21 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) } else { mlx_matmul(&res, x, Wt, s); } - if (free_wt) mlx_array_free(Wt); + if (free_wt) + mlx_array_free(Wt); return res; } -const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) -{ +const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) { return ctx ? &ctx->config : NULL; } /* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the * residual stream in F32 while keeping the large matmul weights in BF16. */ -static void upcast_f32_inplace(mlx_array *a, mlx_stream s) -{ - if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) return; +static void upcast_f32_inplace(mlx_array *a, mlx_stream s) { + if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32) + return; mlx_array f = mlx_array_new(); mlx_astype(&f, *a, MLX_FLOAT32, s); mlx_array_free(*a); @@ -335,14 +325,13 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s) /* We duplicate a minimal config parser here so MLX can load standalone. * (Same logic as embed.c:parse_config, kept inline for simplicity.) */ -static const char *skip_ws_m(const char *p) -{ - while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; +static const char *skip_ws_m(const char *p) { + while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + p++; return p; } -static const char *json_find_m(const char *json, const char *key) -{ +static const char *json_find_m(const char *json, const char *key) { size_t klen = strlen(key); const char *p = json; while ((p = strstr(p, key)) != NULL) { @@ -350,7 +339,8 @@ static const char *json_find_m(const char *json, const char *key) const char *a = p + klen; if (*a == '"') { a = skip_ws_m(a + 1); - if (*a == ':') return skip_ws_m(a + 1); + if (*a == ':') + return skip_ws_m(a + 1); } } p += klen; @@ -358,18 +348,16 @@ static const char *json_find_m(const char *json, const char *key) return NULL; } -static int json_string_equals_m(const char *json, const char *key, - const char *expected) -{ +static int json_string_equals_m(const char *json, const char *key, const char *expected) { const char *v = json_find_m(json, key); - if (!v || *v != '"') return 0; + if (!v || *v != '"') + return 0; v++; size_t len = strlen(expected); return strncmp(v, expected, len) == 0 && v[len] == '"'; } -static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) -{ +static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) { char path[1024]; int path_len = snprintf(path, sizeof(path), "%s/config.json", model_dir); if (path_len < 0 || (size_t)path_len >= sizeof(path)) { @@ -377,8 +365,14 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) return -1; } FILE *f = fopen(path, "rb"); - if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; } - if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return -1; } + if (!f) { + fprintf(stderr, "mlx: cannot open %s\n", path); + return -1; + } + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } long sz = ftell(f); if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { fclose(f); @@ -386,22 +380,32 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) } char *buf = (char *)malloc((size_t)sz + 1); if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - fclose(f); free(buf); return -1; + fclose(f); + free(buf); + return -1; } fclose(f); buf[sz] = '\0'; -#define GI(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : def; } while(0) -#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0) - GI("hidden_size", hidden_size, 0); - GI("num_hidden_layers", n_layers, 0); - GI("num_attention_heads", n_heads, 0); - GI("num_key_value_heads", n_kv_heads, 0); - GI("head_dim", head_dim, EMBED_HEAD_DIM); - GI("intermediate_size", intermediate_size, 0); - GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE); - GF("rms_norm_eps", rms_norm_eps, 1e-6f); - GF("rope_theta", rope_theta, 1000000.0f); +#define GI(k, fb, def) \ + do { \ + const char *v = json_find_m(buf, k); \ + cfg->fb = v ? atoi(v) : def; \ + } while (0) +#define GF(k, fb, def) \ + do { \ + const char *v = json_find_m(buf, k); \ + cfg->fb = v ? (float)atof(v) : def; \ + } while (0) + GI("hidden_size", hidden_size, 0); + GI("num_hidden_layers", n_layers, 0); + GI("num_attention_heads", n_heads, 0); + GI("num_key_value_heads", n_kv_heads, 0); + GI("head_dim", head_dim, EMBED_HEAD_DIM); + GI("intermediate_size", intermediate_size, 0); + GI("vocab_size", vocab_size, EMBED_VOCAB_SIZE); + GF("rms_norm_eps", rms_norm_eps, 1e-6f); + GF("rope_theta", rope_theta, 1000000.0f); #undef GI #undef GF cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; @@ -417,58 +421,53 @@ static int mlx_parse_config(embed_config_t *cfg, const char *model_dir) cfg->q_dim = 0; cfg->kv_dim = 0; - if (cfg->n_heads > 0 && cfg->head_dim > 0 && - cfg->n_heads <= INT_MAX / cfg->head_dim) + if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim) cfg->q_dim = cfg->n_heads * cfg->head_dim; - if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && - cfg->n_kv_heads <= INT_MAX / cfg->head_dim) + if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim) cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; free(buf); - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || - cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || - cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || - cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 || - (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 || - !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || - !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { - fprintf(stderr, "mlx: invalid config in %s " + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 || + cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || + cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) || + cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || + cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { + fprintf(stderr, + "mlx: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", - path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, - cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); + path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim, + cfg->intermediate_size); return -1; } if (cfg->n_layers > EMBED_MAX_LAYERS) { - fprintf(stderr, "mlx: too many layers (%d > %d)\n", - cfg->n_layers, EMBED_MAX_LAYERS); + fprintf(stderr, "mlx: too many layers (%d > %d)\n", cfg->n_layers, EMBED_MAX_LAYERS); return -1; } return 0; } -static int mlx_model_dir_has_file(const char *model_dir, const char *rel) -{ +static int mlx_model_dir_has_file(const char *model_dir, const char *rel) { char path[1024]; int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel); - if (n < 0 || (size_t)n >= sizeof(path)) return 0; + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; FILE *f = fopen(path, "rb"); - if (!f) return 0; + if (!f) + return 0; fclose(f); return 1; } -static int mlx_model_dir_has_late_projection(const char *model_dir) -{ +static int mlx_model_dir_has_late_projection(const char *model_dir) { return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") && mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors"); } -static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) -{ +static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) { embed_mlx_options_t out = {0}; - if (opts) out = *opts; - if (out.quantize_bits != 0 && - out.quantize_bits != 8) { + if (opts) + out = *opts; + if (out.quantize_bits != 0 && out.quantize_bits != 8) { fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); out.quantize_bits = -1; } @@ -477,10 +476,9 @@ static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts return out; } -static int quantize_layer(mlx_layer_t *l, int bits, int group_size, - mlx_stream s) -{ - if (bits == 0) return 0; +static int quantize_layer(mlx_layer_t *l, int bits, int group_size, mlx_stream s) { + if (bits == 0) + return 0; return quantize_linear(&l->wq, bits, group_size, s) || quantize_linear(&l->wk, bits, group_size, s) || quantize_linear(&l->wv, bits, group_size, s) || @@ -490,45 +488,45 @@ static int quantize_layer(mlx_layer_t *l, int bits, int group_size, quantize_linear(&l->down_proj, bits, group_size, s); } -static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) -{ - return prepare_linear_transpose(&l->wq, s) || - prepare_linear_transpose(&l->wk, s) || - prepare_linear_transpose(&l->wv, s) || - prepare_linear_transpose(&l->wo, s) || - prepare_linear_transpose(&l->gate_proj, s) || - prepare_linear_transpose(&l->up_proj, s) || +static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) { + return prepare_linear_transpose(&l->wq, s) || prepare_linear_transpose(&l->wk, s) || + prepare_linear_transpose(&l->wv, s) || prepare_linear_transpose(&l->wo, s) || + prepare_linear_transpose(&l->gate_proj, s) || prepare_linear_transpose(&l->up_proj, s) || prepare_linear_transpose(&l->down_proj, s); } static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, - int layer_start, int layer_end, - const embed_mlx_options_t *opts, - int allow_late) -{ + int layer_start, + int layer_end, + const embed_mlx_options_t *opts, + int allow_late) { embed_mlx_options_t options = normalize_mlx_options(opts); - if (options.quantize_bits < 0) return NULL; + if (options.quantize_bits < 0) + return NULL; if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) { fprintf(stderr, "mlx: late-interaction models require token-level " - "MaxSim support and are not valid pooled embedding models " - "yet\n"); + "MaxSim support and are not valid pooled embedding models " + "yet\n"); return NULL; } embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t)); - if (!ctx) return NULL; + if (!ctx) + return NULL; ctx->quantize_bits = options.quantize_bits; ctx->quantize_group_size = options.quantize_group_size; - if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } + if (mlx_parse_config(&ctx->config, model_dir) != 0) { + free(ctx); + return NULL; + } const embed_config_t *c = &ctx->config; if (layer_end == -1) layer_end = c->n_layers; - if (layer_start < 0 || layer_start >= layer_end || - layer_end > c->n_layers) { - fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", - layer_start, layer_end, c->n_layers); + if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) { + fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", layer_start, + layer_end, c->n_layers); free(ctx); return NULL; } @@ -538,7 +536,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, ctx->ms = multi_safetensors_open(model_dir); if (!ctx->ms) { fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); - free(ctx); return NULL; + free(ctx); + return NULL; } int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; @@ -547,64 +546,73 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, /* Only the first stage needs token embeddings. */ if (layer_start == 0) { int emb_shape[] = {c->vocab_size, h}; - ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", - emb_shape, 2); - if (!arr_ok(ctx->embed_tokens)) goto fail; + ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2); + if (!arr_ok(ctx->embed_tokens)) + goto fail; } /* Layers */ ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); - if (!ctx->layers) goto fail; + if (!ctx->layers) + goto fail; char name[256]; for (int i = layer_start; i < layer_end; i++) { mlx_layer_t *l = &ctx->layers[i]; -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!linear_ok(l->fld)) \ + goto fail; \ + } while (0) - LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); - LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); - LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); - LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); + LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); + LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); + LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); + LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); #undef LD -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!arr_ok(l->fld)) goto fail; \ -} while (0) +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!arr_ok(l->fld)) \ + goto fail; \ + } while (0) - LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); - LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); - LD(input_norm, "layers.%d.input_layernorm.weight", h); + LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); + LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); + LD(input_norm, "layers.%d.input_layernorm.weight", h); LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); #undef LD -#define LD(fld, fmt, ...) do { \ - snprintf(name, sizeof(name), fmt, i); \ - int sh[] = {__VA_ARGS__}; \ - l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!linear_ok(l->fld)) goto fail; \ -} while (0) +#define LD(fld, fmt, ...) \ + do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \ + if (!linear_ok(l->fld)) \ + goto fail; \ + } while (0) - LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); - LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); - LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); + LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); + LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); + LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); #undef LD - if (embed_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); + if (embed_verbose >= 2) + fprintf(stderr, " mlx layer %d loaded\n", i); } if (layer_end == c->n_layers) { int ns[] = {h}; ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); - if (!arr_ok(ctx->norm)) goto fail; + if (!arr_ok(ctx->norm)) + goto fail; } ctx->stream = mlx_default_gpu_stream_new(); @@ -624,13 +632,14 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, } if (ctx->quantize_bits) { for (int i = layer_start; i < layer_end; i++) { - if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, - ctx->quantize_group_size, ctx->stream) != 0) + if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, ctx->quantize_group_size, + ctx->stream) != 0) goto fail; } mlx_clear_cache(); if (embed_verbose >= 1) - fprintf(stderr, "mlx: quantized linear weights to %d-bit " + fprintf(stderr, + "mlx: quantized linear weights to %d-bit " "(group_size=%d)\n", ctx->quantize_bits, ctx->quantize_group_size); } else { @@ -645,7 +654,8 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, mlx_get_active_memory(&active); mlx_get_cache_memory(&cache); mlx_get_peak_memory(&peak); - fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB " + fprintf(stderr, + "mlx: memory active=%.2f GiB cache=%.2f GiB " "peak=%.2f GiB\n", (double)active / (1024.0 * 1024.0 * 1024.0), (double)cache / (1024.0 * 1024.0 * 1024.0), @@ -653,8 +663,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, } if (embed_verbose >= 1) - fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", - layer_start, layer_end, h); + fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h); return ctx; @@ -663,14 +672,12 @@ fail: return NULL; } -embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) -{ +embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) { return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); } embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, - const embed_mlx_options_t *opts) -{ + const embed_mlx_options_t *opts) { return mlx_load_range_ex(model_dir, 0, -1, opts, 0); } @@ -678,19 +685,23 @@ embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, * Free * ======================================================================== */ -static void free_mlx_layer(mlx_layer_t *l) -{ - free_linear(&l->wq); free_linear(&l->wk); - free_linear(&l->wv); free_linear(&l->wo); - mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); - mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); - free_linear(&l->gate_proj); free_linear(&l->up_proj); +static void free_mlx_layer(mlx_layer_t *l) { + free_linear(&l->wq); + free_linear(&l->wk); + free_linear(&l->wv); + free_linear(&l->wo); + mlx_array_free(l->q_norm); + mlx_array_free(l->k_norm); + mlx_array_free(l->input_norm); + mlx_array_free(l->post_attn_norm); + free_linear(&l->gate_proj); + free_linear(&l->up_proj); free_linear(&l->down_proj); } -void embed_mlx_free(embed_mlx_ctx_t *ctx) -{ - if (!ctx) return; +void embed_mlx_free(embed_mlx_ctx_t *ctx) { + if (!ctx) + return; mlx_array_free(ctx->embed_tokens); if (ctx->layers) { for (int i = 0; i < ctx->config.n_layers; i++) @@ -698,24 +709,27 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx) free(ctx->layers); } mlx_array_free(ctx->norm); - if (ctx->stream.ctx) mlx_stream_free(ctx->stream); - if (ctx->ms) multi_safetensors_close(ctx->ms); + if (ctx->stream.ctx) + mlx_stream_free(ctx->stream); + if (ctx->ms) + multi_safetensors_close(ctx->ms); free(ctx); } -embed_mlx_late_ctx_t *embed_mlx_late_load_with_options( - const char *model_dir, const embed_mlx_options_t *opts) -{ +embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir, + const embed_mlx_options_t *opts) { if (!mlx_model_dir_has_late_projection(model_dir)) { fprintf(stderr, "mlx late: missing 1_Dense projection\n"); return NULL; } embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); - if (!late) return NULL; + if (!late) + return NULL; late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1); - if (!late->base) goto fail; + if (!late->base) + goto fail; char dense_dir[1024]; int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir); @@ -726,13 +740,11 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options( late->dense_ms = multi_safetensors_open(dense_dir); if (!late->dense_ms) { - fprintf(stderr, "mlx late: failed to open safetensors in %s\n", - dense_dir); + fprintf(stderr, "mlx late: failed to open safetensors in %s\n", dense_dir); goto fail; } - const safetensor_t *t = multi_safetensors_find(late->dense_ms, - "linear.weight", NULL); + const safetensor_t *t = multi_safetensors_find(late->dense_ms, "linear.weight", NULL); if (!t) { fprintf(stderr, "mlx late: missing linear.weight\n"); goto fail; @@ -743,17 +755,16 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options( goto fail; } - int shape[] = { (int)t->shape[0], late->base->config.hidden_size }; - late->projection = load_linear_tensor(late->dense_ms, "linear.weight", - shape, 2); - if (!linear_ok(late->projection)) goto fail; + int shape[] = {(int)t->shape[0], late->base->config.hidden_size}; + late->projection = load_linear_tensor(late->dense_ms, "linear.weight", shape, 2); + if (!linear_ok(late->projection)) + goto fail; if (prepare_linear_transpose(&late->projection, late->base->stream) != 0) goto fail; late->token_dim = shape[0]; if (embed_verbose >= 1) - fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", - late->token_dim); + fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim); return late; fail: @@ -761,39 +772,38 @@ fail: return NULL; } -embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) -{ +embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) { return embed_mlx_late_load_with_options(model_dir, NULL); } -void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) -{ - if (!ctx) return; +void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) { + if (!ctx) + return; free_linear(&ctx->projection); - if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms); + if (ctx->dense_ms) + multi_safetensors_close(ctx->dense_ms); embed_mlx_free(ctx->base); free(ctx); } -const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) -{ +const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) { return ctx && ctx->base ? &ctx->base->config : NULL; } -int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) -{ - return ctx ? ctx->token_dim : 0; -} +int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } /* ======================================================================== * Forward pass * ======================================================================== */ -static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, - int batch, int max_seq, int has_padding, +static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, + mlx_array x, + int batch, + int max_seq, + int has_padding, mlx_array attn_mask, - int layer_start, int layer_end) -{ + int layer_start, + int layer_end) { const embed_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int n_heads = c->n_heads; @@ -802,8 +812,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, int q_dim = c->q_dim; float scale = 1.0f / sqrtf((float)head_dim); mlx_array null_arr = (mlx_array){0}; - const char *mask_mode = - c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : ""; + const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : ""; /* For non-quantized BF16 weights, run the residual stream, norms, and * attention in F32; matmuls downcast their operands to BF16 internally @@ -812,8 +821,8 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, * models are unaffected. */ mlx_array local_mask = attn_mask; int free_mask = 0; - int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 && - !ctx->layers[layer_start].wq.quantized; + int mixed_bf16 = + mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized; if (mixed_bf16) { if (mlx_array_dtype(x) != MLX_FLOAT32) { mlx_array xf = mlx_array_new(); @@ -821,8 +830,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_array_free(x); x = xf; } - if (has_padding && arr_ok(attn_mask) && - mlx_array_dtype(attn_mask) != MLX_FLOAT32) { + if (has_padding && arr_ok(attn_mask) && mlx_array_dtype(attn_mask) != MLX_FLOAT32) { local_mask = mlx_array_new(); mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S); free_mask = 1; @@ -840,7 +848,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_array v_flat = linear(xn, &l->wv, S); mlx_array_free(xn); - int q_shape[] = {batch, max_seq, n_heads, head_dim}; + int q_shape[] = {batch, max_seq, n_heads, head_dim}; int k_shape[] = {batch, max_seq, n_kv_heads, head_dim}; mlx_array q = mlx_array_new(); mlx_array k = mlx_array_new(); @@ -848,33 +856,40 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_reshape(&q, q_flat, q_shape, 4, S); mlx_reshape(&k, k_flat, k_shape, 4, S); mlx_reshape(&v, v_flat, k_shape, 4, S); - mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); + mlx_array_free(q_flat); + mlx_array_free(k_flat); + mlx_array_free(v_flat); mlx_array qn = mlx_array_new(); mlx_array kn = mlx_array_new(); mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); - mlx_array_free(q); mlx_array_free(k); + mlx_array_free(q); + mlx_array_free(k); /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */ int perm[] = {0, 2, 1, 3}; mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); mlx_transpose_axes(&qt, qn, perm, 4, S); mlx_transpose_axes(&kt, kn, perm, 4, S); - mlx_transpose_axes(&vt, v, perm, 4, S); - mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v); + mlx_transpose_axes(&vt, v, perm, 4, S); + mlx_array_free(qn); + mlx_array_free(kn); + mlx_array_free(v); mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; mlx_array qr = mlx_array_new(), kr = mlx_array_new(); mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); - mlx_array_free(qt); mlx_array_free(kt); + mlx_array_free(qt); + mlx_array_free(kt); mlx_array attn = mlx_array_new(); - mlx_fast_scaled_dot_product_attention( - &attn, qr, kr, vt, scale, mask_mode, - has_padding ? local_mask : null_arr, null_arr, S); - mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); + mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, mask_mode, + has_padding ? local_mask : null_arr, null_arr, S); + mlx_array_free(qr); + mlx_array_free(kr); + mlx_array_free(vt); mlx_array attn_t = mlx_array_new(); mlx_transpose_axes(&attn_t, attn, perm, 4, S); @@ -889,44 +904,49 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array x, mlx_array_free(attn_flat); mlx_array x2 = mlx_array_new(); mlx_add(&x2, x, proj, S); - mlx_array_free(x); mlx_array_free(proj); + mlx_array_free(x); + mlx_array_free(proj); x = x2; mlx_array xn2 = mlx_array_new(); mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); mlx_array gate = linear(xn2, &l->gate_proj, S); - mlx_array up = linear(xn2, &l->up_proj, S); + mlx_array up = linear(xn2, &l->up_proj, S); mlx_array_free(xn2); mlx_array gate_sig = mlx_array_new(); mlx_sigmoid(&gate_sig, gate, S); mlx_array silu_gate = mlx_array_new(); mlx_multiply(&silu_gate, gate, gate_sig, S); - mlx_array_free(gate); mlx_array_free(gate_sig); + mlx_array_free(gate); + mlx_array_free(gate_sig); mlx_array mid = mlx_array_new(); mlx_multiply(&mid, silu_gate, up, S); - mlx_array_free(silu_gate); mlx_array_free(up); + mlx_array_free(silu_gate); + mlx_array_free(up); mlx_array ffn = linear(mid, &l->down_proj, S); mlx_array_free(mid); mlx_array x3 = mlx_array_new(); mlx_add(&x3, x, ffn, S); - mlx_array_free(x); mlx_array_free(ffn); + mlx_array_free(x); + mlx_array_free(ffn); x = x3; } - if (free_mask) mlx_array_free(local_mask); + if (free_mask) + mlx_array_free(local_mask); return x; } -embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( - embed_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, - int normalize) -{ - if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || - ctx->token_dim <= 0 || !linear_ok(ctx->projection)) +embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize) { + if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 || + !linear_ok(ctx->projection)) return NULL; embed_mlx_ctx_t *base = ctx->base; @@ -938,8 +958,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( mlx_stream S = base->stream; for (int i = 0; i < n_tokens; i++) { if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { - fprintf(stderr, "mlx late: invalid token id %d at position %d\n", - token_ids[i], i); + fprintf(stderr, "mlx late: invalid token id %d at position %d\n", token_ids[i], i); return NULL; } } @@ -949,13 +968,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( return NULL; int *ids_data = (int *)malloc(id_bytes); - if (!ids_data) return NULL; + if (!ids_data) + return NULL; memcpy(ids_data, token_ids, id_bytes); int ids_shape[] = {1, n_tokens}; mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32); free(ids_data); - if (!arr_ok(ids)) return NULL; + if (!arr_ok(ids)) + return NULL; embed_mlx_late_vectors_t *result = NULL; mlx_array x = mlx_array_new(); @@ -965,22 +986,25 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( mlx_take_axis(&x, base->embed_tokens, ids, 0, S); mlx_array_free(ids); - if (!arr_ok(x)) goto cleanup; + if (!arr_ok(x)) + goto cleanup; - x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, - 0, c->n_layers); - if (!arr_ok(x)) goto cleanup; + x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, 0, c->n_layers); + if (!arr_ok(x)) + goto cleanup; x_normed = mlx_array_new(); mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S); mlx_array_free(x); x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; + if (!arr_ok(x_normed)) + goto cleanup; projected = linear(x_normed, &ctx->projection, S); mlx_array_free(x_normed); x_normed = (mlx_array){0}; - if (!arr_ok(projected)) goto cleanup; + if (!arr_ok(projected)) + goto cleanup; if (normalize) { mlx_array sq = mlx_array_new(); @@ -1011,16 +1035,19 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device( mlx_array_free(inv_norm); mlx_array_free(projected); projected = normalized; - if (!arr_ok(projected)) goto cleanup; + if (!arr_ok(projected)) + goto cleanup; } int vec_shape[] = {n_tokens, ctx->token_dim}; reshaped = mlx_array_new(); mlx_reshape(&reshaped, projected, vec_shape, 2, S); - if (!arr_ok(reshaped)) goto cleanup; + if (!arr_ok(reshaped)) + goto cleanup; result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result)); - if (!result) goto cleanup; + if (!result) + goto cleanup; result->owner = ctx; result->vectors = reshaped; result->tokens = n_tokens; @@ -1035,28 +1062,23 @@ cleanup: return result; } -void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) -{ - if (!vecs) return; +void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) { + if (!vecs) + return; mlx_array_free(vecs->vectors); free(vecs); } -int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) -{ +int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) { return vecs ? vecs->tokens : 0; } -int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) -{ +int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) { return vecs ? vecs->dim : 0; } -int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, - float *out_vectors) -{ - if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || - vecs->tokens <= 0 || vecs->dim <= 0) +int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) { + if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0) return -1; size_t values, bytes; @@ -1084,32 +1106,33 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, } embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( - embed_mlx_late_ctx_t *ctx, - const embed_mlx_late_vectors_t *const *items, int count) -{ - if (!ctx || !ctx->base || !items || count <= 0) return NULL; + embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) { + if (!ctx || !ctx->base || !items || count <= 0) + return NULL; int dim = 0; int total = 0; for (int i = 0; i < count; i++) { const embed_mlx_late_vectors_t *v = items[i]; - if (!v || v->owner != ctx || !arr_ok(v->vectors) || - v->tokens <= 0 || v->dim <= 0) + if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0) return NULL; - if (i == 0) dim = v->dim; + if (i == 0) + dim = v->dim; if (v->dim != dim || total > INT_MAX - v->tokens) return NULL; total += v->tokens; } mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts)); - if (!parts) return NULL; + if (!parts) + return NULL; for (int i = 0; i < count; i++) parts[i] = items[i]->vectors; mlx_stream S = ctx->base->stream; mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count); free(parts); - if (!vec.ctx) return NULL; + if (!vec.ctx) + return NULL; mlx_array joined = mlx_array_new(); mlx_concatenate_axis(&joined, vec, 0, S); @@ -1119,8 +1142,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( return NULL; } - embed_mlx_late_vectors_t *out = - (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); if (!out) { mlx_array_free(joined); return NULL; @@ -1132,13 +1154,12 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( return out; } -embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *vecs, - const int *token_indices, int count) -{ - if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || - !arr_ok(vecs->vectors) || !token_indices || count <= 0 || - vecs->tokens <= 0 || vecs->dim <= 0) +embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *vecs, + const int *token_indices, + int count) { + if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) || + !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0) return NULL; for (int i = 0; i < count; i++) { @@ -1150,13 +1171,15 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0) return NULL; int *idx_data = (int *)malloc(idx_bytes); - if (!idx_data) return NULL; + if (!idx_data) + return NULL; memcpy(idx_data, token_indices, idx_bytes); int idx_shape[] = {count}; mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32); free(idx_data); - if (!arr_ok(idx)) return NULL; + if (!arr_ok(idx)) + return NULL; mlx_stream S = ctx->base->stream; mlx_array selected = mlx_array_new(); @@ -1167,8 +1190,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( return NULL; } - embed_mlx_late_vectors_t *out = - (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); if (!out) { mlx_array_free(selected); return NULL; @@ -1180,21 +1202,21 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select( return out; } -int embed_mlx_late_maxsim_batch_device( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *query, - const embed_mlx_late_vectors_t *docs, const int *doc_offsets, - int docs_count, float *scores) -{ - if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || - !scores || query->owner != ctx || docs->owner != ctx || - query->dim <= 0 || query->dim != docs->dim || - query->tokens <= 0 || docs->tokens <= 0 || - !arr_ok(query->vectors) || !arr_ok(docs->vectors) || - doc_offsets[0] != 0) +int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx, + const embed_mlx_late_vectors_t *query, + const embed_mlx_late_vectors_t *docs, + const int *doc_offsets, + int docs_count, + float *scores) { + if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores || + query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim || + query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) || + !arr_ok(docs->vectors) || doc_offsets[0] != 0) return -1; int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths)); - if (!lengths) return -1; + if (!lengths) + return -1; int max_doc_tokens = 0; for (int i = 0; i < docs_count; i++) { if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] || @@ -1204,7 +1226,8 @@ int embed_mlx_late_maxsim_batch_device( } int n = doc_offsets[i + 1] - doc_offsets[i]; lengths[i] = n; - if (n > max_doc_tokens) max_doc_tokens = n; + if (n > max_doc_tokens) + max_doc_tokens = n; } if (max_doc_tokens <= 0) { free(lengths); @@ -1214,8 +1237,7 @@ int embed_mlx_late_maxsim_batch_device( mlx_stream S = ctx->base->stream; int dim = query->dim; int rc = -1; - mlx_array *doc_rows = - (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); + mlx_array *doc_rows = (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows)); if (!doc_rows) { free(lengths); return -1; @@ -1250,95 +1272,114 @@ int embed_mlx_late_maxsim_batch_device( mlx_array padded = (mlx_array){0}; mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S); - if (!arr_ok(doc)) goto cleanup; + if (!arr_ok(doc)) + goto cleanup; if (n < max_doc_tokens) { int axes[] = {0}; int low[] = {0}; int high[] = {max_doc_tokens - n}; padded = mlx_array_new(); - mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, - "constant", S); + mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, "constant", S); mlx_array_free(doc); doc = padded; padded = (mlx_array){0}; - if (!arr_ok(doc)) goto cleanup; + if (!arr_ok(doc)) + goto cleanup; } doc_rows[i] = mlx_array_new(); mlx_expand_dims(&doc_rows[i], doc, 0, S); mlx_array_free(doc); - if (!arr_ok(doc_rows[i])) goto cleanup; + if (!arr_ok(doc_rows[i])) + goto cleanup; } doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count); - if (!doc_vec.ctx) goto cleanup; + if (!doc_vec.ctx) + goto cleanup; doc_batch = mlx_array_new(); mlx_concatenate_axis(&doc_batch, doc_vec, 0, S); - if (!arr_ok(doc_batch)) goto cleanup; + if (!arr_ok(doc_batch)) + goto cleanup; query_3d = mlx_array_new(); mlx_expand_dims(&query_3d, query->vectors, 0, S); - if (!arr_ok(query_3d)) goto cleanup; + if (!arr_ok(query_3d)) + goto cleanup; int q_shape[] = {docs_count, query->tokens, dim}; query_batched = mlx_array_new(); mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S); - if (!arr_ok(query_batched)) goto cleanup; + if (!arr_ok(query_batched)) + goto cleanup; int perm[] = {0, 2, 1}; docs_t = mlx_array_new(); mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S); - if (!arr_ok(docs_t)) goto cleanup; + if (!arr_ok(docs_t)) + goto cleanup; sims = mlx_array_new(); mlx_matmul(&sims, query_batched, docs_t, S); - if (!arr_ok(sims)) goto cleanup; + if (!arr_ok(sims)) + goto cleanup; int len_shape[] = {docs_count}; lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32); - if (!arr_ok(lengths_arr)) goto cleanup; + if (!arr_ok(lengths_arr)) + goto cleanup; positions = mlx_array_new(); mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S); - if (!arr_ok(positions)) goto cleanup; + if (!arr_ok(positions)) + goto cleanup; lengths_2d = mlx_array_new(); mlx_expand_dims(&lengths_2d, lengths_arr, 1, S); - if (!arr_ok(lengths_2d)) goto cleanup; + if (!arr_ok(lengths_2d)) + goto cleanup; positions_2d = mlx_array_new(); mlx_expand_dims(&positions_2d, positions, 0, S); - if (!arr_ok(positions_2d)) goto cleanup; + if (!arr_ok(positions_2d)) + goto cleanup; valid = mlx_array_new(); mlx_less(&valid, positions_2d, lengths_2d, S); - if (!arr_ok(valid)) goto cleanup; + if (!arr_ok(valid)) + goto cleanup; valid_3d = mlx_array_new(); mlx_expand_dims(&valid_3d, valid, 1, S); - if (!arr_ok(valid_3d)) goto cleanup; + if (!arr_ok(valid_3d)) + goto cleanup; masked_sims = mlx_array_new(); mlx_where(&masked_sims, valid_3d, sims, invalid_value, S); - if (!arr_ok(masked_sims)) goto cleanup; + if (!arr_ok(masked_sims)) + goto cleanup; maxes = mlx_array_new(); mlx_max_axis(&maxes, masked_sims, 2, false, S); - if (!arr_ok(maxes)) goto cleanup; + if (!arr_ok(maxes)) + goto cleanup; sums = mlx_array_new(); mlx_sum_axis(&sums, maxes, 1, false, S); - if (!arr_ok(sums)) goto cleanup; + if (!arr_ok(sums)) + goto cleanup; sums_f32 = mlx_array_new(); mlx_astype(&sums_f32, sums, MLX_FLOAT32, S); - if (!arr_ok(sums_f32)) goto cleanup; + if (!arr_ok(sums_f32)) + goto cleanup; mlx_array_eval(sums_f32); const float *data = mlx_array_data_float32(sums_f32); - if (!data) goto cleanup; + if (!data) + goto cleanup; memcpy(scores, data, (size_t)docs_count * sizeof(float)); rc = 0; @@ -1359,7 +1400,8 @@ cleanup: mlx_array_free(query_batched); mlx_array_free(query_3d); mlx_array_free(doc_batch); - if (doc_vec.ctx) mlx_vector_array_free(doc_vec); + if (doc_vec.ctx) + mlx_vector_array_free(doc_vec); mlx_array_free(pad_value); for (int i = 0; i < docs_count; i++) mlx_array_free(doc_rows[i]); @@ -1369,31 +1411,32 @@ cleanup: } int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, - const int *token_ids, int n_tokens, - int normalize, float *out_vectors) -{ + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { embed_mlx_late_vectors_t *vecs = - embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, - normalize); - if (!vecs) return -1; + embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize); + if (!vecs) + return -1; int rc = embed_mlx_late_vectors_copy(vecs, out_vectors); embed_mlx_late_vectors_free(vecs); return rc; } static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, - const embed_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) + const embed_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || + ctx->layer_end != ctx->config.n_layers) return -1; const embed_config_t *c = &ctx->config; mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int max_seq = 0; + int hidden = c->hidden_size; + int max_seq = 0; int has_padding = 0; int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; int last_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN; @@ -1401,13 +1444,17 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, return -1; for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; + if (!inputs[b].ids || inputs[b].n_tokens <= 0) + return -1; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; } - if (max_seq <= 0) return -1; + if (max_seq <= 0) + return -1; for (int b = 0; b < batch; b++) { - if (inputs[b].n_tokens != max_seq) has_padding = 1; + if (inputs[b].n_tokens != max_seq) + has_padding = 1; } int needs_pool_mask = has_padding && !last_token_pool; int needs_attn_mask = has_padding && !causal; @@ -1420,24 +1467,18 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 || mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 || mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 || - (last_token_pool && - mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) + (last_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0)) return -1; if (ids_bytes == 0 || out_bytes == 0 || (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) || - (needs_attn_mask && mask_bytes == 0) || - (last_token_pool && last_index_bytes == 0)) + (needs_attn_mask && mask_bytes == 0) || (last_token_pool && last_index_bytes == 0)) return -1; int *padded_ids = (int *)malloc(ids_bytes); - float *pool_mask_data = - needs_pool_mask ? (float *)malloc(mask_bytes) : NULL; - float *attn_mask_data = - needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; + float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL; + float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL; float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL; - int *last_index_data = - last_token_pool ? (int *)malloc(last_index_bytes) : NULL; - if (!padded_ids || - (needs_pool_mask && (!pool_mask_data || !len_data)) || + int *last_index_data = last_token_pool ? (int *)malloc(last_index_bytes) : NULL; + if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) || (needs_attn_mask && !attn_mask_data)) { free(padded_ids); free(pool_mask_data); @@ -1455,7 +1496,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, } for (int b = 0; b < batch; b++) { - if (needs_pool_mask) len_data[b] = (float)inputs[b].n_tokens; + if (needs_pool_mask) + len_data[b] = (float)inputs[b].n_tokens; if (last_token_pool) { int index = inputs[b].n_tokens - 1; for (int d = 0; d < hidden; d++) @@ -1466,8 +1508,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, if (t < inputs[b].n_tokens) { int id = inputs[b].ids[t]; if (id < 0 || id >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - id, b, t); + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t); free(padded_ids); free(pool_mask_data); free(attn_mask_data); @@ -1498,25 +1539,26 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); mlx_array pool_mask = needs_pool_mask - ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) - : (mlx_array){0}; + ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32) + : (mlx_array){0}; mlx_array attn_mask = needs_attn_mask - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array lengths = needs_pool_mask - ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) - : (mlx_array){0}; - mlx_array last_indices = last_token_pool - ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) - : (mlx_array){0}; - free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data); + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + mlx_array lengths = + needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0}; + mlx_array last_indices = + last_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) + : (mlx_array){0}; + free(padded_ids); + free(pool_mask_data); + free(attn_mask_data); + free(len_data); free(last_index_data); - if (!arr_ok(ids) || - (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) || - (needs_attn_mask && !arr_ok(attn_mask)) || - (last_token_pool && !arr_ok(last_indices))) { - mlx_array_free(ids); mlx_array_free(pool_mask); + if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) || + (needs_attn_mask && !arr_ok(attn_mask)) || (last_token_pool && !arr_ok(last_indices))) { + mlx_array_free(ids); + mlx_array_free(pool_mask); mlx_array_free(attn_mask); mlx_array_free(lengths); mlx_array_free(last_indices); @@ -1528,7 +1570,9 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(attn_mask); attn_mask = attn_mask_bf16; if (!arr_ok(attn_mask)) { - mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths); + mlx_array_free(ids); + mlx_array_free(pool_mask); + mlx_array_free(lengths); mlx_array_free(last_indices); return -1; } @@ -1540,9 +1584,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(ids); /* 2. Transformer layers. */ - x = mlx_forward_layers(ctx, x, batch, max_seq, - needs_attn_mask, attn_mask, - 0, c->n_layers); + x = mlx_forward_layers(ctx, x, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers); /* 3. Final RMSNorm. */ mlx_array x_normed = mlx_array_new(); @@ -1573,7 +1615,8 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(masked); mlx_divide(&emb, emb_sum, lengths, S); - mlx_array_free(emb_sum); mlx_array_free(lengths); + mlx_array_free(emb_sum); + mlx_array_free(lengths); mlx_array_free(attn_mask); } else { /* Equal-length batches do not need masks or a separate divide. */ @@ -1595,8 +1638,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, if (c->normalize_embeddings) { rc = 0; for (int b = 0; b < batch; b++) { - if (embed_l2_normalize( - out_embeddings + (size_t)b * hidden, hidden) != 0) { + if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) { rc = -1; break; } @@ -1614,8 +1656,7 @@ typedef struct { const embed_input_t *input; } mlx_batch_item_t; -static int mlx_batch_item_cmp(const void *a, const void *b) -{ +static int mlx_batch_item_cmp(const void *a, const void *b) { const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a; const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b; int da = ia->input->n_tokens; @@ -1623,27 +1664,33 @@ static int mlx_batch_item_cmp(const void *a, const void *b) return (da > db) - (da < db); } -static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) -{ +static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) { if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5) return 1; return padded * 5 > tokens * 11; } -static int mlx_batch_should_split(const embed_input_t *inputs, int batch, - int *out_total_tokens, int *out_max_seq) -{ +static int mlx_batch_should_split(const embed_input_t *inputs, + int batch, + int *out_total_tokens, + int *out_max_seq) { int max_seq = 0; int total_tokens = 0; for (int b = 0; b < batch; b++) { - if (!inputs[b].ids || inputs[b].n_tokens <= 0) return 0; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - if (total_tokens > INT_MAX - inputs[b].n_tokens) return 0; + if (!inputs[b].ids || inputs[b].n_tokens <= 0) + return 0; + if (inputs[b].n_tokens > max_seq) + max_seq = inputs[b].n_tokens; + if (total_tokens > INT_MAX - inputs[b].n_tokens) + return 0; total_tokens += inputs[b].n_tokens; } - if (out_total_tokens) *out_total_tokens = total_tokens; - if (out_max_seq) *out_max_seq = max_seq; - if (batch < 4 || max_seq < 64 || total_tokens <= 0) return 0; + if (out_total_tokens) + *out_total_tokens = total_tokens; + if (out_max_seq) + *out_max_seq = max_seq; + if (batch < 4 || max_seq < 64 || total_tokens <= 0) + return 0; size_t padded; if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) @@ -1653,9 +1700,9 @@ static int mlx_batch_should_split(const embed_input_t *inputs, int batch, } static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx, - int group_count, int group_max, - int next_tokens) -{ + int group_count, + int group_max, + int next_tokens) { if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0) return 1; @@ -1674,9 +1721,10 @@ static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx, return merged <= split_total; } -int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, - int batch, float *out_embeddings) -{ +int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, + const embed_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; @@ -1684,19 +1732,20 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings); const int hidden = ctx->config.hidden_size; - mlx_batch_item_t *items = (mlx_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - embed_input_t *group_inputs = (embed_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); + mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items)); + embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); size_t group_values, group_bytes; if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 || mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); + free(items); + free(group_inputs); return -1; } float *group_out = (float *)malloc(group_bytes); if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return -1; } for (int i = 0; i < batch; i++) { @@ -1706,7 +1755,7 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp); int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { + for (int start = 0; start < batch && rc == 0;) { int group_max = 0; int group_tokens = 0; int end = start; @@ -1717,12 +1766,9 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, int next_total = group_tokens + next_tokens; if (end > start) { size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 || + mlx_padding_exceeds_limit(next_padded, (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens)) break; } group_max = next_max; @@ -1733,26 +1779,28 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, const embed_input_t *inputs, int group_count = end - start; for (int i = 0; i < group_count; i++) group_inputs[i] = *items[start + i].input; - rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, - group_out); - if (rc != 0) break; + rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out); + if (rc != 0) + break; for (int i = 0; i < group_count; i++) { int dst = items[start + i].index; - memcpy(out_embeddings + (size_t)dst * hidden, - group_out + (size_t)i * hidden, + memcpy(out_embeddings + (size_t)dst * hidden, group_out + (size_t)i * hidden, (size_t)hidden * sizeof(float)); } start = end; } - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return rc; } -int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, float *out_embedding) -{ - embed_input_t input = { token_ids, n_tokens }; +int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding) { + embed_input_t input = {token_ids, n_tokens}; return embed_mlx_encode_batch(ctx, &input, 1, out_embedding); } @@ -1762,8 +1810,7 @@ typedef struct { const embed_context_input_t *input; } mlx_context_batch_item_t; -static int mlx_context_batch_item_cmp(const void *a, const void *b) -{ +static int mlx_context_batch_item_cmp(const void *a, const void *b) { const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a; const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b; int da = ia->input->input.n_tokens; @@ -1771,33 +1818,33 @@ static int mlx_context_batch_item_cmp(const void *a, const void *b) return (da > db) - (da < db); } -static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) -{ +static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) { if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4) return 1; return padded * 4 > tokens * 5; } static int mlx_context_batch_should_split(const embed_context_input_t *inputs, - int batch, int *out_total_spans) -{ + int batch, + int *out_total_spans) { int max_seq = 0; int total_tokens = 0; int total_spans = 0; for (int b = 0; b < batch; b++) { int n_tokens = inputs[b].input.n_tokens; - if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || - inputs[b].n_spans <= 0) + if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || inputs[b].n_spans <= 0) return 0; - if (n_tokens > max_seq) max_seq = n_tokens; - if (total_tokens > INT_MAX - n_tokens || - total_spans > INT_MAX - inputs[b].n_spans) + if (n_tokens > max_seq) + max_seq = n_tokens; + if (total_tokens > INT_MAX - n_tokens || total_spans > INT_MAX - inputs[b].n_spans) return 0; total_tokens += n_tokens; total_spans += inputs[b].n_spans; } - if (out_total_spans) *out_total_spans = total_spans; - if (batch < 2 || max_seq < 64 || total_tokens <= 0) return 0; + if (out_total_spans) + *out_total_spans = total_spans; + if (batch < 2 || max_seq < 64 || total_tokens <= 0) + return 0; size_t padded; if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0) @@ -1806,13 +1853,11 @@ static int mlx_context_batch_should_split(const embed_context_input_t *inputs, } static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, - int batch, float *out_embeddings) -{ - if (!ctx || !inputs || batch <= 0 || !out_embeddings || - ctx->layer_start != 0 || - ctx->layer_end != ctx->config.n_layers || - !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { + if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || + ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) return -1; const embed_config_t *c = &ctx->config; @@ -1824,29 +1869,29 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, for (int b = 0; b < batch; b++) { const embed_context_input_t *input = &inputs[b]; int n_tokens = input->input.n_tokens; - if (!input->input.ids || n_tokens <= 0 || !input->spans || - input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans) + if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || + total_spans > INT_MAX - input->n_spans) return -1; total_spans += input->n_spans; - if (n_tokens > max_seq) max_seq = n_tokens; + if (n_tokens > max_seq) + max_seq = n_tokens; for (int t = 0; t < n_tokens; t++) { - if (input->input.ids[t] < 0 || - input->input.ids[t] >= c->vocab_size) { + if (input->input.ids[t] < 0 || input->input.ids[t] >= c->vocab_size) { fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", input->input.ids[t], b, t); return -1; } } for (int s = 0; s < input->n_spans; s++) { - if (input->spans[s].start < 0 || - input->spans[s].start > n_tokens || + if (input->spans[s].start < 0 || input->spans[s].start > n_tokens || input->spans[s].n_tokens <= 0 || input->spans[s].n_tokens > n_tokens - input->spans[s].start) return -1; } } for (int b = 0; b < batch; b++) - if (inputs[b].input.n_tokens != max_seq) has_padding = 1; + if (inputs[b].input.n_tokens != max_seq) + has_padding = 1; size_t rows, ids_bytes, mask_bytes, index_bytes; size_t span_float_bytes, out_values, out_bytes; @@ -1858,10 +1903,10 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 || mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0) return -1; - if (rows > INT_MAX) return -1; + if (rows > INT_MAX) + return -1; int *padded_ids = (int *)malloc(ids_bytes); - float *attn_mask_data = has_padding - ? (float *)malloc(mask_bytes) : NULL; + float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL; if (!padded_ids || (has_padding && !attn_mask_data)) { free(padded_ids); free(attn_mask_data); @@ -1873,10 +1918,12 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, size_t row = (size_t)b * max_seq + t; if (t < input->n_tokens) { padded_ids[row] = input->ids[t]; - if (has_padding) attn_mask_data[row] = 0.0f; + if (has_padding) + attn_mask_data[row] = 0.0f; } else { padded_ids[row] = 0; - if (has_padding) attn_mask_data[row] = -1.0e9f; + if (has_padding) + attn_mask_data[row] = -1.0e9f; } } } @@ -1908,44 +1955,49 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, int ids_shape[] = {batch, max_seq}; int attn_mask_shape[] = {batch, 1, 1, max_seq}; ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; + attn_mask = has_padding ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; free(padded_ids); free(attn_mask_data); - if (!arr_ok(ids)) goto cleanup; - if (has_padding && !arr_ok(attn_mask)) goto cleanup; + if (!arr_ok(ids)) + goto cleanup; + if (has_padding && !arr_ok(attn_mask)) + goto cleanup; if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { mlx_array attn_mask_bf16 = mlx_array_new(); mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); mlx_array_free(attn_mask); attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) goto cleanup; + if (!arr_ok(attn_mask)) + goto cleanup; } x = mlx_array_new(); mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); mlx_array_free(ids); ids = (mlx_array){0}; - if (!arr_ok(x)) goto cleanup; + if (!arr_ok(x)) + goto cleanup; - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - 0, c->n_layers); + x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers); if (has_padding) { mlx_array_free(attn_mask); attn_mask = (mlx_array){0}; } - if (!arr_ok(x)) goto cleanup; + if (!arr_ok(x)) + goto cleanup; x_normed = mlx_array_new(); mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); mlx_array_free(x); x = (mlx_array){0}; - if (!arr_ok(x_normed)) goto cleanup; + if (!arr_ok(x_normed)) + goto cleanup; if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled)); - if (!pooled) goto cleanup; + if (!pooled) + goto cleanup; int pooled_count = 0; for (int b = 0; b < batch; b++) { @@ -1964,23 +2016,25 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, pooled[pooled_count] = mlx_array_new(); mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S); mlx_array_free(slice); - if (!arr_ok(pooled[pooled_count])) goto cleanup; + if (!arr_ok(pooled[pooled_count])) + goto cleanup; pooled_count++; } } pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans); - if (!pooled_vec.ctx) goto cleanup; + if (!pooled_vec.ctx) + goto cleanup; embeddings = mlx_array_new(); mlx_concatenate_axis(&embeddings, pooled_vec, 0, S); - if (!arr_ok(embeddings)) goto cleanup; + if (!arr_ok(embeddings)) + goto cleanup; } else { end_idx_data = (int *)malloc(index_bytes); start_idx_data = (int *)malloc(index_bytes); lengths_data = (float *)malloc(span_float_bytes); start_mask_data = (float *)malloc(span_float_bytes); - if (!end_idx_data || !start_idx_data || !lengths_data || - !start_mask_data) + if (!end_idx_data || !start_idx_data || !lengths_data || !start_mask_data) goto cleanup; int span_count = 0; @@ -2001,57 +2055,65 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, int index_shape[] = {total_spans}; int scale_shape[] = {total_spans, 1}; end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32); - start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, - MLX_INT32); - lengths = mlx_array_new_data(lengths_data, scale_shape, 2, - MLX_FLOAT32); - start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, - MLX_FLOAT32); - free(end_idx_data); end_idx_data = NULL; - free(start_idx_data); start_idx_data = NULL; - free(lengths_data); lengths_data = NULL; - free(start_mask_data); start_mask_data = NULL; - if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || - !arr_ok(start_mask)) + start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, MLX_INT32); + lengths = mlx_array_new_data(lengths_data, scale_shape, 2, MLX_FLOAT32); + start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, MLX_FLOAT32); + free(end_idx_data); + end_idx_data = NULL; + free(start_idx_data); + start_idx_data = NULL; + free(lengths_data); + lengths_data = NULL; + free(start_mask_data); + start_mask_data = NULL; + if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || !arr_ok(start_mask)) goto cleanup; prefix = mlx_array_new(); mlx_cumsum(&prefix, x_normed, 1, false, true, S); mlx_array_free(x_normed); x_normed = (mlx_array){0}; - if (!arr_ok(prefix)) goto cleanup; + if (!arr_ok(prefix)) + goto cleanup; int flat_shape[] = {(int)rows, hidden}; flat_prefix = mlx_array_new(); mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S); mlx_array_free(prefix); prefix = (mlx_array){0}; - if (!arr_ok(flat_prefix)) goto cleanup; + if (!arr_ok(flat_prefix)) + goto cleanup; end_values = mlx_array_new(); mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S); - if (!arr_ok(end_values)) goto cleanup; + if (!arr_ok(end_values)) + goto cleanup; start_values = mlx_array_new(); mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S); - if (!arr_ok(start_values)) goto cleanup; + if (!arr_ok(start_values)) + goto cleanup; masked_start = mlx_array_new(); mlx_multiply(&masked_start, start_values, start_mask, S); - if (!arr_ok(masked_start)) goto cleanup; + if (!arr_ok(masked_start)) + goto cleanup; span_sums = mlx_array_new(); mlx_subtract(&span_sums, end_values, masked_start, S); - if (!arr_ok(span_sums)) goto cleanup; + if (!arr_ok(span_sums)) + goto cleanup; embeddings = mlx_array_new(); mlx_divide(&embeddings, span_sums, lengths, S); - if (!arr_ok(embeddings)) goto cleanup; + if (!arr_ok(embeddings)) + goto cleanup; } embeddings_f32 = mlx_array_new(); mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S); - if (!arr_ok(embeddings_f32)) goto cleanup; + if (!arr_ok(embeddings_f32)) + goto cleanup; mlx_array_eval(embeddings_f32); const float *data = mlx_array_data_float32(embeddings_f32); @@ -2067,9 +2129,11 @@ cleanup: free(end_idx_data); mlx_array_free(embeddings_f32); mlx_array_free(embeddings); - if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec); + if (pooled_vec.ctx) + mlx_vector_array_free(pooled_vec); if (pooled) { - for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]); + for (int s = 0; s < total_spans; s++) + mlx_array_free(pooled[s]); free(pooled); } mlx_array_free(span_sums); @@ -2090,31 +2154,33 @@ cleanup: } int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, int batch, - float *out_embeddings) -{ + const embed_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; int total_spans = 0; if (!mlx_context_batch_should_split(inputs, batch, &total_spans)) - return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, - out_embeddings); + return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings); const int hidden = ctx->config.hidden_size; - mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc( - (size_t)batch * sizeof(*items)); - embed_context_input_t *group_inputs = (embed_context_input_t *)malloc( - (size_t)batch * sizeof(*group_inputs)); + mlx_context_batch_item_t *items = + (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items)); + embed_context_input_t *group_inputs = + (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); size_t group_values, group_bytes; if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 || mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { - free(items); free(group_inputs); + free(items); + free(group_inputs); return -1; } float *group_out = (float *)malloc(group_bytes); if (!items || !group_inputs || !group_out) { - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return -1; } @@ -2128,7 +2194,7 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp); int rc = 0; - for (int start = 0; start < batch && rc == 0; ) { + for (int start = 0; start < batch && rc == 0;) { int group_max = 0; int group_tokens = 0; int end = start; @@ -2139,12 +2205,9 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, int next_total = group_tokens + next_tokens; if (end > start) { size_t next_padded; - if (mlx_mul_size((size_t)next_count, (size_t)next_max, - &next_padded) != 0 || - mlx_context_padding_exceeds_limit(next_padded, - (size_t)next_total) || - !mlx_should_add_to_group(ctx, end - start, - group_max, next_tokens)) + if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 || + mlx_context_padding_exceeds_limit(next_padded, (size_t)next_total) || + !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens)) break; } group_max = next_max; @@ -2155,47 +2218,51 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, int group_count = end - start; for (int i = 0; i < group_count; i++) group_inputs[i] = *items[start + i].input; - rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, - group_count, group_out); - if (rc != 0) break; + rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out); + if (rc != 0) + break; int src_span = 0; for (int i = 0; i < group_count; i++) { int n_spans = group_inputs[i].n_spans; int dst_span = items[start + i].span_offset; memcpy(out_embeddings + (size_t)dst_span * hidden, - group_out + (size_t)src_span * hidden, - (size_t)n_spans * hidden * sizeof(float)); + group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float)); src_span += n_spans; } start = end; } - free(items); free(group_inputs); free(group_out); + free(items); + free(group_inputs); + free(group_out); return rc; } -int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, const embed_span_t *spans, - int n_spans, float *out_embeddings) -{ +int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + const embed_span_t *spans, + int n_spans, + float *out_embeddings) { embed_context_input_t input = { - .input = { token_ids, n_tokens }, + .input = {token_ids, n_tokens}, .spans = spans, .n_spans = n_spans, }; return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings); } -float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) -{ - if (!ctx || !token_ids || n_tokens <= 0) return NULL; +float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { + if (!ctx || !token_ids || n_tokens <= 0) + return NULL; int hidden = ctx->config.hidden_size; size_t out_bytes; if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0) return NULL; float *out = (float *)malloc(out_bytes); - if (!out) return NULL; + if (!out) + return NULL; if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) { free(out); |
