aboutsummaryrefslogtreecommitdiff
path: root/pplx_embed_mlx.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-03-28 21:27:58 +0100
committerOnur Gündüz <ogunduz@gmail.com>2026-03-28 23:10:03 +0100
commitc47e900b94d18cca1bf600e58f10dbcc2e34c1f7 (patch)
treebf7a4247c528841abe71bbbd408eda3ff71b1152 /pplx_embed_mlx.c
parent34e9eefde7870edcc318209738cabb17c71b50ed (diff)
downloadffwd-c47e900b94d18cca1bf600e58f10dbcc2e34c1f7.tar.gz
ffwd-c47e900b94d18cca1bf600e58f10dbcc2e34c1f7.zip
Refactor and enhance pplx_embed and pplx_embed_mlx for improved model support
- Updated `pplx_embed.h` to support new embedding models: `pplx-embed-v1-4b` and `pplx-embed-context-v1-4b`. - Simplified model configuration structure and constants for better maintainability. - Enhanced comments for clarity on model architecture and configuration. - Modified `pplx_embed_mlx.c` to streamline loading of model weights and configuration from `config.json`. - Improved memory management and error handling in the MLX backend. - Updated embedding computation to support dynamic hidden sizes based on model configuration. - Added `pplx_mlx_config` function to retrieve model configuration details. - Updated README.md - Added .gitignore
Diffstat (limited to 'pplx_embed_mlx.c')
-rw-r--r--pplx_embed_mlx.c478
1 files changed, 208 insertions, 270 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index d6637cd..3c82db1 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -1,21 +1,6 @@
-/*
- * pplx_embed_mlx.c - MLX GPU backend for pplx-embed inference
- *
- * Uses mlx-c (Apple's pure C API) for GPU-accelerated transformer inference.
- * The full forward pass runs on Metal: token embed > 28 layers > mean pool > L2 norm.
- *
- * Key operations used:
- * mlx_take - embedding lookup
- * mlx_matmul - linear projections (no bias)
- * mlx_fast_rms_norm - RMSNorm (fused Metal kernel)
- * mlx_fast_rope - NeoX-style rotary embeddings (fused Metal kernel)
- * mlx_fast_scaled_dot_product_attention - GQA attention (fused Metal kernel)
- * sigmoid * gate, multiply - SwiGLU activation
- */
-
#include "pplx_embed_mlx.h"
-#include "pplx_embed.h" /* config constants */
-#include "qwen_asr_safetensors.h" /* safetensors reader */
+#include "pplx_embed.h"
+#include "qwen_asr_safetensors.h"
#include <mlx/c/mlx.h>
@@ -25,30 +10,22 @@
#include <math.h>
/* ========================================================================
- * Per-layer weight arrays (MLX)
+ * Per-layer MLX weight arrays
* ======================================================================== */
typedef struct {
- mlx_array wq; /* [q_dim, hidden] */
- mlx_array wk; /* [kv_dim, hidden] */
- mlx_array wv; /* [kv_dim, hidden] */
- mlx_array wo; /* [hidden, q_dim] */
- mlx_array q_norm; /* [head_dim] */
- mlx_array k_norm; /* [head_dim] */
- mlx_array input_norm; /* [hidden] */
- mlx_array post_attn_norm; /* [hidden] */
- mlx_array gate_proj; /* [intermediate, hidden] */
- mlx_array up_proj; /* [intermediate, hidden] */
- mlx_array down_proj; /* [hidden, intermediate] */
+ mlx_array wq, wk, wv, wo;
+ mlx_array q_norm, k_norm;
+ mlx_array input_norm, post_attn_norm;
+ mlx_array gate_proj, up_proj, down_proj;
} mlx_layer_t;
struct pplx_mlx_ctx {
- mlx_array embed_tokens; /* [vocab, hidden] */
- mlx_layer_t layers[PPLX_NUM_LAYERS];
- mlx_array norm; /* [hidden] */
+ pplx_config_t config;
+ mlx_array embed_tokens;
+ mlx_layer_t *layers; /* heap-allocated [n_layers] */
+ mlx_array norm;
mlx_stream stream;
-
- /* Safetensors kept alive for mmap */
multi_safetensors_t *ms;
};
@@ -56,13 +33,8 @@ struct pplx_mlx_ctx {
* Helpers
* ======================================================================== */
-/* Default GPU stream */
-static mlx_stream gpu_stream(pplx_mlx_ctx_t *ctx) { return ctx->stream; }
+static int arr_ok(mlx_array a) { return a.ctx != NULL; }
-/*
- * Load an F32 tensor from safetensors into an mlx_array.
- * The data is copied into MLX (which will move it to GPU as needed).
- */
static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
const int *shape, int ndim)
{
@@ -76,82 +48,156 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
fprintf(stderr, "mlx: expected F32 for %s\n", name);
return (mlx_array){0};
}
- const void *data = safetensors_data(sf, t);
- return mlx_array_new_data(data, shape, ndim, MLX_FLOAT32);
+ return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, MLX_FLOAT32);
}
-/* Quick check: is arr valid? */
-static int arr_ok(mlx_array a) { return a.ctx != NULL; }
+/* y = x @ W^T */
+static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s)
+{
+ mlx_array Wt = mlx_array_new();
+ mlx_transpose(&Wt, W, s);
+ mlx_array res = mlx_array_new();
+ mlx_matmul(&res, x, Wt, s);
+ mlx_array_free(Wt);
+ return res;
+}
+
+const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx)
+{
+ return ctx ? &ctx->config : NULL;
+}
/* ========================================================================
- * Load
+ * Load (reuses the same parse_config logic via pplx_load then copies)
* ======================================================================== */
+/* We duplicate a minimal config parser here so MLX can load standalone.
+ * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */
+
+static const char *skip_ws_m(const char *p)
+{
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+ return p;
+}
+
+static const char *json_find_m(const char *json, const char *key)
+{
+ size_t klen = strlen(key);
+ const char *p = json;
+ while ((p = strstr(p, key)) != NULL) {
+ if (p > json && *(p - 1) == '"') {
+ const char *a = p + klen;
+ if (*a == '"') {
+ a = skip_ws_m(a + 1);
+ if (*a == ':') return skip_ws_m(a + 1);
+ }
+ }
+ p += klen;
+ }
+ return NULL;
+}
+
+static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir)
+{
+ char path[1024];
+ snprintf(path, sizeof(path), "%s/config.json", model_dir);
+ FILE *f = fopen(path, "rb");
+ if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; }
+ fseek(f, 0, SEEK_END);
+ long sz = ftell(f);
+ fseek(f, 0, SEEK_SET);
+ char *buf = (char *)malloc(sz + 1);
+ if (!buf || fread(buf, 1, sz, f) != (size_t)sz) { fclose(f); free(buf); return -1; }
+ fclose(f);
+ buf[sz] = '\0';
+
+#define GI(k, fb) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : 0; } while(0)
+#define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0)
+ GI("hidden_size", hidden_size);
+ GI("num_hidden_layers", n_layers);
+ GI("num_attention_heads", n_heads);
+ GI("num_key_value_heads", n_kv_heads);
+ GI("intermediate_size", intermediate_size);
+ cfg->head_dim = PPLX_HEAD_DIM;
+ cfg->vocab_size = PPLX_VOCAB_SIZE;
+ GF("rms_norm_eps", rms_norm_eps, 1e-6f);
+ GF("rope_theta", rope_theta, 1000000.0f);
+#undef GI
+#undef GF
+
+ cfg->q_dim = cfg->n_heads * cfg->head_dim;
+ cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
+ free(buf);
+
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0) {
+ fprintf(stderr, "mlx: bad config (hidden=%d layers=%d)\n",
+ cfg->hidden_size, cfg->n_layers);
+ return -1;
+ }
+ return 0;
+}
+
pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
{
pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t));
if (!ctx) return NULL;
+ if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; }
+
+ const pplx_config_t *c = &ctx->config;
ctx->stream = mlx_default_gpu_stream_new();
- /* Open safetensors */
ctx->ms = multi_safetensors_open(model_dir);
if (!ctx->ms) {
fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
- free(ctx);
- return NULL;
+ free(ctx); return NULL;
}
- multi_safetensors_t *ms = ctx->ms;
+ int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
+ int inter = c->intermediate_size, hd = c->head_dim;
- /* Embedding table: [vocab_size, hidden] */
- int emb_shape[] = {PPLX_VOCAB_SIZE, PPLX_HIDDEN_SIZE};
- ctx->embed_tokens = load_tensor(ms, "embed_tokens.weight", emb_shape, 2);
+ /* Embedding table */
+ int emb_shape[] = {c->vocab_size, h};
+ ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2);
if (!arr_ok(ctx->embed_tokens)) goto fail;
- /* Per-layer weights */
+ /* Layers */
+ ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
+ if (!ctx->layers) goto fail;
+
char name[256];
- for (int i = 0; i < PPLX_NUM_LAYERS; i++) {
+ for (int i = 0; i < c->n_layers; i++) {
mlx_layer_t *l = &ctx->layers[i];
- int hidden = PPLX_HIDDEN_SIZE;
- int q_dim = PPLX_Q_DIM;
- int kv_dim = PPLX_KV_DIM;
- int inter = PPLX_INTERMEDIATE_SIZE;
- int hdim = PPLX_HEAD_DIM;
-#define LOAD(field, fmt, ...) do { \
+#define LD(fld, fmt, ...) do { \
snprintf(name, sizeof(name), fmt, i); \
int sh[] = {__VA_ARGS__}; \
- int nd = sizeof(sh) / sizeof(sh[0]); \
- l->field = load_tensor(ms, name, sh, nd); \
- if (!arr_ok(l->field)) goto fail; \
+ l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \
+ if (!arr_ok(l->fld)) goto fail; \
} while (0)
- LOAD(wq, "layers.%d.self_attn.q_proj.weight", q_dim, hidden);
- LOAD(wk, "layers.%d.self_attn.k_proj.weight", kv_dim, hidden);
- LOAD(wv, "layers.%d.self_attn.v_proj.weight", kv_dim, hidden);
- LOAD(wo, "layers.%d.self_attn.o_proj.weight", hidden, q_dim);
- LOAD(q_norm, "layers.%d.self_attn.q_norm.weight", hdim);
- LOAD(k_norm, "layers.%d.self_attn.k_norm.weight", hdim);
- LOAD(input_norm, "layers.%d.input_layernorm.weight", hidden);
- LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", hidden);
- LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, hidden);
- LOAD(up_proj, "layers.%d.mlp.up_proj.weight", inter, hidden);
- LOAD(down_proj, "layers.%d.mlp.down_proj.weight", hidden, inter);
+ LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h);
+ LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h);
+ LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h);
+ LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd);
+ LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd);
+ LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd);
+ LD(input_norm, "layers.%d.input_layernorm.weight", h);
+ LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h);
+ LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h);
+ LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h);
+ LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter);
+#undef LD
-#undef LOAD
-
- if (pplx_verbose >= 2)
- fprintf(stderr, " mlx layer %d loaded\n", i);
+ if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i);
}
- /* Final norm */
- int norm_shape[] = {PPLX_HIDDEN_SIZE};
- ctx->norm = load_tensor(ms, "norm.weight", norm_shape, 1);
+ int ns[] = {h};
+ ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1);
if (!arr_ok(ctx->norm)) goto fail;
if (pplx_verbose >= 1)
- fprintf(stderr, "mlx: all weights loaded\n");
+ fprintf(stderr, "mlx: %d layers loaded (%d-dim)\n", c->n_layers, h);
return ctx;
@@ -164,18 +210,13 @@ fail:
* Free
* ======================================================================== */
-static void free_layer(mlx_layer_t *l)
+static void free_mlx_layer(mlx_layer_t *l)
{
- mlx_array_free(l->wq);
- mlx_array_free(l->wk);
- mlx_array_free(l->wv);
- mlx_array_free(l->wo);
- mlx_array_free(l->q_norm);
- mlx_array_free(l->k_norm);
- mlx_array_free(l->input_norm);
- mlx_array_free(l->post_attn_norm);
- mlx_array_free(l->gate_proj);
- mlx_array_free(l->up_proj);
+ mlx_array_free(l->wq); mlx_array_free(l->wk);
+ mlx_array_free(l->wv); mlx_array_free(l->wo);
+ mlx_array_free(l->q_norm); mlx_array_free(l->k_norm);
+ mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm);
+ mlx_array_free(l->gate_proj); mlx_array_free(l->up_proj);
mlx_array_free(l->down_proj);
}
@@ -183,8 +224,11 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
{
if (!ctx) return;
mlx_array_free(ctx->embed_tokens);
- for (int i = 0; i < PPLX_NUM_LAYERS; i++)
- free_layer(&ctx->layers[i]);
+ if (ctx->layers) {
+ for (int i = 0; i < ctx->config.n_layers; i++)
+ free_mlx_layer(&ctx->layers[i]);
+ free(ctx->layers);
+ }
mlx_array_free(ctx->norm);
mlx_stream_free(ctx->stream);
if (ctx->ms) multi_safetensors_close(ctx->ms);
@@ -192,274 +236,168 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
}
/* ========================================================================
- * Forward pass (all on GPU stream, lazy-evaluated)
- *
- * Notation: S = gpu_stream(ctx)
- *
- * Linear without bias: y = x @ W^T
- * Since mlx_matmul(x, W^T) requires transposing W, and our weights
- * are stored as [out, in], we do: y = x @ W^T via
- * Wt = transpose(W) [in, out]
- * y = matmul(x, Wt) [seq, out]
- *
- * Alternatively: y = matmul(x, transpose(W))
- * All MLX ops are lazy (graph-built) so transpose is free.
+ * Forward pass
* ======================================================================== */
-/* Convenience: transpose a 2D matrix */
-static mlx_array t2d(mlx_array a, mlx_stream s)
-{
- mlx_array res = mlx_array_new();
- mlx_transpose(&res, a, s);
- return res;
-}
-
-/* y = x @ W^T (linear, no bias) */
-static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s)
-{
- mlx_array Wt = t2d(W, s);
- mlx_array res = mlx_array_new();
- mlx_matmul(&res, x, Wt, s);
- mlx_array_free(Wt);
- return res;
-}
-
float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
{
if (!ctx || !token_ids || n_tokens <= 0) return NULL;
- mlx_stream S = gpu_stream(ctx);
- int hidden = PPLX_HIDDEN_SIZE;
- int n_heads = PPLX_NUM_HEADS;
- int n_kv_heads = PPLX_NUM_KV_HEADS;
- int head_dim = PPLX_HEAD_DIM;
- int q_dim = PPLX_Q_DIM;
- int seq = n_tokens;
- float scale = 1.0f / sqrtf((float)head_dim);
+ const pplx_config_t *c = &ctx->config;
+ mlx_stream S = ctx->stream;
- (void)n_heads; (void)n_kv_heads; /* used only in shape literals */
+ int hidden = c->hidden_size;
+ int n_heads = c->n_heads;
+ int n_kv_heads = c->n_kv_heads;
+ int head_dim = c->head_dim;
+ int q_dim = c->q_dim;
+ int seq = n_tokens;
+ float scale = 1.0f / sqrtf((float)head_dim);
- /* ------------------------------------------------------------------
- * 1. Token embedding lookup: x = embed_tokens[token_ids]
- * ------------------------------------------------------------------ */
+ /* 1. Embedding lookup */
int ids_shape[] = {seq};
mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32);
mlx_array x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, /*axis=*/0, S);
+ mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
mlx_array_free(ids);
- /* x: [seq, hidden] */
- /* ------------------------------------------------------------------
- * 2. Transformer layers (28x)
- * ------------------------------------------------------------------ */
- for (int layer = 0; layer < PPLX_NUM_LAYERS; layer++) {
+ /* 2. Transformer layers */
+ for (int layer = 0; layer < c->n_layers; layer++) {
mlx_layer_t *l = &ctx->layers[layer];
- /* ---- 2a. Input RMSNorm ---- */
+ /* Input RMSNorm */
mlx_array xn = mlx_array_new();
- mlx_fast_rms_norm(&xn, x, l->input_norm, 1e-6f, S);
+ mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S);
- /* ---- 2b. QKV projections (no bias) ----
- * q: [seq, q_dim] k: [seq, kv_dim] v: [seq, kv_dim]
- */
+ /* QKV projections */
mlx_array q_flat = linear(xn, l->wq, S);
mlx_array k_flat = linear(xn, l->wk, S);
mlx_array v_flat = linear(xn, l->wv, S);
mlx_array_free(xn);
- /* ---- 2c. Reshape to [seq, n_heads, head_dim] for per-head ops ---- */
+ /* Reshape to [seq, n_heads, head_dim] */
int q_shape[] = {seq, n_heads, head_dim};
int k_shape[] = {seq, n_kv_heads, head_dim};
-
mlx_array q = mlx_array_new();
mlx_array k = mlx_array_new();
mlx_array v = mlx_array_new();
mlx_reshape(&q, q_flat, q_shape, 3, S);
mlx_reshape(&k, k_flat, k_shape, 3, S);
mlx_reshape(&v, v_flat, k_shape, 3, S);
- mlx_array_free(q_flat);
- mlx_array_free(k_flat);
- mlx_array_free(v_flat);
+ mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat);
- /* ---- 2d. Per-head Q/K RMSNorm ----
- * q: [seq, n_heads, head_dim] ; norm over last dim (head_dim)
- */
+ /* Per-head Q/K RMSNorm */
mlx_array qn = mlx_array_new();
mlx_array kn = mlx_array_new();
- mlx_fast_rms_norm(&qn, q, l->q_norm, 1e-6f, S);
- mlx_fast_rms_norm(&kn, k, l->k_norm, 1e-6f, S);
- mlx_array_free(q);
- mlx_array_free(k);
+ mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
+ mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
+ mlx_array_free(q); mlx_array_free(k);
- /* ---- 2e. Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA ----
- * MLX rope expects (..., T, D) where T is second-to-last.
- * [seq, n_heads, head_dim]
- * - expand_dims(0) [1, seq, n_heads, head_dim]
- * - transpose [0,2,1,3] [1, n_heads, seq, head_dim]
- */
+ /* Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA */
int perm[] = {0, 2, 1, 3};
-
- mlx_array qe = mlx_array_new();
- mlx_array ke = mlx_array_new();
- mlx_array ve = mlx_array_new();
+ mlx_array qe = mlx_array_new(), ke = mlx_array_new(), ve = mlx_array_new();
mlx_expand_dims(&qe, qn, 0, S);
mlx_expand_dims(&ke, kn, 0, S);
mlx_expand_dims(&ve, v, 0, S);
- mlx_array_free(qn);
- mlx_array_free(kn);
+ mlx_array_free(qn); mlx_array_free(kn);
- mlx_array qt = mlx_array_new();
- mlx_array kt = mlx_array_new();
- mlx_array vt = mlx_array_new();
+ mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
mlx_transpose_axes(&qt, qe, perm, 4, S);
mlx_transpose_axes(&kt, ke, perm, 4, S);
mlx_transpose_axes(&vt, ve, perm, 4, S);
- mlx_array_free(qe);
- mlx_array_free(ke);
- mlx_array_free(ve);
+ mlx_array_free(qe); mlx_array_free(ke); mlx_array_free(ve);
mlx_array_free(v);
- /* ---- 2f. RoPE (NeoX-style: traditional=false) ----
- * qt: [1, n_heads, seq, head_dim]
- * kt: [1, n_kv_heads, seq, head_dim]
- * MLX rope: T = second-to-last dim = seq
- */
- mlx_optional_float base = {.value = 1000000.0f, .has_value = true};
- mlx_array null_freqs = (mlx_array){0};
+ /* RoPE (NeoX: traditional=false) */
+ mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
+ mlx_array null_arr = (mlx_array){0};
+ mlx_array qr = mlx_array_new(), kr = mlx_array_new();
+ mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
+ mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
+ mlx_array_free(qt); mlx_array_free(kt);
- mlx_array qr = mlx_array_new();
- mlx_array kr = mlx_array_new();
- mlx_fast_rope(&qr, qt, head_dim, /*traditional=*/false,
- base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
- mlx_fast_rope(&kr, kt, head_dim, /*traditional=*/false,
- base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
- mlx_array_free(qt);
- mlx_array_free(kt);
-
- /* ---- 2g. Bidirectional GQA Attention ----
- * qr: [1, n_heads, seq, head_dim]
- * kr: [1, n_kv_heads, seq, head_dim]
- * vt: [1, n_kv_heads, seq, head_dim]
- * mask_mode="" > no causal mask > full bidirectional
- * GQA: SDPA broadcasts KV heads to match Q heads automatically.
- */
+ /* Bidirectional SDPA (mask_mode="" = no causal mask, GQA via broadcast) */
mlx_array attn = mlx_array_new();
- mlx_array null_mask = (mlx_array){0};
- mlx_array null_sinks = (mlx_array){0};
mlx_fast_scaled_dot_product_attention(
- &attn, qr, kr, vt, scale,
- /*mask_mode=*/"",
- null_mask, null_sinks, S);
- mlx_array_free(qr);
- mlx_array_free(kr);
- mlx_array_free(vt);
- /* attn: [1, n_heads, seq, head_dim] */
+ &attn, qr, kr, vt, scale, "", null_arr, null_arr, S);
+ mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
- /* ---- 2h. Transpose back > [1, seq, n_heads, head_dim] > squeeze > reshape [seq, q_dim] ---- */
+ /* Transpose back and reshape to [seq, q_dim] */
mlx_array attn_t = mlx_array_new();
- mlx_transpose_axes(&attn_t, attn, perm, 4, S); /* [1, seq, n_heads, head_dim] */
+ mlx_transpose_axes(&attn_t, attn, perm, 4, S);
mlx_array_free(attn);
mlx_array attn_sq = mlx_array_new();
- mlx_squeeze_axis(&attn_sq, attn_t, 0, S); /* [seq, n_heads, head_dim] */
+ mlx_squeeze_axis(&attn_sq, attn_t, 0, S);
mlx_array_free(attn_t);
- int attn_flat_shape[] = {seq, q_dim};
+ int flat_shape[] = {seq, q_dim};
mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_sq, attn_flat_shape, 2, S);
+ mlx_reshape(&attn_flat, attn_sq, flat_shape, 2, S);
mlx_array_free(attn_sq);
- /* ---- 2i. Output projection + residual ---- */
+ /* Output projection + residual */
mlx_array proj = linear(attn_flat, l->wo, S);
mlx_array_free(attn_flat);
-
mlx_array x2 = mlx_array_new();
mlx_add(&x2, x, proj, S);
- mlx_array_free(x);
- mlx_array_free(proj);
+ mlx_array_free(x); mlx_array_free(proj);
x = x2;
- /* ---- 2j. Post-attention RMSNorm ---- */
+ /* Post-attention RMSNorm */
mlx_array xn2 = mlx_array_new();
- mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, 1e-6f, S);
+ mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
- /* ---- 2k. SwiGLU MLP ----
- * gate = x_norm @ gate_proj^T [seq, inter]
- * up = x_norm @ up_proj^T [seq, inter]
- * mid = silu(gate) * up = (gate * sigmoid(gate)) * up
- * out = mid @ down_proj^T [seq, hidden]
- */
+ /* SwiGLU MLP */
mlx_array gate = linear(xn2, l->gate_proj, S);
mlx_array up = linear(xn2, l->up_proj, S);
mlx_array_free(xn2);
- /* SiLU(gate) = gate * sigmoid(gate) */
mlx_array gate_sig = mlx_array_new();
mlx_sigmoid(&gate_sig, gate, S);
mlx_array silu_gate = mlx_array_new();
mlx_multiply(&silu_gate, gate, gate_sig, S);
- mlx_array_free(gate);
- mlx_array_free(gate_sig);
+ mlx_array_free(gate); mlx_array_free(gate_sig);
- /* mid = silu_gate * up */
mlx_array mid = mlx_array_new();
mlx_multiply(&mid, silu_gate, up, S);
- mlx_array_free(silu_gate);
- mlx_array_free(up);
+ mlx_array_free(silu_gate); mlx_array_free(up);
- /* down projection */
mlx_array ffn = linear(mid, l->down_proj, S);
mlx_array_free(mid);
- /* Residual */
mlx_array x3 = mlx_array_new();
mlx_add(&x3, x, ffn, S);
- mlx_array_free(x);
- mlx_array_free(ffn);
+ mlx_array_free(x); mlx_array_free(ffn);
x = x3;
}
- /* ------------------------------------------------------------------
- * 3. Final RMSNorm
- * ------------------------------------------------------------------ */
+ /* 3. Final RMSNorm */
mlx_array x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, 1e-6f, S);
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
mlx_array_free(x);
- /* ------------------------------------------------------------------
- * 4. Mean pooling: mean(x, axis=0) [hidden]
- * ------------------------------------------------------------------ */
+ /* 4. Mean pool over seq dim */
mlx_array emb = mlx_array_new();
- mlx_mean_axis(&emb, x_normed, /*axis=*/0, /*keepdims=*/false, S);
+ mlx_mean_axis(&emb, x_normed, 0, false, S);
mlx_array_free(x_normed);
- /* ------------------------------------------------------------------
- * 5. L2 normalization: emb / ||emb||_2
- * ------------------------------------------------------------------ */
+ /* 5. L2 normalize */
int norm_axes[] = {0};
- mlx_array norm_val = mlx_array_new();
- mlx_linalg_norm_l2(&norm_val, emb, norm_axes, 1, /*keepdims=*/true, S);
-
- mlx_array emb_normed = mlx_array_new();
- mlx_divide(&emb_normed, emb, norm_val, S);
- mlx_array_free(emb);
- mlx_array_free(norm_val);
+ mlx_array nv = mlx_array_new();
+ mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S);
+ mlx_array emb_n = mlx_array_new();
+ mlx_divide(&emb_n, emb, nv, S);
+ mlx_array_free(emb); mlx_array_free(nv);
- /* ------------------------------------------------------------------
- * 6. Evaluate and copy result to CPU
- * ------------------------------------------------------------------ */
- mlx_array_eval(emb_normed);
-
- const float *data = mlx_array_data_float32(emb_normed);
- if (!data) {
- fprintf(stderr, "mlx: failed to get embedding data\n");
- mlx_array_free(emb_normed);
- return NULL;
+ /* 6. Eval and copy to CPU */
+ mlx_array_eval(emb_n);
+ const float *data = mlx_array_data_float32(emb_n);
+ float *out = NULL;
+ if (data) {
+ out = (float *)malloc(hidden * sizeof(float));
+ if (out) memcpy(out, data, hidden * sizeof(float));
}
-
- float *out = (float *)malloc(hidden * sizeof(float));
- if (out) memcpy(out, data, hidden * sizeof(float));
-
- mlx_array_free(emb_normed);
+ mlx_array_free(emb_n);
return out;
}