aboutsummaryrefslogtreecommitdiff
path: root/pplx_embed_mlx.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-03-28 20:01:21 +0100
committerOnur Gündüz <ogunduz@gmail.com>2026-03-28 20:01:21 +0100
commit34e9eefde7870edcc318209738cabb17c71b50ed (patch)
tree470923a965c647de8a827ffbeea1e87a732360a7 /pplx_embed_mlx.c
parent0128ed8d9d7e289b9b6aae752d7e820f36bd26e1 (diff)
downloadffwd-34e9eefde7870edcc318209738cabb17c71b50ed.tar.gz
ffwd-34e9eefde7870edcc318209738cabb17c71b50ed.zip
Init
Diffstat (limited to 'pplx_embed_mlx.c')
-rw-r--r--pplx_embed_mlx.c465
1 files changed, 465 insertions, 0 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
new file mode 100644
index 0000000..d6637cd
--- /dev/null
+++ b/pplx_embed_mlx.c
@@ -0,0 +1,465 @@
+/*
+ * pplx_embed_mlx.c - MLX GPU backend for pplx-embed inference
+ *
+ * Uses mlx-c (Apple's pure C API) for GPU-accelerated transformer inference.
+ * The full forward pass runs on Metal: token embed > 28 layers > mean pool > L2 norm.
+ *
+ * Key operations used:
+ * mlx_take - embedding lookup
+ * mlx_matmul - linear projections (no bias)
+ * mlx_fast_rms_norm - RMSNorm (fused Metal kernel)
+ * mlx_fast_rope - NeoX-style rotary embeddings (fused Metal kernel)
+ * mlx_fast_scaled_dot_product_attention - GQA attention (fused Metal kernel)
+ * sigmoid * gate, multiply - SwiGLU activation
+ */
+
+#include "pplx_embed_mlx.h"
+#include "pplx_embed.h" /* config constants */
+#include "qwen_asr_safetensors.h" /* safetensors reader */
+
+#include <mlx/c/mlx.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+/* ========================================================================
+ * Per-layer weight arrays (MLX)
+ * ======================================================================== */
+
+typedef struct {
+ mlx_array wq; /* [q_dim, hidden] */
+ mlx_array wk; /* [kv_dim, hidden] */
+ mlx_array wv; /* [kv_dim, hidden] */
+ mlx_array wo; /* [hidden, q_dim] */
+ mlx_array q_norm; /* [head_dim] */
+ mlx_array k_norm; /* [head_dim] */
+ mlx_array input_norm; /* [hidden] */
+ mlx_array post_attn_norm; /* [hidden] */
+ mlx_array gate_proj; /* [intermediate, hidden] */
+ mlx_array up_proj; /* [intermediate, hidden] */
+ mlx_array down_proj; /* [hidden, intermediate] */
+} mlx_layer_t;
+
+struct pplx_mlx_ctx {
+ mlx_array embed_tokens; /* [vocab, hidden] */
+ mlx_layer_t layers[PPLX_NUM_LAYERS];
+ mlx_array norm; /* [hidden] */
+ mlx_stream stream;
+
+ /* Safetensors kept alive for mmap */
+ multi_safetensors_t *ms;
+};
+
+/* ========================================================================
+ * Helpers
+ * ======================================================================== */
+
+/* Default GPU stream */
+static mlx_stream gpu_stream(pplx_mlx_ctx_t *ctx) { return ctx->stream; }
+
+/*
+ * Load an F32 tensor from safetensors into an mlx_array.
+ * The data is copied into MLX (which will move it to GPU as needed).
+ */
+static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
+ const int *shape, int ndim)
+{
+ safetensors_file_t *sf = NULL;
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ if (!t) {
+ fprintf(stderr, "mlx: tensor not found: %s\n", name);
+ return (mlx_array){0};
+ }
+ if (t->dtype != DTYPE_F32) {
+ fprintf(stderr, "mlx: expected F32 for %s\n", name);
+ return (mlx_array){0};
+ }
+ const void *data = safetensors_data(sf, t);
+ return mlx_array_new_data(data, shape, ndim, MLX_FLOAT32);
+}
+
+/* Quick check: is arr valid? */
+static int arr_ok(mlx_array a) { return a.ctx != NULL; }
+
+/* ========================================================================
+ * Load
+ * ======================================================================== */
+
+pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
+{
+ pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t));
+ if (!ctx) return NULL;
+
+ ctx->stream = mlx_default_gpu_stream_new();
+
+ /* Open safetensors */
+ ctx->ms = multi_safetensors_open(model_dir);
+ if (!ctx->ms) {
+ fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
+ free(ctx);
+ return NULL;
+ }
+
+ multi_safetensors_t *ms = ctx->ms;
+
+ /* Embedding table: [vocab_size, hidden] */
+ int emb_shape[] = {PPLX_VOCAB_SIZE, PPLX_HIDDEN_SIZE};
+ ctx->embed_tokens = load_tensor(ms, "embed_tokens.weight", emb_shape, 2);
+ if (!arr_ok(ctx->embed_tokens)) goto fail;
+
+ /* Per-layer weights */
+ char name[256];
+ for (int i = 0; i < PPLX_NUM_LAYERS; i++) {
+ mlx_layer_t *l = &ctx->layers[i];
+ int hidden = PPLX_HIDDEN_SIZE;
+ int q_dim = PPLX_Q_DIM;
+ int kv_dim = PPLX_KV_DIM;
+ int inter = PPLX_INTERMEDIATE_SIZE;
+ int hdim = PPLX_HEAD_DIM;
+
+#define LOAD(field, fmt, ...) do { \
+ snprintf(name, sizeof(name), fmt, i); \
+ int sh[] = {__VA_ARGS__}; \
+ int nd = sizeof(sh) / sizeof(sh[0]); \
+ l->field = load_tensor(ms, name, sh, nd); \
+ if (!arr_ok(l->field)) goto fail; \
+} while (0)
+
+ LOAD(wq, "layers.%d.self_attn.q_proj.weight", q_dim, hidden);
+ LOAD(wk, "layers.%d.self_attn.k_proj.weight", kv_dim, hidden);
+ LOAD(wv, "layers.%d.self_attn.v_proj.weight", kv_dim, hidden);
+ LOAD(wo, "layers.%d.self_attn.o_proj.weight", hidden, q_dim);
+ LOAD(q_norm, "layers.%d.self_attn.q_norm.weight", hdim);
+ LOAD(k_norm, "layers.%d.self_attn.k_norm.weight", hdim);
+ LOAD(input_norm, "layers.%d.input_layernorm.weight", hidden);
+ LOAD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", hidden);
+ LOAD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, hidden);
+ LOAD(up_proj, "layers.%d.mlp.up_proj.weight", inter, hidden);
+ LOAD(down_proj, "layers.%d.mlp.down_proj.weight", hidden, inter);
+
+#undef LOAD
+
+ if (pplx_verbose >= 2)
+ fprintf(stderr, " mlx layer %d loaded\n", i);
+ }
+
+ /* Final norm */
+ int norm_shape[] = {PPLX_HIDDEN_SIZE};
+ ctx->norm = load_tensor(ms, "norm.weight", norm_shape, 1);
+ if (!arr_ok(ctx->norm)) goto fail;
+
+ if (pplx_verbose >= 1)
+ fprintf(stderr, "mlx: all weights loaded\n");
+
+ return ctx;
+
+fail:
+ pplx_mlx_free(ctx);
+ return NULL;
+}
+
+/* ========================================================================
+ * Free
+ * ======================================================================== */
+
+static void free_layer(mlx_layer_t *l)
+{
+ mlx_array_free(l->wq);
+ mlx_array_free(l->wk);
+ mlx_array_free(l->wv);
+ mlx_array_free(l->wo);
+ mlx_array_free(l->q_norm);
+ mlx_array_free(l->k_norm);
+ mlx_array_free(l->input_norm);
+ mlx_array_free(l->post_attn_norm);
+ mlx_array_free(l->gate_proj);
+ mlx_array_free(l->up_proj);
+ mlx_array_free(l->down_proj);
+}
+
+void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
+{
+ if (!ctx) return;
+ mlx_array_free(ctx->embed_tokens);
+ for (int i = 0; i < PPLX_NUM_LAYERS; i++)
+ free_layer(&ctx->layers[i]);
+ mlx_array_free(ctx->norm);
+ mlx_stream_free(ctx->stream);
+ if (ctx->ms) multi_safetensors_close(ctx->ms);
+ free(ctx);
+}
+
+/* ========================================================================
+ * Forward pass (all on GPU stream, lazy-evaluated)
+ *
+ * Notation: S = gpu_stream(ctx)
+ *
+ * Linear without bias: y = x @ W^T
+ * Since mlx_matmul(x, W^T) requires transposing W, and our weights
+ * are stored as [out, in], we do: y = x @ W^T via
+ * Wt = transpose(W) [in, out]
+ * y = matmul(x, Wt) [seq, out]
+ *
+ * Alternatively: y = matmul(x, transpose(W))
+ * All MLX ops are lazy (graph-built) so transpose is free.
+ * ======================================================================== */
+
+/* Convenience: transpose a 2D matrix */
+static mlx_array t2d(mlx_array a, mlx_stream s)
+{
+ mlx_array res = mlx_array_new();
+ mlx_transpose(&res, a, s);
+ return res;
+}
+
+/* y = x @ W^T (linear, no bias) */
+static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s)
+{
+ mlx_array Wt = t2d(W, s);
+ mlx_array res = mlx_array_new();
+ mlx_matmul(&res, x, Wt, s);
+ mlx_array_free(Wt);
+ return res;
+}
+
+float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
+{
+ if (!ctx || !token_ids || n_tokens <= 0) return NULL;
+
+ mlx_stream S = gpu_stream(ctx);
+ int hidden = PPLX_HIDDEN_SIZE;
+ int n_heads = PPLX_NUM_HEADS;
+ int n_kv_heads = PPLX_NUM_KV_HEADS;
+ int head_dim = PPLX_HEAD_DIM;
+ int q_dim = PPLX_Q_DIM;
+ int seq = n_tokens;
+ float scale = 1.0f / sqrtf((float)head_dim);
+
+ (void)n_heads; (void)n_kv_heads; /* used only in shape literals */
+
+ /* ------------------------------------------------------------------
+ * 1. Token embedding lookup: x = embed_tokens[token_ids]
+ * ------------------------------------------------------------------ */
+ int ids_shape[] = {seq};
+ mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32);
+ mlx_array x = mlx_array_new();
+ mlx_take_axis(&x, ctx->embed_tokens, ids, /*axis=*/0, S);
+ mlx_array_free(ids);
+ /* x: [seq, hidden] */
+
+ /* ------------------------------------------------------------------
+ * 2. Transformer layers (28x)
+ * ------------------------------------------------------------------ */
+ for (int layer = 0; layer < PPLX_NUM_LAYERS; layer++) {
+ mlx_layer_t *l = &ctx->layers[layer];
+
+ /* ---- 2a. Input RMSNorm ---- */
+ mlx_array xn = mlx_array_new();
+ mlx_fast_rms_norm(&xn, x, l->input_norm, 1e-6f, S);
+
+ /* ---- 2b. QKV projections (no bias) ----
+ * q: [seq, q_dim] k: [seq, kv_dim] v: [seq, kv_dim]
+ */
+ mlx_array q_flat = linear(xn, l->wq, S);
+ mlx_array k_flat = linear(xn, l->wk, S);
+ mlx_array v_flat = linear(xn, l->wv, S);
+ mlx_array_free(xn);
+
+ /* ---- 2c. Reshape to [seq, n_heads, head_dim] for per-head ops ---- */
+ int q_shape[] = {seq, n_heads, head_dim};
+ int k_shape[] = {seq, n_kv_heads, head_dim};
+
+ mlx_array q = mlx_array_new();
+ mlx_array k = mlx_array_new();
+ mlx_array v = mlx_array_new();
+ mlx_reshape(&q, q_flat, q_shape, 3, S);
+ mlx_reshape(&k, k_flat, k_shape, 3, S);
+ mlx_reshape(&v, v_flat, k_shape, 3, S);
+ mlx_array_free(q_flat);
+ mlx_array_free(k_flat);
+ mlx_array_free(v_flat);
+
+ /* ---- 2d. Per-head Q/K RMSNorm ----
+ * q: [seq, n_heads, head_dim] ; norm over last dim (head_dim)
+ */
+ mlx_array qn = mlx_array_new();
+ mlx_array kn = mlx_array_new();
+ mlx_fast_rms_norm(&qn, q, l->q_norm, 1e-6f, S);
+ mlx_fast_rms_norm(&kn, k, l->k_norm, 1e-6f, S);
+ mlx_array_free(q);
+ mlx_array_free(k);
+
+ /* ---- 2e. Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA ----
+ * MLX rope expects (..., T, D) where T is second-to-last.
+ * [seq, n_heads, head_dim]
+ * - expand_dims(0) [1, seq, n_heads, head_dim]
+ * - transpose [0,2,1,3] [1, n_heads, seq, head_dim]
+ */
+ int perm[] = {0, 2, 1, 3};
+
+ mlx_array qe = mlx_array_new();
+ mlx_array ke = mlx_array_new();
+ mlx_array ve = mlx_array_new();
+ mlx_expand_dims(&qe, qn, 0, S);
+ mlx_expand_dims(&ke, kn, 0, S);
+ mlx_expand_dims(&ve, v, 0, S);
+ mlx_array_free(qn);
+ mlx_array_free(kn);
+
+ mlx_array qt = mlx_array_new();
+ mlx_array kt = mlx_array_new();
+ mlx_array vt = mlx_array_new();
+ mlx_transpose_axes(&qt, qe, perm, 4, S);
+ mlx_transpose_axes(&kt, ke, perm, 4, S);
+ mlx_transpose_axes(&vt, ve, perm, 4, S);
+ mlx_array_free(qe);
+ mlx_array_free(ke);
+ mlx_array_free(ve);
+ mlx_array_free(v);
+
+ /* ---- 2f. RoPE (NeoX-style: traditional=false) ----
+ * qt: [1, n_heads, seq, head_dim]
+ * kt: [1, n_kv_heads, seq, head_dim]
+ * MLX rope: T = second-to-last dim = seq
+ */
+ mlx_optional_float base = {.value = 1000000.0f, .has_value = true};
+ mlx_array null_freqs = (mlx_array){0};
+
+ mlx_array qr = mlx_array_new();
+ mlx_array kr = mlx_array_new();
+ mlx_fast_rope(&qr, qt, head_dim, /*traditional=*/false,
+ base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
+ mlx_fast_rope(&kr, kt, head_dim, /*traditional=*/false,
+ base, /*scale=*/1.0f, /*offset=*/0, null_freqs, S);
+ mlx_array_free(qt);
+ mlx_array_free(kt);
+
+ /* ---- 2g. Bidirectional GQA Attention ----
+ * qr: [1, n_heads, seq, head_dim]
+ * kr: [1, n_kv_heads, seq, head_dim]
+ * vt: [1, n_kv_heads, seq, head_dim]
+ * mask_mode="" > no causal mask > full bidirectional
+ * GQA: SDPA broadcasts KV heads to match Q heads automatically.
+ */
+ mlx_array attn = mlx_array_new();
+ mlx_array null_mask = (mlx_array){0};
+ mlx_array null_sinks = (mlx_array){0};
+ mlx_fast_scaled_dot_product_attention(
+ &attn, qr, kr, vt, scale,
+ /*mask_mode=*/"",
+ null_mask, null_sinks, S);
+ mlx_array_free(qr);
+ mlx_array_free(kr);
+ mlx_array_free(vt);
+ /* attn: [1, n_heads, seq, head_dim] */
+
+ /* ---- 2h. Transpose back > [1, seq, n_heads, head_dim] > squeeze > reshape [seq, q_dim] ---- */
+ mlx_array attn_t = mlx_array_new();
+ mlx_transpose_axes(&attn_t, attn, perm, 4, S); /* [1, seq, n_heads, head_dim] */
+ mlx_array_free(attn);
+
+ mlx_array attn_sq = mlx_array_new();
+ mlx_squeeze_axis(&attn_sq, attn_t, 0, S); /* [seq, n_heads, head_dim] */
+ mlx_array_free(attn_t);
+
+ int attn_flat_shape[] = {seq, q_dim};
+ mlx_array attn_flat = mlx_array_new();
+ mlx_reshape(&attn_flat, attn_sq, attn_flat_shape, 2, S);
+ mlx_array_free(attn_sq);
+
+ /* ---- 2i. Output projection + residual ---- */
+ mlx_array proj = linear(attn_flat, l->wo, S);
+ mlx_array_free(attn_flat);
+
+ mlx_array x2 = mlx_array_new();
+ mlx_add(&x2, x, proj, S);
+ mlx_array_free(x);
+ mlx_array_free(proj);
+ x = x2;
+
+ /* ---- 2j. Post-attention RMSNorm ---- */
+ mlx_array xn2 = mlx_array_new();
+ mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, 1e-6f, S);
+
+ /* ---- 2k. SwiGLU MLP ----
+ * gate = x_norm @ gate_proj^T [seq, inter]
+ * up = x_norm @ up_proj^T [seq, inter]
+ * mid = silu(gate) * up = (gate * sigmoid(gate)) * up
+ * out = mid @ down_proj^T [seq, hidden]
+ */
+ mlx_array gate = linear(xn2, l->gate_proj, S);
+ mlx_array up = linear(xn2, l->up_proj, S);
+ mlx_array_free(xn2);
+
+ /* SiLU(gate) = gate * sigmoid(gate) */
+ mlx_array gate_sig = mlx_array_new();
+ mlx_sigmoid(&gate_sig, gate, S);
+ mlx_array silu_gate = mlx_array_new();
+ mlx_multiply(&silu_gate, gate, gate_sig, S);
+ mlx_array_free(gate);
+ mlx_array_free(gate_sig);
+
+ /* mid = silu_gate * up */
+ mlx_array mid = mlx_array_new();
+ mlx_multiply(&mid, silu_gate, up, S);
+ mlx_array_free(silu_gate);
+ mlx_array_free(up);
+
+ /* down projection */
+ mlx_array ffn = linear(mid, l->down_proj, S);
+ mlx_array_free(mid);
+
+ /* Residual */
+ mlx_array x3 = mlx_array_new();
+ mlx_add(&x3, x, ffn, S);
+ mlx_array_free(x);
+ mlx_array_free(ffn);
+ x = x3;
+ }
+
+ /* ------------------------------------------------------------------
+ * 3. Final RMSNorm
+ * ------------------------------------------------------------------ */
+ mlx_array x_normed = mlx_array_new();
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, 1e-6f, S);
+ mlx_array_free(x);
+
+ /* ------------------------------------------------------------------
+ * 4. Mean pooling: mean(x, axis=0) [hidden]
+ * ------------------------------------------------------------------ */
+ mlx_array emb = mlx_array_new();
+ mlx_mean_axis(&emb, x_normed, /*axis=*/0, /*keepdims=*/false, S);
+ mlx_array_free(x_normed);
+
+ /* ------------------------------------------------------------------
+ * 5. L2 normalization: emb / ||emb||_2
+ * ------------------------------------------------------------------ */
+ int norm_axes[] = {0};
+ mlx_array norm_val = mlx_array_new();
+ mlx_linalg_norm_l2(&norm_val, emb, norm_axes, 1, /*keepdims=*/true, S);
+
+ mlx_array emb_normed = mlx_array_new();
+ mlx_divide(&emb_normed, emb, norm_val, S);
+ mlx_array_free(emb);
+ mlx_array_free(norm_val);
+
+ /* ------------------------------------------------------------------
+ * 6. Evaluate and copy result to CPU
+ * ------------------------------------------------------------------ */
+ mlx_array_eval(emb_normed);
+
+ const float *data = mlx_array_data_float32(emb_normed);
+ if (!data) {
+ fprintf(stderr, "mlx: failed to get embedding data\n");
+ mlx_array_free(emb_normed);
+ return NULL;
+ }
+
+ float *out = (float *)malloc(hidden * sizeof(float));
+ if (out) memcpy(out, data, hidden * sizeof(float));
+
+ mlx_array_free(emb_normed);
+ return out;
+}