#include "pplx_embed_mlx.h" #include "pplx_embed.h" #include "qwen_asr_safetensors.h" #include #include #include #include #include /* ======================================================================== * Per-layer MLX weight arrays * ======================================================================== */ typedef struct { mlx_array wq, wk, wv, wo; mlx_array q_norm, k_norm; mlx_array input_norm, post_attn_norm; mlx_array gate_proj, up_proj, down_proj; } mlx_layer_t; struct pplx_mlx_ctx { pplx_config_t config; mlx_array embed_tokens; mlx_layer_t *layers; /* heap-allocated [n_layers] */ mlx_array norm; mlx_stream stream; multi_safetensors_t *ms; }; /* ======================================================================== * Helpers * ======================================================================== */ static int arr_ok(mlx_array a) { return a.ctx != NULL; } static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { safetensors_file_t *sf = NULL; const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { fprintf(stderr, "mlx: tensor not found: %s\n", name); return (mlx_array){0}; } if (t->dtype != DTYPE_F32) { fprintf(stderr, "mlx: expected F32 for %s\n", name); return (mlx_array){0}; } return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, MLX_FLOAT32); } /* y = x @ W^T */ static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s) { mlx_array Wt = mlx_array_new(); mlx_transpose(&Wt, W, s); mlx_array res = mlx_array_new(); mlx_matmul(&res, x, Wt, s); mlx_array_free(Wt); return res; } const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx) { return ctx ? &ctx->config : NULL; } /* ======================================================================== * Load (reuses the same parse_config logic via pplx_load then copies) * ======================================================================== */ /* We duplicate a minimal config parser here so MLX can load standalone. * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */ static const char *skip_ws_m(const char *p) { while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++; return p; } static const char *json_find_m(const char *json, const char *key) { size_t klen = strlen(key); const char *p = json; while ((p = strstr(p, key)) != NULL) { if (p > json && *(p - 1) == '"') { const char *a = p + klen; if (*a == '"') { a = skip_ws_m(a + 1); if (*a == ':') return skip_ws_m(a + 1); } } p += klen; } return NULL; } static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir) { char path[1024]; snprintf(path, sizeof(path), "%s/config.json", model_dir); FILE *f = fopen(path, "rb"); if (!f) { fprintf(stderr, "mlx: cannot open %s\n", path); return -1; } fseek(f, 0, SEEK_END); long sz = ftell(f); fseek(f, 0, SEEK_SET); char *buf = (char *)malloc(sz + 1); if (!buf || fread(buf, 1, sz, f) != (size_t)sz) { fclose(f); free(buf); return -1; } fclose(f); buf[sz] = '\0'; #define GI(k, fb) do { const char *v = json_find_m(buf, k); cfg->fb = v ? atoi(v) : 0; } while(0) #define GF(k, fb, def) do { const char *v = json_find_m(buf, k); cfg->fb = v ? (float)atof(v) : def; } while(0) GI("hidden_size", hidden_size); GI("num_hidden_layers", n_layers); GI("num_attention_heads", n_heads); GI("num_key_value_heads", n_kv_heads); GI("intermediate_size", intermediate_size); cfg->head_dim = PPLX_HEAD_DIM; cfg->vocab_size = PPLX_VOCAB_SIZE; GF("rms_norm_eps", rms_norm_eps, 1e-6f); GF("rope_theta", rope_theta, 1000000.0f); #undef GI #undef GF cfg->q_dim = cfg->n_heads * cfg->head_dim; cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim; free(buf); if (cfg->hidden_size <= 0 || cfg->n_layers <= 0) { fprintf(stderr, "mlx: bad config (hidden=%d layers=%d)\n", cfg->hidden_size, cfg->n_layers); return -1; } return 0; } pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) { pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); if (!ctx) return NULL; if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } const pplx_config_t *c = &ctx->config; ctx->stream = mlx_default_gpu_stream_new(); ctx->ms = multi_safetensors_open(model_dir); if (!ctx->ms) { fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir); free(ctx); return NULL; } int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; int inter = c->intermediate_size, hd = c->head_dim; /* Embedding table */ int emb_shape[] = {c->vocab_size, h}; ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2); if (!arr_ok(ctx->embed_tokens)) goto fail; /* Layers */ ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); if (!ctx->layers) goto fail; char name[256]; for (int i = 0; i < c->n_layers; i++) { mlx_layer_t *l = &ctx->layers[i]; #define LD(fld, fmt, ...) do { \ snprintf(name, sizeof(name), fmt, i); \ int sh[] = {__VA_ARGS__}; \ l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ if (!arr_ok(l->fld)) goto fail; \ } while (0) LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); LD(input_norm, "layers.%d.input_layernorm.weight", h); LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); #undef LD if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); } int ns[] = {h}; ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); if (!arr_ok(ctx->norm)) goto fail; if (pplx_verbose >= 1) fprintf(stderr, "mlx: %d layers loaded (%d-dim)\n", c->n_layers, h); return ctx; fail: pplx_mlx_free(ctx); return NULL; } /* ======================================================================== * Free * ======================================================================== */ static void free_mlx_layer(mlx_layer_t *l) { mlx_array_free(l->wq); mlx_array_free(l->wk); mlx_array_free(l->wv); mlx_array_free(l->wo); mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); mlx_array_free(l->gate_proj); mlx_array_free(l->up_proj); mlx_array_free(l->down_proj); } void pplx_mlx_free(pplx_mlx_ctx_t *ctx) { if (!ctx) return; mlx_array_free(ctx->embed_tokens); if (ctx->layers) { for (int i = 0; i < ctx->config.n_layers; i++) free_mlx_layer(&ctx->layers[i]); free(ctx->layers); } mlx_array_free(ctx->norm); mlx_stream_free(ctx->stream); if (ctx->ms) multi_safetensors_close(ctx->ms); free(ctx); } /* ======================================================================== * Forward pass * ======================================================================== */ float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { if (!ctx || !token_ids || n_tokens <= 0) return NULL; const pplx_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int hidden = c->hidden_size; int n_heads = c->n_heads; int n_kv_heads = c->n_kv_heads; int head_dim = c->head_dim; int q_dim = c->q_dim; int seq = n_tokens; float scale = 1.0f / sqrtf((float)head_dim); /* 1. Embedding lookup */ int ids_shape[] = {seq}; mlx_array ids = mlx_array_new_data(token_ids, ids_shape, 1, MLX_INT32); mlx_array x = mlx_array_new(); mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); mlx_array_free(ids); /* 2. Transformer layers */ for (int layer = 0; layer < c->n_layers; layer++) { mlx_layer_t *l = &ctx->layers[layer]; /* Input RMSNorm */ mlx_array xn = mlx_array_new(); mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); /* QKV projections */ mlx_array q_flat = linear(xn, l->wq, S); mlx_array k_flat = linear(xn, l->wk, S); mlx_array v_flat = linear(xn, l->wv, S); mlx_array_free(xn); /* Reshape to [seq, n_heads, head_dim] */ int q_shape[] = {seq, n_heads, head_dim}; int k_shape[] = {seq, n_kv_heads, head_dim}; mlx_array q = mlx_array_new(); mlx_array k = mlx_array_new(); mlx_array v = mlx_array_new(); mlx_reshape(&q, q_flat, q_shape, 3, S); mlx_reshape(&k, k_flat, k_shape, 3, S); mlx_reshape(&v, v_flat, k_shape, 3, S); mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat); /* Per-head Q/K RMSNorm */ mlx_array qn = mlx_array_new(); mlx_array kn = mlx_array_new(); mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S); mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S); mlx_array_free(q); mlx_array_free(k); /* Transpose to [1, n_heads, seq, head_dim] for RoPE + SDPA */ int perm[] = {0, 2, 1, 3}; mlx_array qe = mlx_array_new(), ke = mlx_array_new(), ve = mlx_array_new(); mlx_expand_dims(&qe, qn, 0, S); mlx_expand_dims(&ke, kn, 0, S); mlx_expand_dims(&ve, v, 0, S); mlx_array_free(qn); mlx_array_free(kn); mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new(); mlx_transpose_axes(&qt, qe, perm, 4, S); mlx_transpose_axes(&kt, ke, perm, 4, S); mlx_transpose_axes(&vt, ve, perm, 4, S); mlx_array_free(qe); mlx_array_free(ke); mlx_array_free(ve); mlx_array_free(v); /* RoPE (NeoX: traditional=false) */ mlx_optional_float base = {.value = c->rope_theta, .has_value = true}; mlx_array null_arr = (mlx_array){0}; mlx_array qr = mlx_array_new(), kr = mlx_array_new(); mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S); mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S); mlx_array_free(qt); mlx_array_free(kt); /* Bidirectional SDPA (mask_mode="" = no causal mask, GQA via broadcast) */ mlx_array attn = mlx_array_new(); mlx_fast_scaled_dot_product_attention( &attn, qr, kr, vt, scale, "", null_arr, null_arr, S); mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt); /* Transpose back and reshape to [seq, q_dim] */ mlx_array attn_t = mlx_array_new(); mlx_transpose_axes(&attn_t, attn, perm, 4, S); mlx_array_free(attn); mlx_array attn_sq = mlx_array_new(); mlx_squeeze_axis(&attn_sq, attn_t, 0, S); mlx_array_free(attn_t); int flat_shape[] = {seq, q_dim}; mlx_array attn_flat = mlx_array_new(); mlx_reshape(&attn_flat, attn_sq, flat_shape, 2, S); mlx_array_free(attn_sq); /* Output projection + residual */ mlx_array proj = linear(attn_flat, l->wo, S); mlx_array_free(attn_flat); mlx_array x2 = mlx_array_new(); mlx_add(&x2, x, proj, S); mlx_array_free(x); mlx_array_free(proj); x = x2; /* Post-attention RMSNorm */ mlx_array xn2 = mlx_array_new(); mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); /* SwiGLU MLP */ mlx_array gate = linear(xn2, l->gate_proj, S); mlx_array up = linear(xn2, l->up_proj, S); mlx_array_free(xn2); mlx_array gate_sig = mlx_array_new(); mlx_sigmoid(&gate_sig, gate, S); mlx_array silu_gate = mlx_array_new(); mlx_multiply(&silu_gate, gate, gate_sig, S); mlx_array_free(gate); mlx_array_free(gate_sig); mlx_array mid = mlx_array_new(); mlx_multiply(&mid, silu_gate, up, S); mlx_array_free(silu_gate); mlx_array_free(up); mlx_array ffn = linear(mid, l->down_proj, S); mlx_array_free(mid); mlx_array x3 = mlx_array_new(); mlx_add(&x3, x, ffn, S); mlx_array_free(x); mlx_array_free(ffn); x = x3; } /* 3. Final RMSNorm */ mlx_array x_normed = mlx_array_new(); mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); mlx_array_free(x); /* 4. Mean pool over seq dim */ mlx_array emb = mlx_array_new(); mlx_mean_axis(&emb, x_normed, 0, false, S); mlx_array_free(x_normed); /* 5. L2 normalize */ int norm_axes[] = {0}; mlx_array nv = mlx_array_new(); mlx_linalg_norm_l2(&nv, emb, norm_axes, 1, true, S); mlx_array emb_n = mlx_array_new(); mlx_divide(&emb_n, emb, nv, S); mlx_array_free(emb); mlx_array_free(nv); /* 6. Eval and copy to CPU */ mlx_array_eval(emb_n); const float *data = mlx_array_data_float32(emb_n); float *out = NULL; if (data) { out = (float *)malloc(hidden * sizeof(float)); if (out) memcpy(out, data, hidden * sizeof(float)); } mlx_array_free(emb_n); return out; }