aboutsummaryrefslogtreecommitdiff
path: root/pplx_embed_mlx.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-30 11:11:42 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:26:52 +0200
commit44a235498ab54ffac97b7a0be659f6da83062d3f (patch)
tree18dbe232204a443b6406ae4455996bffd576cdb4 /pplx_embed_mlx.c
parent10bf114474d5b79193159e5c865734362cf7abfb (diff)
downloadffwd-44a235498ab54ffac97b7a0be659f6da83062d3f.tar.gz
ffwd-44a235498ab54ffac97b7a0be659f6da83062d3f.zip
dist: add distributed layer-slice foundation
Diffstat (limited to 'pplx_embed_mlx.c')
-rw-r--r--pplx_embed_mlx.c462
1 files changed, 255 insertions, 207 deletions
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index dd17d17..345bca9 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -8,6 +8,7 @@
#include <stdlib.h>
#include <string.h>
#include <math.h>
+#include <limits.h>
/* ========================================================================
* Per-layer MLX weight arrays
@@ -244,107 +245,21 @@ void pplx_mlx_free(pplx_mlx_ctx_t *ctx)
* Forward pass
* ======================================================================== */
-int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
- int batch, float *out_embeddings)
+static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x,
+ int batch, int max_seq, int has_padding,
+ mlx_array attn_mask,
+ int layer_start, int layer_end)
{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
-
const pplx_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
-
- int hidden = c->hidden_size;
- int n_heads = c->n_heads;
+ int n_heads = c->n_heads;
int n_kv_heads = c->n_kv_heads;
- int head_dim = c->head_dim;
- int q_dim = c->q_dim;
- int max_seq = 0;
- int has_padding = 0;
-
- for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1;
- if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
- }
- if (max_seq <= 0) return -1;
-
- for (int b = 0; b < batch; b++) {
- if (inputs[b].n_tokens != max_seq) has_padding = 1;
- }
-
- size_t elems = (size_t)batch * (size_t)max_seq;
- int *padded_ids = (int *)malloc(elems * sizeof(int));
- float *pool_mask_data = (float *)malloc(elems * sizeof(float));
- float *attn_mask_data = has_padding ? (float *)malloc(elems * sizeof(float)) : NULL;
- float *len_data = (float *)malloc((size_t)batch * sizeof(float));
- if (!padded_ids || !pool_mask_data || (has_padding && !attn_mask_data) || !len_data) {
- free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
- return -1;
- }
-
- for (int b = 0; b < batch; b++) {
- len_data[b] = (float)inputs[b].n_tokens;
- for (int t = 0; t < max_seq; t++) {
- size_t idx = (size_t)b * max_seq + t;
- if (t < inputs[b].n_tokens) {
- int id = inputs[b].ids[t];
- if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
- id, b, t);
- free(padded_ids); free(pool_mask_data);
- free(attn_mask_data); free(len_data);
- return -1;
- }
- padded_ids[idx] = id;
- pool_mask_data[idx] = 1.0f;
- if (has_padding) attn_mask_data[idx] = 0.0f;
- } else {
- padded_ids[idx] = 0;
- pool_mask_data[idx] = 0.0f;
- if (has_padding) attn_mask_data[idx] = -1.0e9f;
- }
- }
- }
-
- int ids_shape[] = {batch, max_seq};
- int pool_mask_shape[] = {batch, max_seq, 1};
- int attn_mask_shape[] = {batch, 1, 1, max_seq};
- int len_shape[] = {batch, 1};
-
- mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- mlx_array pool_mask = mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32);
- mlx_array attn_mask = has_padding
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array lengths = mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32);
- free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
-
- if (!arr_ok(ids) || !arr_ok(pool_mask) || (has_padding && !arr_ok(attn_mask)) ||
- !arr_ok(lengths)) {
- mlx_array_free(ids); mlx_array_free(pool_mask);
- if (has_padding) mlx_array_free(attn_mask);
- mlx_array_free(lengths);
- return -1;
- }
- if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) {
- mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths);
- return -1;
- }
- }
-
+ int head_dim = c->head_dim;
+ int q_dim = c->q_dim;
float scale = 1.0f / sqrtf((float)head_dim);
mlx_array null_arr = (mlx_array){0};
- /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */
- mlx_array x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
-
- /* 2. Transformer layers. */
- for (int layer = 0; layer < c->n_layers; layer++) {
+ for (int layer = layer_start; layer < layer_end; layer++) {
mlx_layer_t *l = &ctx->layers[layer];
mlx_array xn = mlx_array_new();
@@ -432,6 +347,104 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
mlx_array_free(x); mlx_array_free(ffn);
x = x3;
}
+ return x;
+}
+
+int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
+ int batch, float *out_embeddings)
+{
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+
+ const pplx_config_t *c = &ctx->config;
+ mlx_stream S = ctx->stream;
+
+ int hidden = c->hidden_size;
+ int max_seq = 0;
+ int has_padding = 0;
+
+ for (int b = 0; b < batch; b++) {
+ if (!inputs[b].ids || inputs[b].n_tokens <= 0) return -1;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ }
+ if (max_seq <= 0) return -1;
+
+ for (int b = 0; b < batch; b++) {
+ if (inputs[b].n_tokens != max_seq) has_padding = 1;
+ }
+
+ size_t elems = (size_t)batch * (size_t)max_seq;
+ int *padded_ids = (int *)malloc(elems * sizeof(int));
+ float *pool_mask_data = (float *)malloc(elems * sizeof(float));
+ float *attn_mask_data = has_padding ? (float *)malloc(elems * sizeof(float)) : NULL;
+ float *len_data = (float *)malloc((size_t)batch * sizeof(float));
+ if (!padded_ids || !pool_mask_data || (has_padding && !attn_mask_data) || !len_data) {
+ free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+ return -1;
+ }
+
+ for (int b = 0; b < batch; b++) {
+ len_data[b] = (float)inputs[b].n_tokens;
+ for (int t = 0; t < max_seq; t++) {
+ size_t idx = (size_t)b * max_seq + t;
+ if (t < inputs[b].n_tokens) {
+ int id = inputs[b].ids[t];
+ if (id < 0 || id >= c->vocab_size) {
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
+ id, b, t);
+ free(padded_ids); free(pool_mask_data);
+ free(attn_mask_data); free(len_data);
+ return -1;
+ }
+ padded_ids[idx] = id;
+ pool_mask_data[idx] = 1.0f;
+ if (has_padding) attn_mask_data[idx] = 0.0f;
+ } else {
+ padded_ids[idx] = 0;
+ pool_mask_data[idx] = 0.0f;
+ if (has_padding) attn_mask_data[idx] = -1.0e9f;
+ }
+ }
+ }
+
+ int ids_shape[] = {batch, max_seq};
+ int pool_mask_shape[] = {batch, max_seq, 1};
+ int attn_mask_shape[] = {batch, 1, 1, max_seq};
+ int len_shape[] = {batch, 1};
+
+ mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
+ mlx_array pool_mask = mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32);
+ mlx_array attn_mask = has_padding
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ mlx_array lengths = mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32);
+ free(padded_ids); free(pool_mask_data); free(attn_mask_data); free(len_data);
+
+ if (!arr_ok(ids) || !arr_ok(pool_mask) || (has_padding && !arr_ok(attn_mask)) ||
+ !arr_ok(lengths)) {
+ mlx_array_free(ids); mlx_array_free(pool_mask);
+ if (has_padding) mlx_array_free(attn_mask);
+ mlx_array_free(lengths);
+ return -1;
+ }
+ if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
+ mlx_array attn_mask_bf16 = mlx_array_new();
+ mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
+ mlx_array_free(attn_mask);
+ attn_mask = attn_mask_bf16;
+ if (!arr_ok(attn_mask)) {
+ mlx_array_free(ids); mlx_array_free(pool_mask); mlx_array_free(lengths);
+ return -1;
+ }
+ }
+
+ /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */
+ mlx_array x = mlx_array_new();
+ mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
+ mlx_array_free(ids);
+
+ /* 2. Transformer layers. */
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
+ 0, c->n_layers);
/* 3. Final RMSNorm. */
mlx_array x_normed = mlx_array_new();
@@ -478,140 +491,175 @@ int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding);
}
-static int pplx_mlx_forward_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, float *out_states)
+int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
+ const pplx_input_t *inputs, int batch,
+ const float *input_states,
+ int layer_start, int layer_end,
+ int apply_final_norm,
+ float *out_states)
{
- if (!ctx || !token_ids || n_tokens <= 0 || !out_states) return -1;
+ if (!ctx || !inputs || batch <= 0 || !out_states ||
+ layer_start < 0 || layer_start > layer_end ||
+ layer_end > ctx->config.n_layers ||
+ (layer_start == 0 && input_states) ||
+ (layer_start != 0 && !input_states) ||
+ (apply_final_norm && layer_end != ctx->config.n_layers))
+ return -1;
const pplx_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
-
- int hidden = c->hidden_size;
- int n_heads = c->n_heads;
- int n_kv_heads = c->n_kv_heads;
- int head_dim = c->head_dim;
- int q_dim = c->q_dim;
-
- for (int i = 0; i < n_tokens; i++) {
- if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at position %d\n",
- token_ids[i], i);
+ int hidden = c->hidden_size;
+ int max_seq = 0;
+ int total_seq = 0;
+ int has_padding = 0;
+ for (int b = 0; b < batch; b++) {
+ if ((layer_start == 0 && !inputs[b].ids) || inputs[b].n_tokens <= 0 ||
+ total_seq > INT_MAX - inputs[b].n_tokens)
return -1;
- }
+ total_seq += inputs[b].n_tokens;
+ if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
+ }
+ for (int b = 0; b < batch; b++) {
+ if (inputs[b].n_tokens != max_seq) has_padding = 1;
}
- int ids_shape[] = {1, n_tokens};
- mlx_array ids = mlx_array_new_data((void *)token_ids, ids_shape, 2, MLX_INT32);
- if (!arr_ok(ids)) return -1;
-
- float scale = 1.0f / sqrtf((float)head_dim);
- mlx_array null_arr = (mlx_array){0};
-
- mlx_array x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
-
- for (int layer = 0; layer < c->n_layers; layer++) {
- mlx_layer_t *l = &ctx->layers[layer];
-
- mlx_array xn = mlx_array_new();
- mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S);
-
- mlx_array q_flat = linear(xn, l->wq, S);
- mlx_array k_flat = linear(xn, l->wk, S);
- mlx_array v_flat = linear(xn, l->wv, S);
- mlx_array_free(xn);
-
- int q_shape[] = {1, n_tokens, n_heads, head_dim};
- int k_shape[] = {1, n_tokens, n_kv_heads, head_dim};
- mlx_array q = mlx_array_new(), k = mlx_array_new(), v = mlx_array_new();
- mlx_reshape(&q, q_flat, q_shape, 4, S);
- mlx_reshape(&k, k_flat, k_shape, 4, S);
- mlx_reshape(&v, v_flat, k_shape, 4, S);
- mlx_array_free(q_flat); mlx_array_free(k_flat); mlx_array_free(v_flat);
-
- mlx_array qn = mlx_array_new(), kn = mlx_array_new();
- mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
- mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
- mlx_array_free(q); mlx_array_free(k);
-
- int perm[] = {0, 2, 1, 3};
- mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
- mlx_transpose_axes(&qt, qn, perm, 4, S);
- mlx_transpose_axes(&kt, kn, perm, 4, S);
- mlx_transpose_axes(&vt, v, perm, 4, S);
- mlx_array_free(qn); mlx_array_free(kn); mlx_array_free(v);
-
- mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
- mlx_array qr = mlx_array_new(), kr = mlx_array_new();
- mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_array_free(qt); mlx_array_free(kt);
-
- mlx_array attn = mlx_array_new();
- mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, "",
- null_arr, null_arr, S);
- mlx_array_free(qr); mlx_array_free(kr); mlx_array_free(vt);
-
- mlx_array attn_t = mlx_array_new();
- mlx_transpose_axes(&attn_t, attn, perm, 4, S);
- mlx_array_free(attn);
-
- int flat_shape[] = {1, n_tokens, q_dim};
- mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S);
- mlx_array_free(attn_t);
-
- mlx_array proj = linear(attn_flat, l->wo, S);
- mlx_array_free(attn_flat);
- mlx_array x2 = mlx_array_new();
- mlx_add(&x2, x, proj, S);
- mlx_array_free(x); mlx_array_free(proj);
- x = x2;
+ if ((size_t)batch > SIZE_MAX / (size_t)max_seq) return -1;
+ size_t rows = (size_t)batch * (size_t)max_seq;
+ if (rows > SIZE_MAX / (size_t)hidden ||
+ rows * (size_t)hidden > SIZE_MAX / sizeof(float))
+ return -1;
+ size_t state_values = rows * (size_t)hidden;
- mlx_array xn2 = mlx_array_new();
- mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
+ int *padded_ids = layer_start == 0
+ ? (int *)malloc(rows * sizeof(int)) : NULL;
+ float *padded_states = layer_start != 0
+ ? (float *)calloc(state_values, sizeof(float)) : NULL;
+ float *attn_mask_data = has_padding
+ ? (float *)malloc(rows * sizeof(float)) : NULL;
+ if ((layer_start == 0 && !padded_ids) ||
+ (layer_start != 0 && !padded_states) ||
+ (has_padding && !attn_mask_data)) {
+ free(padded_ids); free(padded_states); free(attn_mask_data);
+ return -1;
+ }
- mlx_array gate = linear(xn2, l->gate_proj, S);
- mlx_array up = linear(xn2, l->up_proj, S);
- mlx_array_free(xn2);
+ int packed_offset = 0;
+ for (int b = 0; b < batch; b++) {
+ for (int t = 0; t < max_seq; t++) {
+ size_t dense_row = (size_t)b * max_seq + t;
+ if (t < inputs[b].n_tokens) {
+ if (layer_start == 0) {
+ int id = inputs[b].ids[t];
+ if (id < 0 || id >= c->vocab_size) {
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
+ id, b, t);
+ free(padded_ids); free(padded_states);
+ free(attn_mask_data);
+ return -1;
+ }
+ padded_ids[dense_row] = id;
+ } else {
+ memcpy(padded_states + dense_row * hidden,
+ input_states + (size_t)(packed_offset + t) * hidden,
+ (size_t)hidden * sizeof(float));
+ }
+ if (has_padding) attn_mask_data[dense_row] = 0.0f;
+ } else {
+ if (layer_start == 0) padded_ids[dense_row] = 0;
+ if (has_padding) attn_mask_data[dense_row] = -1.0e9f;
+ }
+ }
+ packed_offset += inputs[b].n_tokens;
+ }
- mlx_array gate_sig = mlx_array_new();
- mlx_sigmoid(&gate_sig, gate, S);
- mlx_array silu_gate = mlx_array_new();
- mlx_multiply(&silu_gate, gate, gate_sig, S);
- mlx_array_free(gate); mlx_array_free(gate_sig);
+ int attn_mask_shape[] = {batch, 1, 1, max_seq};
+ mlx_array attn_mask = has_padding
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ free(attn_mask_data);
+ if (has_padding && !arr_ok(attn_mask)) {
+ free(padded_ids); free(padded_states);
+ return -1;
+ }
+ if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
+ mlx_array attn_mask_bf16 = mlx_array_new();
+ mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
+ mlx_array_free(attn_mask);
+ attn_mask = attn_mask_bf16;
+ if (!arr_ok(attn_mask)) {
+ free(padded_ids); free(padded_states);
+ return -1;
+ }
+ }
- mlx_array mid = mlx_array_new();
- mlx_multiply(&mid, silu_gate, up, S);
- mlx_array_free(silu_gate); mlx_array_free(up);
+ mlx_array x;
+ if (layer_start == 0) {
+ int ids_shape[] = {batch, max_seq};
+ mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
+ free(padded_ids);
+ if (!arr_ok(ids)) {
+ if (has_padding) mlx_array_free(attn_mask);
+ return -1;
+ }
+ x = mlx_array_new();
+ mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
+ mlx_array_free(ids);
+ } else {
+ int states_shape[] = {batch, max_seq, hidden};
+ mlx_array states = mlx_array_new_data(padded_states, states_shape, 3,
+ MLX_FLOAT32);
+ free(padded_states);
+ if (!arr_ok(states)) {
+ if (has_padding) mlx_array_free(attn_mask);
+ return -1;
+ }
+ if (mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
+ x = mlx_array_new();
+ mlx_astype(&x, states, MLX_BFLOAT16, S);
+ mlx_array_free(states);
+ } else {
+ x = states;
+ }
+ }
- mlx_array ffn = linear(mid, l->down_proj, S);
- mlx_array_free(mid);
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
+ layer_start, layer_end);
+ if (has_padding) mlx_array_free(attn_mask);
- mlx_array x3 = mlx_array_new();
- mlx_add(&x3, x, ffn, S);
- mlx_array_free(x); mlx_array_free(ffn);
- x = x3;
+ if (apply_final_norm) {
+ mlx_array x_normed = mlx_array_new();
+ mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
+ mlx_array_free(x);
+ x = x_normed;
}
- mlx_array x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
-
mlx_array x_f32 = mlx_array_new();
- mlx_astype(&x_f32, x_normed, MLX_FLOAT32, S);
- mlx_array_free(x_normed);
+ mlx_astype(&x_f32, x, MLX_FLOAT32, S);
+ mlx_array_free(x);
mlx_array_eval(x_f32);
const float *data = mlx_array_data_float32(x_f32);
- int rc = -1;
- if (data) {
- memcpy(out_states, data, (size_t)n_tokens * hidden * sizeof(float));
- rc = 0;
+ if (!data) {
+ mlx_array_free(x_f32);
+ return -1;
+ }
+ packed_offset = 0;
+ for (int b = 0; b < batch; b++) {
+ memcpy(out_states + (size_t)packed_offset * hidden,
+ data + (size_t)b * max_seq * hidden,
+ (size_t)inputs[b].n_tokens * hidden * sizeof(float));
+ packed_offset += inputs[b].n_tokens;
}
mlx_array_free(x_f32);
- return rc;
+ return 0;
+}
+
+static int pplx_mlx_forward_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
+ int n_tokens, float *out_states)
+{
+ pplx_input_t input = { token_ids, n_tokens };
+ return pplx_mlx_forward_slice_batch(ctx, &input, 1, NULL, 0,
+ ctx->config.n_layers, 1, out_states);
}
int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,