aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-30 11:23:52 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:26:57 +0200
commitf6e959455add3260a483c89e6fb9aa214d4e6789 (patch)
treecf4ace129480645fe8e19d04e3b7688549b93be4
parent1a69ce474b8698ab8701119305e32c55cac7239b (diff)
downloadffwd-f6e959455add3260a483c89e6fb9aa214d4e6789.tar.gz
ffwd-f6e959455add3260a483c89e6fb9aa214d4e6789.zip
dist: load distributed layer shards selectively
-rw-r--r--pplx_embed.c60
-rw-r--r--pplx_embed.h12
-rw-r--r--pplx_embed_mlx.c71
-rw-r--r--pplx_embed_mlx.h8
4 files changed, 122 insertions, 29 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index 1999c8c..d4b2d02 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -37,6 +37,8 @@ struct pplx_model {
pplx_config_t config;
pplx_weights_t weights;
void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
+ int layer_start; /* loaded transformer range, inclusive */
+ int layer_end; /* loaded transformer range, exclusive */
};
struct pplx_workspace {
@@ -556,7 +558,8 @@ static int ensure_offsets(pplx_workspace_t *ws, int batch)
* Model / workspace load and free
* ======================================================================== */
-pplx_model_t *pplx_model_load(const char *model_dir)
+static pplx_model_t *model_load_range(const char *model_dir,
+ int layer_start, int layer_end)
{
pplx_model_t *model = (pplx_model_t *)calloc(1, sizeof(pplx_model_t));
if (!model) return NULL;
@@ -568,6 +571,17 @@ pplx_model_t *pplx_model_load(const char *model_dir)
}
const pplx_config_t *cfg = &model->config;
+ if (layer_end == -1)
+ layer_end = cfg->n_layers;
+ if (layer_start < 0 || layer_start >= layer_end ||
+ layer_end > cfg->n_layers) {
+ fprintf(stderr, "pplx_model_load: invalid layer range [%d, %d) "
+ "for %d-layer model\n", layer_start, layer_end, cfg->n_layers);
+ free(model);
+ return NULL;
+ }
+ model->layer_start = layer_start;
+ model->layer_end = layer_end;
/* Open safetensors (handles single file or multi-shard) */
multi_safetensors_t *ms = multi_safetensors_open(model_dir);
@@ -581,12 +595,14 @@ pplx_model_t *pplx_model_load(const char *model_dir)
if (pplx_verbose >= 1)
fprintf(stderr, "pplx_model_load: loading weights from %s\n", model_dir);
- /* Token embeddings */
+ /* Only the first stage needs token embeddings. */
pplx_weights_t *w = &model->weights;
- const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size };
- w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight",
- embed_shape, 2);
- if (!w->embed_tokens.data) goto fail;
+ if (layer_start == 0) {
+ const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size };
+ w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight",
+ embed_shape, 2);
+ if (!w->embed_tokens.data) goto fail;
+ }
/* Allocate layer array */
w->layers = (pplx_layer_t *)calloc(cfg->n_layers, sizeof(pplx_layer_t));
@@ -594,7 +610,7 @@ pplx_model_t *pplx_model_load(const char *model_dir)
/* Per-layer weights */
char name[256];
- for (int i = 0; i < cfg->n_layers; i++) {
+ for (int i = layer_start; i < layer_end; i++) {
pplx_layer_t *l = &w->layers[i];
#define LOAD_NORM(field, fmt, d0) do { \
@@ -641,14 +657,17 @@ pplx_model_t *pplx_model_load(const char *model_dir)
fprintf(stderr, " layer %d loaded\n", i);
}
- /* Final RMSNorm */
- const int64_t norm_shape[1] = { cfg->hidden_size };
- w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1);
- if (!w->norm) goto fail;
+ /* Only the final stage applies the output RMSNorm. */
+ if (layer_end == cfg->n_layers) {
+ const int64_t norm_shape[1] = { cfg->hidden_size };
+ w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1);
+ if (!w->norm) goto fail;
+ }
if (pplx_verbose >= 1)
- fprintf(stderr, "pplx_model_load: %d layers loaded (%d-dim embeddings)\n",
- cfg->n_layers, cfg->hidden_size);
+ fprintf(stderr, "pplx_model_load: layers [%d, %d) loaded "
+ "(%d-dim embeddings)\n",
+ layer_start, layer_end, cfg->hidden_size);
return model;
@@ -657,6 +676,17 @@ fail:
return NULL;
}
+pplx_model_t *pplx_model_load(const char *model_dir)
+{
+ return model_load_range(model_dir, 0, -1);
+}
+
+pplx_model_t *pplx_model_load_slice(const char *model_dir,
+ int layer_start, int layer_end)
+{
+ return model_load_range(model_dir, layer_start, layer_end);
+}
+
void pplx_model_free(pplx_model_t *model)
{
if (!model) return;
@@ -805,9 +835,11 @@ static int forward_packed_slice_inplace(const pplx_model_t *model,
const pplx_weights_t *w = &model->weights;
if (layer_start < 0 || layer_start > layer_end ||
layer_end > cfg->n_layers ||
+ layer_start < model->layer_start || layer_end > model->layer_end ||
(layer_start == 0 && input_states) ||
(layer_start != 0 && !input_states) ||
- (apply_final_norm && layer_end != cfg->n_layers))
+ (layer_start == 0 && !w->embed_tokens.data) ||
+ (apply_final_norm && (layer_end != cfg->n_layers || !w->norm)))
return -1;
int hidden = cfg->hidden_size;
diff --git a/pplx_embed.h b/pplx_embed.h
index 1a2479d..6a159de 100644
--- a/pplx_embed.h
+++ b/pplx_embed.h
@@ -67,6 +67,18 @@ typedef struct {
* Returns NULL on error.
*/
pplx_model_t *pplx_model_load(const char *model_dir);
+
+/*
+ * Load one contiguous transformer layer range for distributed execution.
+ *
+ * layer_start is inclusive and layer_end is exclusive. A range beginning at
+ * zero also loads the token embedding table. A range ending at n_layers also
+ * loads the final RMSNorm. Safetensors remain mmap-backed, so unselected
+ * weights are not touched.
+ */
+pplx_model_t *pplx_model_load_slice(const char *model_dir,
+ int layer_start, int layer_end);
+
void pplx_model_free(pplx_model_t *model);
/*
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index 345bca9..99946d4 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -28,6 +28,8 @@ struct pplx_mlx_ctx {
mlx_array norm;
mlx_stream stream;
multi_safetensors_t *ms;
+ int layer_start; /* loaded transformer range, inclusive */
+ int layer_end; /* loaded transformer range, exclusive */
};
/* ========================================================================
@@ -36,6 +38,11 @@ struct pplx_mlx_ctx {
static int arr_ok(mlx_array a) { return a.ctx != NULL; }
+static mlx_dtype mlx_weight_dtype(const pplx_mlx_ctx_t *ctx)
+{
+ return mlx_array_dtype(ctx->layers[ctx->layer_start].wq);
+}
+
static mlx_array load_tensor(multi_safetensors_t *ms, const char *name,
const int *shape, int ndim)
{
@@ -143,7 +150,8 @@ static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir)
return 0;
}
-pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
+static pplx_mlx_ctx_t *mlx_load_range(const char *model_dir,
+ int layer_start, int layer_end)
{
pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t));
if (!ctx) return NULL;
@@ -151,6 +159,17 @@ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; }
const pplx_config_t *c = &ctx->config;
+ if (layer_end == -1)
+ layer_end = c->n_layers;
+ if (layer_start < 0 || layer_start >= layer_end ||
+ layer_end > c->n_layers) {
+ fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n",
+ layer_start, layer_end, c->n_layers);
+ free(ctx);
+ return NULL;
+ }
+ ctx->layer_start = layer_start;
+ ctx->layer_end = layer_end;
ctx->stream = mlx_default_gpu_stream_new();
ctx->ms = multi_safetensors_open(model_dir);
@@ -162,17 +181,20 @@ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim;
int inter = c->intermediate_size, hd = c->head_dim;
- /* Embedding table */
- int emb_shape[] = {c->vocab_size, h};
- ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2);
- if (!arr_ok(ctx->embed_tokens)) goto fail;
+ /* Only the first stage needs token embeddings. */
+ if (layer_start == 0) {
+ int emb_shape[] = {c->vocab_size, h};
+ ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight",
+ emb_shape, 2);
+ if (!arr_ok(ctx->embed_tokens)) goto fail;
+ }
/* Layers */
ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
if (!ctx->layers) goto fail;
char name[256];
- for (int i = 0; i < c->n_layers; i++) {
+ for (int i = layer_start; i < layer_end; i++) {
mlx_layer_t *l = &ctx->layers[i];
#define LD(fld, fmt, ...) do { \
@@ -198,12 +220,15 @@ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i);
}
- int ns[] = {h};
- ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1);
- if (!arr_ok(ctx->norm)) goto fail;
+ if (layer_end == c->n_layers) {
+ int ns[] = {h};
+ ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1);
+ if (!arr_ok(ctx->norm)) goto fail;
+ }
if (pplx_verbose >= 1)
- fprintf(stderr, "mlx: %d layers loaded (%d-dim)\n", c->n_layers, h);
+ fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n",
+ layer_start, layer_end, h);
return ctx;
@@ -212,6 +237,17 @@ fail:
return NULL;
}
+pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir)
+{
+ return mlx_load_range(model_dir, 0, -1);
+}
+
+pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir,
+ int layer_start, int layer_end)
+{
+ return mlx_load_range(model_dir, layer_start, layer_end);
+}
+
/* ========================================================================
* Free
* ======================================================================== */
@@ -353,7 +389,9 @@ static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x,
int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
int batch, float *out_embeddings)
{
- if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1;
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
+ ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers)
+ return -1;
const pplx_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
@@ -426,7 +464,7 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs,
mlx_array_free(lengths);
return -1;
}
- if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
+ if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
mlx_array attn_mask_bf16 = mlx_array_new();
mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
mlx_array_free(attn_mask);
@@ -501,9 +539,12 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
if (!ctx || !inputs || batch <= 0 || !out_states ||
layer_start < 0 || layer_start > layer_end ||
layer_end > ctx->config.n_layers ||
+ layer_start < ctx->layer_start || layer_end > ctx->layer_end ||
(layer_start == 0 && input_states) ||
(layer_start != 0 && !input_states) ||
- (apply_final_norm && layer_end != ctx->config.n_layers))
+ (layer_start == 0 && !arr_ok(ctx->embed_tokens)) ||
+ (apply_final_norm &&
+ (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm))))
return -1;
const pplx_config_t *c = &ctx->config;
@@ -581,7 +622,7 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
free(padded_ids); free(padded_states);
return -1;
}
- if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
+ if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
mlx_array attn_mask_bf16 = mlx_array_new();
mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
mlx_array_free(attn_mask);
@@ -613,7 +654,7 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
if (has_padding) mlx_array_free(attn_mask);
return -1;
}
- if (mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) {
+ if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
x = mlx_array_new();
mlx_astype(&x, states, MLX_BFLOAT16, S);
mlx_array_free(states);
diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h
index 1ec6fab..b45fc8a 100644
--- a/pplx_embed_mlx.h
+++ b/pplx_embed_mlx.h
@@ -20,6 +20,14 @@ typedef struct pplx_mlx_ctx pplx_mlx_ctx_t;
*/
pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir);
+/*
+ * Load one contiguous transformer layer range for distributed execution.
+ * Uses the same half-open range and endpoint ownership rules as
+ * pplx_model_load_slice().
+ */
+pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir,
+ int layer_start, int layer_end);
+
void pplx_mlx_free(pplx_mlx_ctx_t *ctx);
/*