diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-30 11:23:52 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:26:57 +0200 |
| commit | f6e959455add3260a483c89e6fb9aa214d4e6789 (patch) | |
| tree | cf4ace129480645fe8e19d04e3b7688549b93be4 | |
| parent | 1a69ce474b8698ab8701119305e32c55cac7239b (diff) | |
| download | ffwd-f6e959455add3260a483c89e6fb9aa214d4e6789.tar.gz ffwd-f6e959455add3260a483c89e6fb9aa214d4e6789.zip | |
dist: load distributed layer shards selectively
| -rw-r--r-- | pplx_embed.c | 60 | ||||
| -rw-r--r-- | pplx_embed.h | 12 | ||||
| -rw-r--r-- | pplx_embed_mlx.c | 71 | ||||
| -rw-r--r-- | pplx_embed_mlx.h | 8 |
4 files changed, 122 insertions, 29 deletions
diff --git a/pplx_embed.c b/pplx_embed.c index 1999c8c..d4b2d02 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -37,6 +37,8 @@ struct pplx_model { pplx_config_t config; pplx_weights_t weights; void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ }; struct pplx_workspace { @@ -556,7 +558,8 @@ static int ensure_offsets(pplx_workspace_t *ws, int batch) * Model / workspace load and free * ======================================================================== */ -pplx_model_t *pplx_model_load(const char *model_dir) +static pplx_model_t *model_load_range(const char *model_dir, + int layer_start, int layer_end) { pplx_model_t *model = (pplx_model_t *)calloc(1, sizeof(pplx_model_t)); if (!model) return NULL; @@ -568,6 +571,17 @@ pplx_model_t *pplx_model_load(const char *model_dir) } const pplx_config_t *cfg = &model->config; + if (layer_end == -1) + layer_end = cfg->n_layers; + if (layer_start < 0 || layer_start >= layer_end || + layer_end > cfg->n_layers) { + fprintf(stderr, "pplx_model_load: invalid layer range [%d, %d) " + "for %d-layer model\n", layer_start, layer_end, cfg->n_layers); + free(model); + return NULL; + } + model->layer_start = layer_start; + model->layer_end = layer_end; /* Open safetensors (handles single file or multi-shard) */ multi_safetensors_t *ms = multi_safetensors_open(model_dir); @@ -581,12 +595,14 @@ pplx_model_t *pplx_model_load(const char *model_dir) if (pplx_verbose >= 1) fprintf(stderr, "pplx_model_load: loading weights from %s\n", model_dir); - /* Token embeddings */ + /* Only the first stage needs token embeddings. */ pplx_weights_t *w = &model->weights; - const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size }; - w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", - embed_shape, 2); - if (!w->embed_tokens.data) goto fail; + if (layer_start == 0) { + const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size }; + w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", + embed_shape, 2); + if (!w->embed_tokens.data) goto fail; + } /* Allocate layer array */ w->layers = (pplx_layer_t *)calloc(cfg->n_layers, sizeof(pplx_layer_t)); @@ -594,7 +610,7 @@ pplx_model_t *pplx_model_load(const char *model_dir) /* Per-layer weights */ char name[256]; - for (int i = 0; i < cfg->n_layers; i++) { + for (int i = layer_start; i < layer_end; i++) { pplx_layer_t *l = &w->layers[i]; #define LOAD_NORM(field, fmt, d0) do { \ @@ -641,14 +657,17 @@ pplx_model_t *pplx_model_load(const char *model_dir) fprintf(stderr, " layer %d loaded\n", i); } - /* Final RMSNorm */ - const int64_t norm_shape[1] = { cfg->hidden_size }; - w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1); - if (!w->norm) goto fail; + /* Only the final stage applies the output RMSNorm. */ + if (layer_end == cfg->n_layers) { + const int64_t norm_shape[1] = { cfg->hidden_size }; + w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1); + if (!w->norm) goto fail; + } if (pplx_verbose >= 1) - fprintf(stderr, "pplx_model_load: %d layers loaded (%d-dim embeddings)\n", - cfg->n_layers, cfg->hidden_size); + fprintf(stderr, "pplx_model_load: layers [%d, %d) loaded " + "(%d-dim embeddings)\n", + layer_start, layer_end, cfg->hidden_size); return model; @@ -657,6 +676,17 @@ fail: return NULL; } +pplx_model_t *pplx_model_load(const char *model_dir) +{ + return model_load_range(model_dir, 0, -1); +} + +pplx_model_t *pplx_model_load_slice(const char *model_dir, + int layer_start, int layer_end) +{ + return model_load_range(model_dir, layer_start, layer_end); +} + void pplx_model_free(pplx_model_t *model) { if (!model) return; @@ -805,9 +835,11 @@ static int forward_packed_slice_inplace(const pplx_model_t *model, const pplx_weights_t *w = &model->weights; if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers || + layer_start < model->layer_start || layer_end > model->layer_end || (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) || - (apply_final_norm && layer_end != cfg->n_layers)) + (layer_start == 0 && !w->embed_tokens.data) || + (apply_final_norm && (layer_end != cfg->n_layers || !w->norm))) return -1; int hidden = cfg->hidden_size; diff --git a/pplx_embed.h b/pplx_embed.h index 1a2479d..6a159de 100644 --- a/pplx_embed.h +++ b/pplx_embed.h @@ -67,6 +67,18 @@ typedef struct { * Returns NULL on error. */ pplx_model_t *pplx_model_load(const char *model_dir); + +/* + * Load one contiguous transformer layer range for distributed execution. + * + * layer_start is inclusive and layer_end is exclusive. A range beginning at + * zero also loads the token embedding table. A range ending at n_layers also + * loads the final RMSNorm. Safetensors remain mmap-backed, so unselected + * weights are not touched. + */ +pplx_model_t *pplx_model_load_slice(const char *model_dir, + int layer_start, int layer_end); + void pplx_model_free(pplx_model_t *model); /* diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index 345bca9..99946d4 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -28,6 +28,8 @@ struct pplx_mlx_ctx { mlx_array norm; mlx_stream stream; multi_safetensors_t *ms; + int layer_start; /* loaded transformer range, inclusive */ + int layer_end; /* loaded transformer range, exclusive */ }; /* ======================================================================== @@ -36,6 +38,11 @@ struct pplx_mlx_ctx { static int arr_ok(mlx_array a) { return a.ctx != NULL; } +static mlx_dtype mlx_weight_dtype(const pplx_mlx_ctx_t *ctx) +{ + return mlx_array_dtype(ctx->layers[ctx->layer_start].wq); +} + static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { @@ -143,7 +150,8 @@ static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir) return 0; } -pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) +static pplx_mlx_ctx_t *mlx_load_range(const char *model_dir, + int layer_start, int layer_end) { pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); if (!ctx) return NULL; @@ -151,6 +159,17 @@ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } const pplx_config_t *c = &ctx->config; + if (layer_end == -1) + layer_end = c->n_layers; + if (layer_start < 0 || layer_start >= layer_end || + layer_end > c->n_layers) { + fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", + layer_start, layer_end, c->n_layers); + free(ctx); + return NULL; + } + ctx->layer_start = layer_start; + ctx->layer_end = layer_end; ctx->stream = mlx_default_gpu_stream_new(); ctx->ms = multi_safetensors_open(model_dir); @@ -162,17 +181,20 @@ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) int h = c->hidden_size, qd = c->q_dim, kvd = c->kv_dim; int inter = c->intermediate_size, hd = c->head_dim; - /* Embedding table */ - int emb_shape[] = {c->vocab_size, h}; - ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", emb_shape, 2); - if (!arr_ok(ctx->embed_tokens)) goto fail; + /* Only the first stage needs token embeddings. */ + if (layer_start == 0) { + int emb_shape[] = {c->vocab_size, h}; + ctx->embed_tokens = load_tensor(ctx->ms, "embed_tokens.weight", + emb_shape, 2); + if (!arr_ok(ctx->embed_tokens)) goto fail; + } /* Layers */ ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t)); if (!ctx->layers) goto fail; char name[256]; - for (int i = 0; i < c->n_layers; i++) { + for (int i = layer_start; i < layer_end; i++) { mlx_layer_t *l = &ctx->layers[i]; #define LD(fld, fmt, ...) do { \ @@ -198,12 +220,15 @@ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) if (pplx_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); } - int ns[] = {h}; - ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); - if (!arr_ok(ctx->norm)) goto fail; + if (layer_end == c->n_layers) { + int ns[] = {h}; + ctx->norm = load_tensor(ctx->ms, "norm.weight", ns, 1); + if (!arr_ok(ctx->norm)) goto fail; + } if (pplx_verbose >= 1) - fprintf(stderr, "mlx: %d layers loaded (%d-dim)\n", c->n_layers, h); + fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", + layer_start, layer_end, h); return ctx; @@ -212,6 +237,17 @@ fail: return NULL; } +pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) +{ + return mlx_load_range(model_dir, 0, -1); +} + +pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, + int layer_start, int layer_end) +{ + return mlx_load_range(model_dir, layer_start, layer_end); +} + /* ======================================================================== * Free * ======================================================================== */ @@ -353,7 +389,9 @@ static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, int batch, float *out_embeddings) { - if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; + if (!ctx || !inputs || batch <= 0 || !out_embeddings || + ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) + return -1; const pplx_config_t *c = &ctx->config; mlx_stream S = ctx->stream; @@ -426,7 +464,7 @@ int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *inputs, mlx_array_free(lengths); return -1; } - if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) { + if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { mlx_array attn_mask_bf16 = mlx_array_new(); mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); mlx_array_free(attn_mask); @@ -501,9 +539,12 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, if (!ctx || !inputs || batch <= 0 || !out_states || layer_start < 0 || layer_start > layer_end || layer_end > ctx->config.n_layers || + layer_start < ctx->layer_start || layer_end > ctx->layer_end || (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) || - (apply_final_norm && layer_end != ctx->config.n_layers)) + (layer_start == 0 && !arr_ok(ctx->embed_tokens)) || + (apply_final_norm && + (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm)))) return -1; const pplx_config_t *c = &ctx->config; @@ -581,7 +622,7 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, free(padded_ids); free(padded_states); return -1; } - if (has_padding && mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) { + if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { mlx_array attn_mask_bf16 = mlx_array_new(); mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); mlx_array_free(attn_mask); @@ -613,7 +654,7 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, if (has_padding) mlx_array_free(attn_mask); return -1; } - if (mlx_array_dtype(ctx->embed_tokens) == MLX_BFLOAT16) { + if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { x = mlx_array_new(); mlx_astype(&x, states, MLX_BFLOAT16, S); mlx_array_free(states); diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h index 1ec6fab..b45fc8a 100644 --- a/pplx_embed_mlx.h +++ b/pplx_embed_mlx.h @@ -20,6 +20,14 @@ typedef struct pplx_mlx_ctx pplx_mlx_ctx_t; */ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); +/* + * Load one contiguous transformer layer range for distributed execution. + * Uses the same half-open range and endpoint ownership rules as + * pplx_model_load_slice(). + */ +pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, + int layer_start, int layer_end); + void pplx_mlx_free(pplx_mlx_ctx_t *ctx); /* |
