aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-11 15:34:50 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit6b3c4bb1482917e4fb24480a1d082da5f191467b (patch)
tree6a29d58959d1c5d3a0febcc9cb06005ac5a009ed
parent54e6e6bbef396eb1d38c09517d8157301fb8f151 (diff)
downloadffwd-6b3c4bb1482917e4fb24480a1d082da5f191467b.tar.gz
ffwd-6b3c4bb1482917e4fb24480a1d082da5f191467b.zip
dist: remove distributed layer-sharding (protocol, CLI flags, slice APIs)
-rw-r--r--Makefile10
-rw-r--r--embed.c42
-rw-r--r--embed.h32
-rw-r--r--embed_cli.c170
-rw-r--r--embed_distributed.c888
-rw-r--r--embed_distributed.h58
-rw-r--r--embed_mlx.c186
-rw-r--r--embed_mlx.h24
-rw-r--r--tests/test_cli.c12
9 files changed, 19 insertions, 1403 deletions
diff --git a/Makefile b/Makefile
index ec3c77f..46d1abc 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,6 @@ UNAME_S := $(shell uname -s)
# Source files
SRCS = embed.c \
- embed_distributed.c \
qwen_kernels.c \
qwen_kernels_generic.c \
qwen_kernels_neon.c \
@@ -167,11 +166,11 @@ TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c qwen_tokenizer.c \
qwen_safetensors.c $(KERNEL_SRCS)
TEST_LATE_SRCS = tests/test_late.c embed.c qwen_tokenizer.c \
qwen_safetensors.c $(KERNEL_SRCS)
-TEST_SERVER_SRCS = tests/test_server.c embed.c embed_distributed.c \
+TEST_SERVER_SRCS = tests/test_server.c embed.c \
qwen_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) \
deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c
# The CLI check builds the real CLI (CPU backend) plus a driver that runs it.
-TEST_CLI_BIN_SRCS = embed_cli.c embed.c embed_distributed.c \
+TEST_CLI_BIN_SRCS = embed_cli.c embed.c \
qwen_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS)
test:
@@ -335,16 +334,13 @@ $(SERVER_TARGET): $(LIB) embed_server.o
embed.o: embed.c embed.h qwen_kernels.h qwen_safetensors.h
$(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
-embed_distributed.o: embed_distributed.c embed_distributed.h embed.h embed_mlx.h
- $(CC) $(CFLAGS) -c -o $@ $<
-
embed_server.o: embed_server.c embed_server.h embed.h embed_mlx.h qwen_tokenizer.h deps/ae/ae.h deps/ae/anet.h
$(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
embed_mlx.o: embed_mlx.c embed_mlx.h embed.h qwen_safetensors.h
$(CC) $(CFLAGS) -c -o $@ $<
-embed_cli.o: embed_cli.c embed.h embed_distributed.h qwen_kernels.h qwen_tokenizer.h
+embed_cli.o: embed_cli.c embed.h qwen_kernels.h qwen_tokenizer.h
$(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
diff --git a/embed.c b/embed.c
index f2b3724..76ca4e2 100644
--- a/embed.c
+++ b/embed.c
@@ -717,12 +717,6 @@ pplx_model_t *pplx_model_load(const char *model_dir)
return model_load_range_ex(model_dir, 0, -1, 0);
}
-pplx_model_t *pplx_model_load_slice(const char *model_dir,
- int layer_start, int layer_end)
-{
- return model_load_range_ex(model_dir, layer_start, layer_end, 0);
-}
-
void pplx_model_free(pplx_model_t *model)
{
if (!model) return;
@@ -1442,42 +1436,6 @@ int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws,
return rc;
}
-int pplx_model_forward_slice_batch(const pplx_model_t *model,
- pplx_workspace_t *ws,
- const pplx_input_t *inputs, int batch,
- const float *input_states,
- int layer_start, int layer_end,
- int apply_final_norm,
- float *out_states)
-{
- if (!model || !ws || !inputs || batch <= 0 || !out_states ||
- layer_start < 0 || layer_start > layer_end ||
- layer_end > model->config.n_layers)
- return -1;
-
- if (ensure_offsets(ws, batch) != 0) return -1;
- int *offsets = ws->offsets;
-
- int total_seq = 0;
- int max_seq = 0;
- if (build_offsets(inputs, batch, layer_start == 0, offsets,
- &total_seq, &max_seq) != 0)
- return -1;
-
- int rc = forward_packed_slice_inplace(model, ws, inputs, batch, offsets,
- total_seq, max_seq, input_states,
- layer_start, layer_end,
- apply_final_norm);
- if (rc == 0) {
- size_t count;
- if (mul_size((size_t)total_seq,
- (size_t)model->config.hidden_size, &count) != 0)
- return -1;
- memcpy(out_states, ws->x, count * sizeof(float));
- }
- return rc;
-}
-
int pplx_pool_batch(const pplx_config_t *cfg, const float *states,
const int *seq_lengths, int batch,
float *out_embeddings)
diff --git a/embed.h b/embed.h
index 2098e9e..37f58d5 100644
--- a/embed.h
+++ b/embed.h
@@ -88,17 +88,6 @@ typedef struct {
*/
PPLX_API pplx_model_t *pplx_model_load(const char *model_dir);
-/*
- * Load one contiguous transformer layer range for distributed execution.
- *
- * layer_start is inclusive and layer_end is exclusive. A range beginning at
- * zero also loads the token embedding table. A range ending at n_layers also
- * loads the final RMSNorm. Safetensors remain mmap-backed, so unselected
- * weights are not touched.
- */
-PPLX_API pplx_model_t *pplx_model_load_slice(const char *model_dir,
- int layer_start, int layer_end);
-
PPLX_API void pplx_model_free(pplx_model_t *model);
/*
@@ -158,27 +147,6 @@ PPLX_API int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t
float *out_embeddings);
/*
- * Run a contiguous transformer layer range for a packed/ragged batch.
- *
- * layer_start is inclusive and layer_end is exclusive. For a range beginning
- * at layer 0, input_states must be NULL and every input must provide token
- * ids. For a later range, input_states must contain the preceding range's
- * packed [sum(n_tokens), hidden_size] output; token ids are not required.
- *
- * apply_final_norm is valid only for a range ending at the final transformer
- * layer. out_states receives packed [sum(n_tokens), hidden_size] states.
- *
- * This is the core execution boundary used by layer-sharded inference.
- */
-PPLX_API int pplx_model_forward_slice_batch(const pplx_model_t *model,
- pplx_workspace_t *ws,
- const pplx_input_t *inputs, int batch,
- const float *input_states,
- int layer_start, int layer_end,
- int apply_final_norm,
- float *out_states);
-
-/*
* Mean-pool packed final hidden states for a ragged batch.
*
* states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive
diff --git a/embed_cli.c b/embed_cli.c
index ad14857..10f992d 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -1,7 +1,6 @@
/* embed_cli.c - pplx-embed command-line tool */
#include "embed.h"
-#include "embed_distributed.h"
#include "qwen_kernels.h"
#include "qwen_tokenizer.h"
@@ -35,13 +34,6 @@ static void print_usage(const char *prog)
" --mlx Use Apple MLX GPU backend\n"
" --cuda Use CUDA/cuBLAS GPU backend\n"
" --stream Read lines from stdin, write one JSON embedding per line\n"
- " --dist-worker\n"
- " Serve one final distributed layer shard\n"
- " --dist-remote HOST:PORT\n"
- " Use one remote final shard for CLI or --stream inference\n"
- " --layers A:B Distributed half-open layer range, for example 0:14\n"
- " --dist-activation-bits N\n"
- " Distributed activation transport width: 32 or 16 (default: 32)\n"
" --mlx-quant-bits N\n"
" Quantize MLX linear weights to 8 bits at load time\n"
" --mlx-quant-group-size N\n"
@@ -51,8 +43,6 @@ static void print_usage(const char *prog)
" --cuda-weight-dtype DTYPE\n"
" CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
" weight memory and uses BF16 tensor cores\n"
- " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n"
- " --port N Bind port for --dist-worker (default: 8000)\n"
" -b, --batch-size N\n"
" Max texts per engine batch (default: all; --stream: 1)\n"
" -t, --threads N\n"
@@ -68,14 +58,11 @@ static void print_usage(const char *prog)
" 2+ text args Print cosine similarity matrix\n"
" no args Batch: read all stdin lines, then similarity matrix\n"
" --stream Streaming: read stdin lines, write JSON per line\n"
- " --dist-worker Final layer-shard worker\n"
"\n"
"Examples:\n"
" %s -d ./model \"query: what is AI?\"\n"
- " %s -d ./model --mlx --stream < texts.txt\n"
- " %s -d ./model --dist-worker --layers 14:28 --port 9000\n"
- " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n",
- prog, prog, prog, prog, prog);
+ " %s -d ./model --mlx --stream < texts.txt\n",
+ prog, prog, prog);
}
/* ========================================================================
@@ -85,7 +72,6 @@ static void print_usage(const char *prog)
typedef struct {
pplx_model_t *model;
pplx_workspace_t *workspace;
- pplx_dist_client_t *dist;
#ifdef USE_MLX
pplx_mlx_ctx_t *mlx_ctx;
#endif
@@ -105,9 +91,6 @@ typedef struct {
static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs,
int batch, float *out_embeddings)
{
- if (e->dist)
- return pplx_dist_client_embed_batch(e->dist, inputs, batch,
- out_embeddings);
#ifdef USE_MLX
if (e->mlx_ctx)
return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings);
@@ -120,30 +103,6 @@ static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs,
inputs, batch, out_embeddings);
}
-static int parse_layers(const char *arg, int *start, int *end)
-{
- int used = 0;
- if (!arg || sscanf(arg, "%d:%d%n", start, end, &used) != 2 ||
- arg[used] != '\0' || *start < 0 || *start >= *end)
- return -1;
- return 0;
-}
-
-static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port)
-{
- const char *colon = arg ? strrchr(arg, ':') : NULL;
- if (!colon || colon == arg || !colon[1]) return -1;
- size_t n = (size_t)(colon - arg);
- if (n >= hostlen) return -1;
- memcpy(host, arg, n);
- host[n] = '\0';
- int used = 0;
- if (sscanf(colon + 1, "%d%n", port, &used) != 1 ||
- colon[1 + used] != '\0' || *port <= 0 || *port > 65535)
- return -1;
- return 0;
-}
-
static int tokenize_text(engine_t *e, const char *text, token_buf_t *out)
{
memset(out, 0, sizeof(*out));
@@ -474,18 +433,11 @@ int main(int argc, char *argv[])
int use_mlx = 0;
int use_cuda = 0;
int stdin_mode = 0;
- int dist_worker_mode = 0;
- const char *dist_remote = NULL;
- int layer_start = -1;
- int layer_end = -1;
int batch_size = 0;
int mlx_quantize_bits = 0;
int mlx_quantize_group_size = 64;
const char *cuda_fast_gemm = NULL;
const char *cuda_weights = NULL;
- int dist_activation_bits = 32;
- const char *host = "127.0.0.1";
- int port = 8000;
int arg_start = 1;
while (arg_start < argc && argv[arg_start][0] == '-') {
@@ -543,29 +495,12 @@ int main(int argc, char *argv[])
else if (!strcmp(f, "--cuda-weight-dtype")) {
cuda_weights = argv[++arg_start];
}
- else if (!strcmp(f, "--host")) { host = argv[++arg_start]; }
- else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); }
- else if (!strcmp(f, "--layers")) {
- if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) {
- fprintf(stderr, "--layers expects START:END with START < END\n");
- return 1;
- }
- }
- else if (!strcmp(f, "--dist-activation-bits")) {
- dist_activation_bits = atoi(argv[++arg_start]);
- if (dist_activation_bits != 32 && dist_activation_bits != 16) {
- fprintf(stderr, "--dist-activation-bits must be 32 or 16\n");
- return 1;
- }
- }
else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
print_embs = 1;
}
else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
else if (!strcmp(f, "-vv")) { verbose = 2; }
else if (!strcmp(f, "--stream")) { stdin_mode = 1; }
- else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; }
- else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; }
else if (!strcmp(f, "--mlx")) {
#ifdef USE_MLX
use_mlx = 1;
@@ -637,28 +572,6 @@ int main(int argc, char *argv[])
return 1;
}
- if (dist_worker_mode && dist_remote) {
- fprintf(stderr, "--dist-worker and --dist-remote are mutually exclusive\n");
- return 1;
- }
- if ((dist_worker_mode || dist_remote) &&
- (layer_start < 0 || layer_end < 0)) {
- fprintf(stderr, "distributed mode requires --layers START:END\n");
- return 1;
- }
- if (dist_activation_bits != 32 && !dist_remote) {
- fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n");
- return 1;
- }
- if (dist_remote && layer_start != 0) {
- fprintf(stderr, "--dist-remote coordinator layers must start at zero\n");
- return 1;
- }
- if ((dist_worker_mode || dist_remote) && use_cuda) {
- fprintf(stderr, "distributed mode is not available with --cuda yet\n");
- return 1;
- }
-
/* Threads (CPU backend) */
if (!use_mlx && !use_cuda) {
if (n_threads <= 0) n_threads = qwen_get_num_cpus();
@@ -674,17 +587,6 @@ int main(int argc, char *argv[])
return 1;
}
- pplx_dist_options_t dist_opts = {
- .mlx_quantize_bits = mlx_quantize_bits,
- .mlx_quantize_group_size = mlx_quantize_group_size,
- .activation_bits = dist_activation_bits,
- };
-
- if (dist_worker_mode)
- return pplx_dist_run_worker_with_options(
- model_dir, host, port, layer_start, layer_end, use_mlx,
- &dist_opts);
-
/* Tokenizer */
char vocab_path[1024];
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
@@ -699,25 +601,7 @@ int main(int argc, char *argv[])
t0 = now_ms();
#ifdef USE_MLX
- if (dist_remote) {
- char remote_host[1024];
- int remote_port = 0;
- if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host),
- &remote_port) != 0) {
- fprintf(stderr, "--dist-remote expects HOST:PORT\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dist = pplx_dist_client_open_with_options(
- model_dir, remote_host, remote_port, layer_end, use_mlx,
- &dist_opts);
- if (!e.dist) {
- fprintf(stderr, "failed to connect distributed worker\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_dist_client_config(e.dist)->hidden_size;
- } else if (use_mlx) {
+ if (use_mlx) {
pplx_mlx_options_t mlx_opts = {
.quantize_bits = mlx_quantize_bits,
.quantize_group_size = mlx_quantize_group_size,
@@ -743,40 +627,20 @@ int main(int argc, char *argv[])
} else
#endif
{
- if (dist_remote) {
- char remote_host[1024];
- int remote_port = 0;
- if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host),
- &remote_port) != 0) {
- fprintf(stderr, "--dist-remote expects HOST:PORT\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dist = pplx_dist_client_open_with_options(
- model_dir, remote_host, remote_port, layer_end, use_mlx,
- &dist_opts);
- if (!e.dist) {
- fprintf(stderr, "failed to connect distributed worker\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_dist_client_config(e.dist)->hidden_size;
- } else {
- e.model = pplx_model_load(model_dir);
- if (!e.model) {
- fprintf(stderr, "failed to load model\n");
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.workspace = pplx_workspace_new(e.model);
- if (!e.workspace) {
- fprintf(stderr, "failed to allocate workspace\n");
- pplx_model_free(e.model);
- qwen_tokenizer_free(tok);
- return 1;
- }
- e.dim = pplx_model_config(e.model)->hidden_size;
+ e.model = pplx_model_load(model_dir);
+ if (!e.model) {
+ fprintf(stderr, "failed to load model\n");
+ qwen_tokenizer_free(tok);
+ return 1;
+ }
+ e.workspace = pplx_workspace_new(e.model);
+ if (!e.workspace) {
+ fprintf(stderr, "failed to allocate workspace\n");
+ pplx_model_free(e.model);
+ qwen_tokenizer_free(tok);
+ return 1;
}
+ e.dim = pplx_model_config(e.model)->hidden_size;
}
if (verbose >= 1)
fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n",
@@ -789,7 +653,6 @@ int main(int argc, char *argv[])
fprintf(stderr, "failed to allocate tokenizer workspace\n");
if (e.workspace) pplx_workspace_free(e.workspace);
if (e.model) pplx_model_free(e.model);
- if (e.dist) pplx_dist_client_free(e.dist);
#ifdef USE_MLX
if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
#endif
@@ -810,7 +673,6 @@ int main(int argc, char *argv[])
/* Cleanup */
if (e.workspace) pplx_workspace_free(e.workspace);
if (e.model) pplx_model_free(e.model);
- if (e.dist) pplx_dist_client_free(e.dist);
#ifdef USE_MLX
if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx);
#endif
diff --git a/embed_distributed.c b/embed_distributed.c
deleted file mode 100644
index 92d22c1..0000000
--- a/embed_distributed.c
+++ /dev/null
@@ -1,888 +0,0 @@
-/*
- * embed_distributed.c - Stateless two-node layer-sharded inference.
- *
- * This first transport is deliberately synchronous: one coordinator sends a
- * packed hidden-state batch to one final worker and receives pooled
- * embeddings. The protocol is bounded and versioned so later route discovery,
- * retries, and richer transport can extend it without changing model math.
- */
-
-#include "embed_distributed.h"
-
-#ifdef USE_MLX
-#include "embed_mlx.h"
-#endif
-
-#include <arpa/inet.h>
-#include <errno.h>
-#include <limits.h>
-#include <netdb.h>
-#include <netinet/tcp.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/socket.h>
-#include <sys/time.h>
-#include <unistd.h>
-
-#define PPLX_DIST_MAGIC UINT32_C(0x504c5844) /* "PLXD" */
-#define PPLX_DIST_VERSION 2u
-
-#define PPLX_DIST_MSG_HELLO 1u
-#define PPLX_DIST_MSG_WORK 2u
-#define PPLX_DIST_MSG_RESULT 3u
-
-#define PPLX_DIST_MAX_FRAME_BYTES (512u * 1024u * 1024u)
-#define PPLX_DIST_MAX_BATCH 512u
-#define PPLX_DIST_MAX_TOKENS 120000u
-#define PPLX_DIST_MAX_ERROR_BYTES 1024u
-#define PPLX_DIST_SOCKET_TIMEOUT_SEC 60
-#define PPLX_DIST_ACTIVATION_BITS_DEFAULT 32u
-
-typedef struct {
- uint32_t magic;
- uint32_t version;
- uint32_t type;
- uint32_t body_bytes;
-} dist_frame_header;
-
-typedef struct {
- uint32_t signature_hi;
- uint32_t signature_lo;
- uint32_t hidden_size;
- uint32_t n_layers;
- uint32_t vocab_size;
- uint32_t n_heads;
- uint32_t n_kv_heads;
- uint32_t head_dim;
- uint32_t intermediate_size;
- uint32_t layer_start;
- uint32_t layer_end;
-} dist_hello;
-
-typedef struct {
- uint32_t request_hi;
- uint32_t request_lo;
- uint32_t batch;
- uint32_t total_tokens;
- uint32_t hidden_size;
- uint32_t lengths_bytes;
- uint32_t states_bytes;
- uint32_t states_bits;
-} dist_work;
-
-typedef struct {
- uint32_t request_hi;
- uint32_t request_lo;
- uint32_t status;
- uint32_t batch;
- uint32_t dimensions;
- uint32_t payload_bytes;
-} dist_result;
-
-typedef struct {
- pplx_model_t *model;
- pplx_workspace_t *workspace;
-#ifdef USE_MLX
- pplx_mlx_ctx_t *mlx;
-#endif
- const pplx_config_t *cfg;
-} dist_stage;
-
-struct pplx_dist_client {
- dist_stage local;
- int fd;
- int local_layer_end;
- uint64_t next_request_id;
- uint32_t activation_bits;
-};
-
-static int host_is_little_endian(void)
-{
- const uint16_t one = 1;
- return *(const unsigned char *)&one == 1;
-}
-
-static uint64_t fnv_u32(uint64_t h, uint32_t v)
-{
- for (int i = 0; i < 4; i++) {
- h ^= (unsigned char)(v >> (i * 8));
- h *= UINT64_C(1099511628211);
- }
- return h;
-}
-
-static uint64_t config_signature(const pplx_config_t *cfg)
-{
- uint64_t h = UINT64_C(1469598103934665603);
- uint32_t eps = 0, theta = 0;
- memcpy(&eps, &cfg->rms_norm_eps, sizeof(eps));
- memcpy(&theta, &cfg->rope_theta, sizeof(theta));
- h = fnv_u32(h, (uint32_t)cfg->hidden_size);
- h = fnv_u32(h, (uint32_t)cfg->n_layers);
- h = fnv_u32(h, (uint32_t)cfg->n_heads);
- h = fnv_u32(h, (uint32_t)cfg->n_kv_heads);
- h = fnv_u32(h, (uint32_t)cfg->head_dim);
- h = fnv_u32(h, (uint32_t)cfg->intermediate_size);
- h = fnv_u32(h, (uint32_t)cfg->vocab_size);
- h = fnv_u32(h, eps);
- h = fnv_u32(h, theta);
- return h;
-}
-
-static void u64_to_halves(uint64_t v, uint32_t *hi, uint32_t *lo)
-{
- *hi = (uint32_t)(v >> 32);
- *lo = (uint32_t)v;
-}
-
-static uint64_t u64_from_halves(uint32_t hi, uint32_t lo)
-{
- return ((uint64_t)hi << 32) | lo;
-}
-
-static int mul_u32(uint32_t a, uint32_t b, uint32_t *out)
-{
- if (a != 0 && b > UINT32_MAX / a) return -1;
- *out = a * b;
- return 0;
-}
-
-static uint32_t activation_bits_or_default(uint32_t bits)
-{
- return bits ? bits : PPLX_DIST_ACTIVATION_BITS_DEFAULT;
-}
-
-static int activation_bits_valid(uint32_t bits)
-{
- bits = activation_bits_or_default(bits);
- return bits == 32u || bits == 16u;
-}
-
-static int activation_wire_bytes(uint32_t bits, uint64_t values, uint32_t *out)
-{
- bits = activation_bits_or_default(bits);
- if (!activation_bits_valid(bits)) return -1;
- uint64_t bytes = values * (uint64_t)(bits / 8u);
- if (bytes > UINT32_MAX) return -1;
- if (out) *out = (uint32_t)bytes;
- return 0;
-}
-
-static uint16_t f32_to_f16(float f)
-{
- uint32_t bits;
- memcpy(&bits, &f, sizeof(bits));
-
- uint32_t sign = (bits >> 16) & 0x8000u;
- int32_t exp = (int32_t)((bits >> 23) & 0xffu) - 127 + 15;
- uint32_t mant = bits & 0x7fffffu;
-
- if (exp <= 0) {
- if (exp < -10) return (uint16_t)sign;
- mant |= 0x800000u;
- uint32_t shift = (uint32_t)(14 - exp);
- uint32_t half_mant = mant >> shift;
- uint32_t round_bit = (mant >> (shift - 1)) & 1u;
- uint32_t sticky = mant & ((1u << (shift - 1)) - 1u);
- if (round_bit && (sticky || (half_mant & 1u))) half_mant++;
- return (uint16_t)(sign | half_mant);
- }
-
- if (exp >= 31) {
- if (((bits >> 23) & 0xffu) == 0xffu && mant != 0)
- return (uint16_t)(sign | 0x7e00u);
- return (uint16_t)(sign | 0x7c00u);
- }
-
- uint32_t half = sign | ((uint32_t)exp << 10) | (mant >> 13);
- uint32_t round = mant & 0x1fffu;
- if (round > 0x1000u || (round == 0x1000u && (half & 1u))) half++;
- return (uint16_t)half;
-}
-
-static float f16_to_f32(uint16_t h)
-{
- uint32_t sign = (uint32_t)(h & 0x8000u) << 16;
- int32_t exp = (int32_t)((h >> 10) & 0x1fu);
- uint32_t mant = h & 0x03ffu;
- uint32_t bits;
-
- if (exp == 0) {
- if (mant == 0) {
- bits = sign;
- } else {
- exp = 1;
- while ((mant & 0x0400u) == 0) {
- mant <<= 1;
- exp--;
- }
- mant &= 0x03ffu;
- bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13);
- }
- } else if (exp == 31) {
- bits = sign | 0x7f800000u | (mant << 13);
- } else {
- bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13);
- }
-
- float f;
- memcpy(&f, &bits, sizeof(f));
- return f;
-}
-
-static int encode_activations(void *dst, const float *src, uint64_t values,
- uint32_t bits)
-{
- bits = activation_bits_or_default(bits);
- if (!dst || !src || !activation_bits_valid(bits)) return -1;
- if (bits == 32u) {
- memcpy(dst, src, values * sizeof(float));
- return 0;
- }
- uint16_t *out = dst;
- for (uint64_t i = 0; i < values; i++) out[i] = f32_to_f16(src[i]);
- return 0;
-}
-
-static int decode_activations(float *dst, const void *wire, uint64_t values,
- uint32_t bits)
-{
- bits = activation_bits_or_default(bits);
- if (!dst || !wire || !activation_bits_valid(bits)) return -1;
- if (bits == 32u) {
- memcpy(dst, wire, values * sizeof(float));
- return 0;
- }
- const uint16_t *src = wire;
- for (uint64_t i = 0; i < values; i++) dst[i] = f16_to_f32(src[i]);
- return 0;
-}
-
-static void frame_to_wire(dist_frame_header *h)
-{
- h->magic = htonl(h->magic);
- h->version = htonl(h->version);
- h->type = htonl(h->type);
- h->body_bytes = htonl(h->body_bytes);
-}
-
-static void hello_to_wire(dist_hello *h)
-{
- uint32_t *v = (uint32_t *)h;
- for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = htonl(v[i]);
-}
-
-static void hello_from_wire(dist_hello *h)
-{
- uint32_t *v = (uint32_t *)h;
- for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = ntohl(v[i]);
-}
-
-static void work_to_wire(dist_work *w)
-{
- uint32_t *v = (uint32_t *)w;
- for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = htonl(v[i]);
-}
-
-static void work_from_wire(dist_work *w)
-{
- uint32_t *v = (uint32_t *)w;
- for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = ntohl(v[i]);
-}
-
-static void result_to_wire(dist_result *r)
-{
- uint32_t *v = (uint32_t *)r;
- for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = htonl(v[i]);
-}
-
-static void result_from_wire(dist_result *r)
-{
- uint32_t *v = (uint32_t *)r;
- for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = ntohl(v[i]);
-}
-
-static int write_full(int fd, const void *buf, size_t len)
-{
- const unsigned char *p = buf;
- while (len) {
-#ifdef MSG_NOSIGNAL
- ssize_t n = send(fd, p, len, MSG_NOSIGNAL);
-#else
- ssize_t n = send(fd, p, len, 0);
-#endif
- if (n < 0 && errno == EINTR) continue;
- if (n <= 0) return -1;
- p += (size_t)n;
- len -= (size_t)n;
- }
- return 0;
-}
-
-static int read_full(int fd, void *buf, size_t len)
-{
- unsigned char *p = buf;
- while (len) {
- ssize_t n = recv(fd, p, len, 0);
- if (n < 0 && errno == EINTR) continue;
- if (n <= 0) return -1;
- p += (size_t)n;
- len -= (size_t)n;
- }
- return 0;
-}
-
-static int write_frame_header(int fd, uint32_t type, uint32_t body_bytes)
-{
- dist_frame_header h = {PPLX_DIST_MAGIC, PPLX_DIST_VERSION, type, body_bytes};
- frame_to_wire(&h);
- return write_full(fd, &h, sizeof(h));
-}
-
-static int read_frame_header(int fd, uint32_t *type, uint32_t *body_bytes)
-{
- dist_frame_header h;
- if (read_full(fd, &h, sizeof(h)) != 0) return -1;
- h.magic = ntohl(h.magic);
- h.version = ntohl(h.version);
- h.type = ntohl(h.type);
- h.body_bytes = ntohl(h.body_bytes);
- if (h.magic != PPLX_DIST_MAGIC || h.version != PPLX_DIST_VERSION ||
- h.body_bytes > PPLX_DIST_MAX_FRAME_BYTES)
- return -1;
- *type = h.type;
- *body_bytes = h.body_bytes;
- return 0;
-}
-
-static void set_socket_options(int fd)
-{
- int yes = 1;
- struct timeval tv = {PPLX_DIST_SOCKET_TIMEOUT_SEC, 0};
- setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes));
- setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes));
- setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
- setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
-#ifdef SO_NOSIGPIPE
- setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &yes, sizeof(yes));
-#endif
-}
-
-static int open_listener(const char *host, int port)
-{
- char service[16];
- snprintf(service, sizeof(service), "%d", port);
- struct addrinfo hints;
- memset(&hints, 0, sizeof(hints));
- hints.ai_family = AF_UNSPEC;
- hints.ai_socktype = SOCK_STREAM;
- hints.ai_flags = AI_PASSIVE;
- struct addrinfo *list = NULL;
- if (getaddrinfo(host, service, &hints, &list) != 0) return -1;
-
- int listen_fd = -1;
- for (struct addrinfo *ai = list; ai; ai = ai->ai_next) {
- int fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
- if (fd < 0) continue;
- int yes = 1;
- setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
- if (bind(fd, ai->ai_addr, ai->ai_addrlen) == 0 &&
- listen(fd, 16) == 0) {
- listen_fd = fd;
- break;
- }
- close(fd);
- }
- freeaddrinfo(list);
- return listen_fd;
-}
-
-static int connect_endpoint(const char *host, int port)
-{
- char service[16];
- snprintf(service, sizeof(service), "%d", port);
- struct addrinfo hints;
- memset(&hints, 0, sizeof(hints));
- hints.ai_family = AF_UNSPEC;
- hints.ai_socktype = SOCK_STREAM;
- struct addrinfo *list = NULL;
- if (getaddrinfo(host, service, &hints, &list) != 0) return -1;
-
- int fd = -1;
- for (struct addrinfo *ai = list; ai; ai = ai->ai_next) {
- fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
- if (fd < 0) continue;
- set_socket_options(fd);
- if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) break;
- close(fd);
- fd = -1;
- }
- freeaddrinfo(list);
- return fd;
-}
-
-static int stage_load(dist_stage *stage, const char *model_dir,
- int layer_start, int layer_end, int use_mlx,
- const pplx_dist_options_t *opts)
-{
- memset(stage, 0, sizeof(*stage));
-#ifndef USE_MLX
- (void)opts;
-#endif
-#ifdef USE_MLX
- if (use_mlx) {
- pplx_mlx_options_t mlx_opts = {0};
- if (opts) {
- mlx_opts.quantize_bits = opts->mlx_quantize_bits;
- mlx_opts.quantize_group_size = opts->mlx_quantize_group_size;
- }
- stage->mlx = pplx_mlx_load_slice_with_options(
- model_dir, layer_start, layer_end, &mlx_opts);
- if (!stage->mlx) return -1;
- stage->cfg = pplx_mlx_config(stage->mlx);
- return 0;
- }
-#else
- if (use_mlx) return -1;
-#endif
- stage->model = pplx_model_load_slice(model_dir, layer_start, layer_end);
- if (!stage->model) return -1;
- stage->workspace = pplx_workspace_new(stage->model);
- if (!stage->workspace) {
- pplx_model_free(stage->model);
- stage->model = NULL;
- return -1;
- }
- stage->cfg = pplx_model_config(stage->model);
- return 0;
-}
-
-static void stage_free(dist_stage *stage)
-{
- if (!stage) return;
- if (stage->workspace) pplx_workspace_free(stage->workspace);
- if (stage->model) pplx_model_free(stage->model);
-#ifdef USE_MLX
- if (stage->mlx) pplx_mlx_free(stage->mlx);
-#endif
- memset(stage, 0, sizeof(*stage));
-}
-
-static int stage_forward(dist_stage *stage, const pplx_input_t *inputs,
- int batch, const float *input_states,
- int layer_start, int layer_end, int apply_final_norm,
- float *out_states)
-{
-#ifdef USE_MLX
- if (stage->mlx)
- return pplx_mlx_forward_slice_batch(stage->mlx, inputs, batch,
- input_states, layer_start,
- layer_end, apply_final_norm,
- out_states);
-#endif
- return pplx_model_forward_slice_batch(stage->model, stage->workspace,
- inputs, batch, input_states,
- layer_start, layer_end,
- apply_final_norm, out_states);
-}
-
-static int send_hello(int fd, const pplx_config_t *cfg,
- int layer_start, int layer_end)
-{
- uint64_t sig = config_signature(cfg);
- dist_hello h;
- u64_to_halves(sig, &h.signature_hi, &h.signature_lo);
- h.hidden_size = (uint32_t)cfg->hidden_size;
- h.n_layers = (uint32_t)cfg->n_layers;
- h.vocab_size = (uint32_t)cfg->vocab_size;
- h.n_heads = (uint32_t)cfg->n_heads;
- h.n_kv_heads = (uint32_t)cfg->n_kv_heads;
- h.head_dim = (uint32_t)cfg->head_dim;
- h.intermediate_size = (uint32_t)cfg->intermediate_size;
- h.layer_start = (uint32_t)layer_start;
- h.layer_end = (uint32_t)layer_end;
- hello_to_wire(&h);
- if (write_frame_header(fd, PPLX_DIST_MSG_HELLO, sizeof(h)) != 0)
- return -1;
- return write_full(fd, &h, sizeof(h));
-}
-
-static int recv_hello(int fd, const pplx_config_t *cfg, int expected_start)
-{
- uint32_t type = 0, bytes = 0;
- dist_hello h;
- if (read_frame_header(fd, &type, &bytes) != 0 ||
- type != PPLX_DIST_MSG_HELLO || bytes != sizeof(h) ||
- read_full(fd, &h, sizeof(h)) != 0)
- return -1;
- hello_from_wire(&h);
- if (u64_from_halves(h.signature_hi, h.signature_lo) !=
- config_signature(cfg) ||
- h.hidden_size != (uint32_t)cfg->hidden_size ||
- h.n_layers != (uint32_t)cfg->n_layers ||
- h.layer_start != (uint32_t)expected_start ||
- h.layer_end != (uint32_t)cfg->n_layers)
- return -1;
- return 0;
-}
-
-static int send_error_result(int fd, uint64_t request_id, const char *msg)
-{
- size_t len = msg ? strlen(msg) : 0;
- if (len > PPLX_DIST_MAX_ERROR_BYTES) len = PPLX_DIST_MAX_ERROR_BYTES;
- dist_result result;
- memset(&result, 0, sizeof(result));
- u64_to_halves(request_id, &result.request_hi, &result.request_lo);
- result.status = 1;
- result.payload_bytes = (uint32_t)len;
- dist_result wire = result;
- result_to_wire(&wire);
- if (write_frame_header(fd, PPLX_DIST_MSG_RESULT,
- (uint32_t)sizeof(wire) + (uint32_t)len) != 0 ||
- write_full(fd, &wire, sizeof(wire)) != 0)
- return -1;
- return len ? write_full(fd, msg, len) : 0;
-}
-
-static int worker_process_work(int fd, dist_stage *stage,
- int layer_start, int layer_end,
- const void *body, uint32_t body_bytes)
-{
- uint64_t request_id = 0;
- if (body_bytes < sizeof(dist_work))
- return send_error_result(fd, request_id, "truncated WORK frame");
-
- dist_work work;
- memcpy(&work, body, sizeof(work));
- work_from_wire(&work);
- request_id = u64_from_halves(work.request_hi, work.request_lo);
-
- uint32_t expected_lengths = 0, expected_state_values = 0;
- uint32_t expected_state_f32_bytes = 0, expected_state_wire_bytes = 0;
- uint32_t states_bits = activation_bits_or_default(work.states_bits);
- if (work.batch == 0 || work.batch > PPLX_DIST_MAX_BATCH ||
- work.total_tokens == 0 || work.total_tokens > PPLX_DIST_MAX_TOKENS ||
- work.hidden_size != (uint32_t)stage->cfg->hidden_size ||
- !activation_bits_valid(states_bits) ||
- mul_u32(work.batch, sizeof(uint32_t), &expected_lengths) != 0 ||
- mul_u32(work.total_tokens, work.hidden_size,
- &expected_state_values) != 0 ||
- mul_u32(expected_state_values, sizeof(float),
- &expected_state_f32_bytes) != 0 ||
- activation_wire_bytes(states_bits, expected_state_values,
- &expected_state_wire_bytes) != 0 ||
- work.lengths_bytes != expected_lengths ||
- work.states_bytes != expected_state_wire_bytes)
- return send_error_result(fd, request_id, "invalid WORK frame");
- uint64_t expected_body = (uint64_t)sizeof(work) + expected_lengths +
- expected_state_wire_bytes;
- if (expected_body > UINT32_MAX || body_bytes != (uint32_t)expected_body)
- return send_error_result(fd, request_id, "invalid WORK frame");
-
- const unsigned char *p = (const unsigned char *)body + sizeof(work);
- int *lengths = malloc((size_t)work.batch * sizeof(*lengths));
- pplx_input_t *inputs = calloc(work.batch, sizeof(*inputs));
- float *states = malloc(expected_state_f32_bytes);
- float *out_states = malloc(expected_state_f32_bytes);
- uint32_t emb_bytes = 0;
- if (mul_u32(work.batch, work.hidden_size, &emb_bytes) != 0 ||
- mul_u32(emb_bytes, sizeof(float), &emb_bytes) != 0) {
- free(lengths); free(inputs); free(states); free(out_states);
- return send_error_result(fd, request_id, "embedding result is too large");
- }
- float *embeddings = malloc(emb_bytes);
- if (!lengths || !inputs || !states || !out_states || !embeddings) {
- free(lengths); free(inputs); free(states); free(out_states);
- free(embeddings);
- return send_error_result(fd, request_id, "out of memory");
- }
-
- uint32_t total = 0;
- for (uint32_t i = 0; i < work.batch; i++) {
- uint32_t n = 0;
- memcpy(&n, p + (size_t)i * sizeof(n), sizeof(n));
- n = ntohl(n);
- if (n == 0 || n > PPLX_DIST_MAX_TOKENS - total) {
- free(lengths); free(inputs); free(states); free(out_states);
- free(embeddings);
- return send_error_result(fd, request_id, "invalid sequence length");
- }
- lengths[i] = (int)n;
- inputs[i].n_tokens = (int)n;
- total += n;
- }
- if (total != work.total_tokens) {
- free(lengths); free(inputs); free(states); free(out_states);
- free(embeddings);
- return send_error_result(fd, request_id, "sequence lengths do not match token total");
- }
- if (decode_activations(states, p + expected_lengths,
- expected_state_values, states_bits) != 0) {
- free(lengths); free(inputs); free(states); free(out_states);
- free(embeddings);
- return send_error_result(fd, request_id, "invalid activation payload");
- }
-
- int rc = stage_forward(stage, inputs, (int)work.batch, states,
- layer_start, layer_end, 1, out_states);
- if (rc == 0)
- rc = pplx_pool_batch(stage->cfg, out_states, lengths,
- (int)work.batch, embeddings);
- free(lengths); free(inputs); free(states); free(out_states);
- if (rc != 0) {
- free(embeddings);
- return send_error_result(fd, request_id, "worker inference failed");
- }
-
- dist_result result;
- memset(&result, 0, sizeof(result));
- u64_to_halves(request_id, &result.request_hi, &result.request_lo);
- result.batch = work.batch;
- result.dimensions = work.hidden_size;
- result.payload_bytes = emb_bytes;
- dist_result wire = result;
- result_to_wire(&wire);
- rc = write_frame_header(fd, PPLX_DIST_MSG_RESULT,
- (uint32_t)sizeof(wire) + emb_bytes);
- if (rc == 0) rc = write_full(fd, &wire, sizeof(wire));
- if (rc == 0) rc = write_full(fd, embeddings, emb_bytes);
- free(embeddings);
- return rc;
-}
-
-static int worker_serve_connection(int fd, dist_stage *stage,
- int layer_start, int layer_end)
-{
- if (send_hello(fd, stage->cfg, layer_start, layer_end) != 0) return -1;
- for (;;) {
- uint32_t type = 0, bytes = 0;
- if (read_frame_header(fd, &type, &bytes) != 0) return -1;
- if (type != PPLX_DIST_MSG_WORK || bytes == 0) return -1;
- void *body = malloc(bytes);
- if (!body) return -1;
- int rc = read_full(fd, body, bytes);
- if (rc == 0)
- rc = worker_process_work(fd, stage, layer_start, layer_end,
- body, bytes);
- free(body);
- if (rc != 0) return -1;
- }
-}
-
-int pplx_dist_run_worker(const char *model_dir, const char *host, int port,
- int layer_start, int layer_end, int use_mlx)
-{
- return pplx_dist_run_worker_with_options(model_dir, host, port,
- layer_start, layer_end, use_mlx,
- NULL);
-}
-
-int pplx_dist_run_worker_with_options(
- const char *model_dir, const char *host, int port,
- int layer_start, int layer_end, int use_mlx,
- const pplx_dist_options_t *opts)
-{
- if (!model_dir || !host || port <= 0 || port > 65535 ||
- !host_is_little_endian()) {
- fprintf(stderr, "pplx-dist: invalid worker configuration\n");
- return 1;
- }
- dist_stage stage;
- if (stage_load(&stage, model_dir, layer_start, layer_end, use_mlx,
- opts) != 0) {
- fprintf(stderr, "pplx-dist: failed to load worker shard\n");
- return 1;
- }
- if (layer_end != stage.cfg->n_layers) {
- fprintf(stderr, "pplx-dist: initial worker must own final layer %d\n",
- stage.cfg->n_layers);
- stage_free(&stage);
- return 1;
- }
- int listen_fd = open_listener(host, port);
- if (listen_fd < 0) {
- fprintf(stderr, "pplx-dist: listen failed on %s:%d: %s\n",
- host, port, strerror(errno));
- stage_free(&stage);
- return 1;
- }
- fprintf(stderr, "pplx-dist: worker listening on %s:%d layers=[%d,%d)\n",
- host, port, layer_start, layer_end);
- for (;;) {
- int fd = accept(listen_fd, NULL, NULL);
- if (fd < 0) {
- if (errno == EINTR) continue;
- fprintf(stderr, "pplx-dist: accept failed: %s\n", strerror(errno));
- break;
- }
- set_socket_options(fd);
- if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator connected\n");
- worker_serve_connection(fd, &stage, layer_start, layer_end);
- close(fd);
- if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator disconnected\n");
- }
- close(listen_fd);
- stage_free(&stage);
- return 1;
-}
-
-pplx_dist_client_t *pplx_dist_client_open(const char *model_dir,
- const char *host, int port,
- int local_layer_end, int use_mlx)
-{
- return pplx_dist_client_open_with_options(model_dir, host, port,
- local_layer_end, use_mlx, NULL);
-}
-
-pplx_dist_client_t *pplx_dist_client_open_with_options(
- const char *model_dir, const char *host, int port,
- int local_layer_end, int use_mlx, const pplx_dist_options_t *opts)
-{
- if (!model_dir || !host || port <= 0 || port > 65535 ||
- local_layer_end <= 0 || !host_is_little_endian())
- return NULL;
-
- pplx_dist_client_t *client = calloc(1, sizeof(*client));
- if (!client) return NULL;
- client->fd = -1;
- if (stage_load(&client->local, model_dir, 0, local_layer_end, use_mlx,
- opts) != 0)
- goto fail;
- client->activation_bits = activation_bits_or_default(
- opts ? (uint32_t)opts->activation_bits : 0);
- if (!activation_bits_valid(client->activation_bits))
- goto fail;
- client->fd = connect_endpoint(host, port);
- if (client->fd < 0 ||
- recv_hello(client->fd, client->local.cfg, local_layer_end) != 0)
- goto fail;
- client->local_layer_end = local_layer_end;
- client->next_request_id = 1;
- return client;
-
-fail:
- pplx_dist_client_free(client);
- return NULL;
-}
-
-void pplx_dist_client_free(pplx_dist_client_t *client)
-{
- if (!client) return;
- if (client->fd >= 0) close(client->fd);
- stage_free(&client->local);
- free(client);
-}
-
-const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client)
-{
- return client ? client->local.cfg : NULL;
-}
-
-static int client_transport_error(pplx_dist_client_t *client)
-{
- if (client->fd >= 0) {
- close(client->fd);
- client->fd = -1;
- }
- return -1;
-}
-
-int pplx_dist_client_embed_batch(pplx_dist_client_t *client,
- const pplx_input_t *inputs, int batch,
- float *out_embeddings)
-{
- if (!client || client->fd < 0 || !inputs || batch <= 0 ||
- batch > (int)PPLX_DIST_MAX_BATCH || !out_embeddings)
- return -1;
-
- uint32_t total_tokens = 0;
- for (int i = 0; i < batch; i++) {
- if (!inputs[i].ids || inputs[i].n_tokens <= 0 ||
- (uint32_t)inputs[i].n_tokens > PPLX_DIST_MAX_TOKENS - total_tokens)
- return -1;
- total_tokens += (uint32_t)inputs[i].n_tokens;
- }
-
- uint32_t state_values = 0, state_f32_bytes = 0, state_wire_bytes = 0;
- uint32_t lengths_bytes = 0;
- uint32_t hidden = (uint32_t)client->local.cfg->hidden_size;
- uint32_t states_bits = activation_bits_or_default(client->activation_bits);
- if (mul_u32(total_tokens, hidden, &state_values) != 0 ||
- mul_u32(state_values, sizeof(float), &state_f32_bytes) != 0 ||
- activation_wire_bytes(states_bits, state_values, &state_wire_bytes) != 0 ||
- mul_u32((uint32_t)batch, sizeof(uint32_t), &lengths_bytes) != 0)
- return -1;
- uint64_t body_bytes64 = (uint64_t)sizeof(dist_work) + lengths_bytes +
- state_wire_bytes;
- if (body_bytes64 > PPLX_DIST_MAX_FRAME_BYTES)
- return -1;
- uint32_t body_bytes = (uint32_t)body_bytes64;
- float *states = malloc(state_f32_bytes);
- void *wire_states = malloc(state_wire_bytes);
- if (!states || !wire_states) {
- free(states); free(wire_states);
- return -1;
- }
- int rc = stage_forward(&client->local, inputs, batch, NULL, 0,
- client->local_layer_end, 0, states);
- if (rc != 0) {
- free(states); free(wire_states);
- return -1;
- }
- if (encode_activations(wire_states, states, state_values, states_bits) != 0) {
- free(states); free(wire_states);
- return -1;
- }
-
- uint64_t request_id = client->next_request_id++;
- dist_work work;
- u64_to_halves(request_id, &work.request_hi, &work.request_lo);
- work.batch = (uint32_t)batch;
- work.total_tokens = total_tokens;
- work.hidden_size = hidden;
- work.lengths_bytes = lengths_bytes;
- work.states_bytes = state_wire_bytes;
- work.states_bits = states_bits;
- dist_work wire = work;
- work_to_wire(&wire);
- rc = write_frame_header(client->fd, PPLX_DIST_MSG_WORK, body_bytes);
- if (rc == 0) rc = write_full(client->fd, &wire, sizeof(wire));
- for (int i = 0; rc == 0 && i < batch; i++) {
- uint32_t n = htonl((uint32_t)inputs[i].n_tokens);
- rc = write_full(client->fd, &n, sizeof(n));
- }
- if (rc == 0) rc = write_full(client->fd, wire_states, state_wire_bytes);
- free(states); free(wire_states);
- if (rc != 0) return client_transport_error(client);
-
- uint32_t type = 0, bytes = 0;
- dist_result result;
- if (read_frame_header(client->fd, &type, &bytes) != 0 ||
- type != PPLX_DIST_MSG_RESULT || bytes < sizeof(result) ||
- read_full(client->fd, &result, sizeof(result)) != 0)
- return client_transport_error(client);
- result_from_wire(&result);
- uint32_t payload_bytes = bytes - (uint32_t)sizeof(result);
- if (result.payload_bytes != payload_bytes ||
- u64_from_halves(result.request_hi, result.request_lo) != request_id)
- return client_transport_error(client);
- if (result.status != 0) {
- char error[PPLX_DIST_MAX_ERROR_BYTES + 1];
- if (payload_bytes > PPLX_DIST_MAX_ERROR_BYTES ||
- read_full(client->fd, error, payload_bytes) != 0)
- return client_transport_error(client);
- error[payload_bytes] = '\0';
- fprintf(stderr, "pplx-dist: worker error: %s\n", error);
- return client_transport_error(client);
- }
- uint32_t expected = 0;
- if (result.batch != (uint32_t)batch || result.dimensions != hidden ||
- mul_u32((uint32_t)batch, hidden, &expected) != 0 ||
- mul_u32(expected, sizeof(float), &expected) != 0 ||
- payload_bytes != expected ||
- read_full(client->fd, out_embeddings, payload_bytes) != 0)
- return client_transport_error(client);
- return 0;
-}
diff --git a/embed_distributed.h b/embed_distributed.h
deleted file mode 100644
index 3bd3e48..0000000
--- a/embed_distributed.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * embed_distributed.h - Stateless two-node layer-sharded inference.
- */
-
-#ifndef PPLX_DISTRIBUTED_H
-#define PPLX_DISTRIBUTED_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying PPLX_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef PPLX_API
-#if defined(__GNUC__)
-#define PPLX_API __attribute__((visibility("default")))
-#else
-#define PPLX_API
-#endif
-#endif
-
-#include "embed.h"
-
-typedef struct pplx_dist_client pplx_dist_client_t;
-
-typedef struct {
- int mlx_quantize_bits;
- int mlx_quantize_group_size;
- int activation_bits; /* 0 defaults to fp32; supported: 32, 16 */
-} pplx_dist_options_t;
-
-/*
- * Open a coordinator-side connection. The coordinator owns layers
- * [0, local_layer_end); the remote worker must own [local_layer_end, N).
- */
-PPLX_API pplx_dist_client_t *pplx_dist_client_open(const char *model_dir,
- const char *host, int port,
- int local_layer_end, int use_mlx);
-PPLX_API pplx_dist_client_t *pplx_dist_client_open_with_options(
- const char *model_dir, const char *host, int port,
- int local_layer_end, int use_mlx, const pplx_dist_options_t *opts);
-PPLX_API void pplx_dist_client_free(pplx_dist_client_t *client);
-
-PPLX_API const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client);
-
-PPLX_API int pplx_dist_client_embed_batch(pplx_dist_client_t *client,
- const pplx_input_t *inputs, int batch,
- float *out_embeddings);
-
-/*
- * Run a final-stage worker until the process is terminated. The worker owns
- * [layer_start, layer_end), and layer_end must be the model's final layer.
- */
-PPLX_API int pplx_dist_run_worker(const char *model_dir, const char *host, int port,
- int layer_start, int layer_end, int use_mlx);
-PPLX_API int pplx_dist_run_worker_with_options(
- const char *model_dir, const char *host, int port,
- int layer_start, int layer_end, int use_mlx,
- const pplx_dist_options_t *opts);
-
-#endif /* PPLX_DISTRIBUTED_H */
diff --git a/embed_mlx.c b/embed_mlx.c
index 4ef283c..aa0d1ed 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -654,19 +654,6 @@ pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir,
return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
}
-pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir,
- int layer_start, int layer_end)
-{
- return mlx_load_range_ex(model_dir, layer_start, layer_end, NULL, 0);
-}
-
-pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options(
- const char *model_dir, int layer_start, int layer_end,
- const pplx_mlx_options_t *opts)
-{
- return mlx_load_range_ex(model_dir, layer_start, layer_end, opts, 0);
-}
-
/* ========================================================================
* Free
* ======================================================================== */
@@ -1674,179 +1661,6 @@ int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids,
return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding);
}
-int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
- const pplx_input_t *inputs, int batch,
- const float *input_states,
- int layer_start, int layer_end,
- int apply_final_norm,
- float *out_states)
-{
- if (!ctx || !inputs || batch <= 0 || !out_states ||
- layer_start < 0 || layer_start > layer_end ||
- layer_end > ctx->config.n_layers ||
- layer_start < ctx->layer_start || layer_end > ctx->layer_end ||
- (layer_start == 0 && input_states) ||
- (layer_start != 0 && !input_states) ||
- (layer_start == 0 && !arr_ok(ctx->embed_tokens)) ||
- (apply_final_norm &&
- (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm))))
- return -1;
-
- const pplx_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
- int hidden = c->hidden_size;
- int max_seq = 0;
- int total_seq = 0;
- int has_padding = 0;
- for (int b = 0; b < batch; b++) {
- if ((layer_start == 0 && !inputs[b].ids) || inputs[b].n_tokens <= 0 ||
- total_seq > INT_MAX - inputs[b].n_tokens)
- return -1;
- total_seq += inputs[b].n_tokens;
- if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens;
- }
- for (int b = 0; b < batch; b++) {
- if (inputs[b].n_tokens != max_seq) has_padding = 1;
- }
-
- size_t rows, ids_bytes, mask_bytes, state_bytes, row_bytes;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 ||
- mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 ||
- mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 ||
- mlx_mul_size((size_t)hidden, sizeof(float), &row_bytes) != 0)
- return -1;
- if (rows == 0 || row_bytes == 0 || (has_padding && mask_bytes == 0))
- return -1;
- if (layer_start != 0 &&
- mlx_mul_size(rows, (size_t)hidden, &state_bytes) != 0)
- return -1;
- if (layer_start == 0)
- state_bytes = 0;
-
- int *padded_ids = layer_start == 0
- ? (int *)malloc(ids_bytes) : NULL;
- float *padded_states = layer_start != 0
- ? (float *)calloc(1, state_bytes) : NULL;
- float *attn_mask_data = has_padding
- ? (float *)malloc(mask_bytes) : NULL;
- if ((layer_start == 0 && !padded_ids) ||
- (layer_start != 0 && !padded_states) ||
- (has_padding && !attn_mask_data)) {
- free(padded_ids); free(padded_states); free(attn_mask_data);
- return -1;
- }
-
- int packed_offset = 0;
- for (int b = 0; b < batch; b++) {
- for (int t = 0; t < max_seq; t++) {
- size_t dense_row = (size_t)b * max_seq + t;
- if (t < inputs[b].n_tokens) {
- if (layer_start == 0) {
- int id = inputs[b].ids[t];
- if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
- id, b, t);
- free(padded_ids); free(padded_states);
- free(attn_mask_data);
- return -1;
- }
- padded_ids[dense_row] = id;
- } else {
- memcpy(padded_states + dense_row * hidden,
- input_states + (size_t)(packed_offset + t) * hidden,
- row_bytes);
- }
- if (has_padding) attn_mask_data[dense_row] = 0.0f;
- } else {
- if (layer_start == 0) padded_ids[dense_row] = 0;
- if (has_padding) attn_mask_data[dense_row] = -1.0e9f;
- }
- }
- packed_offset += inputs[b].n_tokens;
- }
-
- int attn_mask_shape[] = {batch, 1, 1, max_seq};
- mlx_array attn_mask = has_padding
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- free(attn_mask_data);
- if (has_padding && !arr_ok(attn_mask)) {
- free(padded_ids); free(padded_states);
- return -1;
- }
- if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) {
- free(padded_ids); free(padded_states);
- return -1;
- }
- }
-
- mlx_array x;
- if (layer_start == 0) {
- int ids_shape[] = {batch, max_seq};
- mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- free(padded_ids);
- if (!arr_ok(ids)) {
- if (has_padding) mlx_array_free(attn_mask);
- return -1;
- }
- x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
- } else {
- int states_shape[] = {batch, max_seq, hidden};
- mlx_array states = mlx_array_new_data(padded_states, states_shape, 3,
- MLX_FLOAT32);
- free(padded_states);
- if (!arr_ok(states)) {
- if (has_padding) mlx_array_free(attn_mask);
- return -1;
- }
- if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- x = mlx_array_new();
- mlx_astype(&x, states, MLX_BFLOAT16, S);
- mlx_array_free(states);
- } else {
- x = states;
- }
- }
-
- x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
- layer_start, layer_end);
- if (has_padding) mlx_array_free(attn_mask);
-
- if (apply_final_norm) {
- mlx_array x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- x = x_normed;
- }
-
- mlx_array x_f32 = mlx_array_new();
- mlx_astype(&x_f32, x, MLX_FLOAT32, S);
- mlx_array_free(x);
- mlx_array_eval(x_f32);
-
- const float *data = mlx_array_data_float32(x_f32);
- if (!data) {
- mlx_array_free(x_f32);
- return -1;
- }
- packed_offset = 0;
- for (int b = 0; b < batch; b++) {
- memcpy(out_states + (size_t)packed_offset * hidden,
- data + (size_t)b * max_seq * hidden,
- (size_t)inputs[b].n_tokens * row_bytes);
- packed_offset += inputs[b].n_tokens;
- }
- mlx_array_free(x_f32);
- return 0;
-}
-
typedef struct {
int index;
int span_offset;
diff --git a/embed_mlx.h b/embed_mlx.h
index 52a8801..ccc20c5 100644
--- a/embed_mlx.h
+++ b/embed_mlx.h
@@ -40,17 +40,6 @@ PPLX_API pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir);
PPLX_API pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir,
const pplx_mlx_options_t *opts);
-/*
- * Load one contiguous transformer layer range for distributed execution.
- * Uses the same half-open range and endpoint ownership rules as
- * pplx_model_load_slice().
- */
-PPLX_API pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir,
- int layer_start, int layer_end);
-PPLX_API pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options(
- const char *model_dir, int layer_start, int layer_end,
- const pplx_mlx_options_t *opts);
-
PPLX_API void pplx_mlx_free(pplx_mlx_ctx_t *ctx);
/*
@@ -126,19 +115,6 @@ PPLX_API int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *input
int batch, float *out_embeddings);
/*
- * MLX equivalent of pplx_model_forward_slice_batch().
- *
- * Hidden states cross the API boundary as packed float32 rows even though MLX
- * executes a padded dense batch internally.
- */
-PPLX_API int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
- const pplx_input_t *inputs, int batch,
- const float *input_states,
- int layer_start, int layer_end,
- int apply_final_norm,
- float *out_states);
-
-/*
* Run one contextual sequence and pool selected token spans.
* Returns 0 on success, -1 on error.
*/
diff --git a/tests/test_cli.c b/tests/test_cli.c
index fd464f0..0a5212a 100644
--- a/tests/test_cli.c
+++ b/tests/test_cli.c
@@ -167,9 +167,6 @@ int main(int argc, char **argv)
{"--cuda", "--cuda without CUDA build"},
{"--mlx-quant-bits 4", "--mlx-quant-bits 4"},
{"--mlx-quant-group-size 0", "--mlx-quant-group-size 0"},
- {"--layers 5:2", "inverted --layers range"},
- {"--layers nope", "malformed --layers"},
- {"--dist-activation-bits 8", "--dist-activation-bits 8"},
{"--cuda-gemm-mode f32 -d X t", "--cuda-gemm-mode without CUDA"},
{"--cuda-weight-dtype bf16 -d X t", "--cuda-weight-dtype without CUDA"},
};
@@ -179,15 +176,6 @@ int main(int argc, char **argv)
/* Validation that runs after the model dir is accepted. */
snprintf(args, sizeof(args), "-d '%s' --mlx-quant-bits 8 t", g_dir);
expect(run_cli(args, NULL) == 1, "--mlx-quant-bits without --mlx");
- snprintf(args, sizeof(args), "-d '%s' --dist-worker", g_dir);
- expect(run_cli(args, NULL) == 1, "--dist-worker without --layers");
- snprintf(args, sizeof(args),
- "-d '%s' --dist-worker --dist-remote h:1 --layers 0:1", g_dir);
- expect(run_cli(args, NULL) == 1, "--dist-worker with --dist-remote");
- snprintf(args, sizeof(args),
- "-d '%s' --dist-activation-bits 16 t", g_dir);
- expect(run_cli(args, NULL) == 1,
- "--dist-activation-bits without --dist-remote");
if (g_failures) {
fprintf(stderr, "%d CLI check(s) failed\n", g_failures);