diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-11 15:34:50 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 6b3c4bb1482917e4fb24480a1d082da5f191467b (patch) | |
| tree | 6a29d58959d1c5d3a0febcc9cb06005ac5a009ed | |
| parent | 54e6e6bbef396eb1d38c09517d8157301fb8f151 (diff) | |
| download | ffwd-6b3c4bb1482917e4fb24480a1d082da5f191467b.tar.gz ffwd-6b3c4bb1482917e4fb24480a1d082da5f191467b.zip | |
dist: remove distributed layer-sharding (protocol, CLI flags, slice APIs)
| -rw-r--r-- | Makefile | 10 | ||||
| -rw-r--r-- | embed.c | 42 | ||||
| -rw-r--r-- | embed.h | 32 | ||||
| -rw-r--r-- | embed_cli.c | 170 | ||||
| -rw-r--r-- | embed_distributed.c | 888 | ||||
| -rw-r--r-- | embed_distributed.h | 58 | ||||
| -rw-r--r-- | embed_mlx.c | 186 | ||||
| -rw-r--r-- | embed_mlx.h | 24 | ||||
| -rw-r--r-- | tests/test_cli.c | 12 |
9 files changed, 19 insertions, 1403 deletions
@@ -19,7 +19,6 @@ UNAME_S := $(shell uname -s) # Source files SRCS = embed.c \ - embed_distributed.c \ qwen_kernels.c \ qwen_kernels_generic.c \ qwen_kernels_neon.c \ @@ -167,11 +166,11 @@ TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c qwen_tokenizer.c \ qwen_safetensors.c $(KERNEL_SRCS) TEST_LATE_SRCS = tests/test_late.c embed.c qwen_tokenizer.c \ qwen_safetensors.c $(KERNEL_SRCS) -TEST_SERVER_SRCS = tests/test_server.c embed.c embed_distributed.c \ +TEST_SERVER_SRCS = tests/test_server.c embed.c \ qwen_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) \ deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c # The CLI check builds the real CLI (CPU backend) plus a driver that runs it. -TEST_CLI_BIN_SRCS = embed_cli.c embed.c embed_distributed.c \ +TEST_CLI_BIN_SRCS = embed_cli.c embed.c \ qwen_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) test: @@ -335,16 +334,13 @@ $(SERVER_TARGET): $(LIB) embed_server.o embed.o: embed.c embed.h qwen_kernels.h qwen_safetensors.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< -embed_distributed.o: embed_distributed.c embed_distributed.h embed.h embed_mlx.h - $(CC) $(CFLAGS) -c -o $@ $< - embed_server.o: embed_server.c embed_server.h embed.h embed_mlx.h qwen_tokenizer.h deps/ae/ae.h deps/ae/anet.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< embed_mlx.o: embed_mlx.c embed_mlx.h embed.h qwen_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< -embed_cli.o: embed_cli.c embed.h embed_distributed.h qwen_kernels.h qwen_tokenizer.h +embed_cli.o: embed_cli.c embed.h qwen_kernels.h qwen_tokenizer.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< @@ -717,12 +717,6 @@ pplx_model_t *pplx_model_load(const char *model_dir) return model_load_range_ex(model_dir, 0, -1, 0); } -pplx_model_t *pplx_model_load_slice(const char *model_dir, - int layer_start, int layer_end) -{ - return model_load_range_ex(model_dir, layer_start, layer_end, 0); -} - void pplx_model_free(pplx_model_t *model) { if (!model) return; @@ -1442,42 +1436,6 @@ int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t *ws, return rc; } -int pplx_model_forward_slice_batch(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states) -{ - if (!model || !ws || !inputs || batch <= 0 || !out_states || - layer_start < 0 || layer_start > layer_end || - layer_end > model->config.n_layers) - return -1; - - if (ensure_offsets(ws, batch) != 0) return -1; - int *offsets = ws->offsets; - - int total_seq = 0; - int max_seq = 0; - if (build_offsets(inputs, batch, layer_start == 0, offsets, - &total_seq, &max_seq) != 0) - return -1; - - int rc = forward_packed_slice_inplace(model, ws, inputs, batch, offsets, - total_seq, max_seq, input_states, - layer_start, layer_end, - apply_final_norm); - if (rc == 0) { - size_t count; - if (mul_size((size_t)total_seq, - (size_t)model->config.hidden_size, &count) != 0) - return -1; - memcpy(out_states, ws->x, count * sizeof(float)); - } - return rc; -} - int pplx_pool_batch(const pplx_config_t *cfg, const float *states, const int *seq_lengths, int batch, float *out_embeddings) @@ -88,17 +88,6 @@ typedef struct { */ PPLX_API pplx_model_t *pplx_model_load(const char *model_dir); -/* - * Load one contiguous transformer layer range for distributed execution. - * - * layer_start is inclusive and layer_end is exclusive. A range beginning at - * zero also loads the token embedding table. A range ending at n_layers also - * loads the final RMSNorm. Safetensors remain mmap-backed, so unselected - * weights are not touched. - */ -PPLX_API pplx_model_t *pplx_model_load_slice(const char *model_dir, - int layer_start, int layer_end); - PPLX_API void pplx_model_free(pplx_model_t *model); /* @@ -158,27 +147,6 @@ PPLX_API int pplx_model_embed_batch(const pplx_model_t *model, pplx_workspace_t float *out_embeddings); /* - * Run a contiguous transformer layer range for a packed/ragged batch. - * - * layer_start is inclusive and layer_end is exclusive. For a range beginning - * at layer 0, input_states must be NULL and every input must provide token - * ids. For a later range, input_states must contain the preceding range's - * packed [sum(n_tokens), hidden_size] output; token ids are not required. - * - * apply_final_norm is valid only for a range ending at the final transformer - * layer. out_states receives packed [sum(n_tokens), hidden_size] states. - * - * This is the core execution boundary used by layer-sharded inference. - */ -PPLX_API int pplx_model_forward_slice_batch(const pplx_model_t *model, - pplx_workspace_t *ws, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states); - -/* * Mean-pool packed final hidden states for a ragged batch. * * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive diff --git a/embed_cli.c b/embed_cli.c index ad14857..10f992d 100644 --- a/embed_cli.c +++ b/embed_cli.c @@ -1,7 +1,6 @@ /* embed_cli.c - pplx-embed command-line tool */ #include "embed.h" -#include "embed_distributed.h" #include "qwen_kernels.h" #include "qwen_tokenizer.h" @@ -35,13 +34,6 @@ static void print_usage(const char *prog) " --mlx Use Apple MLX GPU backend\n" " --cuda Use CUDA/cuBLAS GPU backend\n" " --stream Read lines from stdin, write one JSON embedding per line\n" - " --dist-worker\n" - " Serve one final distributed layer shard\n" - " --dist-remote HOST:PORT\n" - " Use one remote final shard for CLI or --stream inference\n" - " --layers A:B Distributed half-open layer range, for example 0:14\n" - " --dist-activation-bits N\n" - " Distributed activation transport width: 32 or 16 (default: 32)\n" " --mlx-quant-bits N\n" " Quantize MLX linear weights to 8 bits at load time\n" " --mlx-quant-group-size N\n" @@ -51,8 +43,6 @@ static void print_usage(const char *prog) " --cuda-weight-dtype DTYPE\n" " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" " weight memory and uses BF16 tensor cores\n" - " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n" - " --port N Bind port for --dist-worker (default: 8000)\n" " -b, --batch-size N\n" " Max texts per engine batch (default: all; --stream: 1)\n" " -t, --threads N\n" @@ -68,14 +58,11 @@ static void print_usage(const char *prog) " 2+ text args Print cosine similarity matrix\n" " no args Batch: read all stdin lines, then similarity matrix\n" " --stream Streaming: read stdin lines, write JSON per line\n" - " --dist-worker Final layer-shard worker\n" "\n" "Examples:\n" " %s -d ./model \"query: what is AI?\"\n" - " %s -d ./model --mlx --stream < texts.txt\n" - " %s -d ./model --dist-worker --layers 14:28 --port 9000\n" - " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n", - prog, prog, prog, prog, prog); + " %s -d ./model --mlx --stream < texts.txt\n", + prog, prog, prog); } /* ======================================================================== @@ -85,7 +72,6 @@ static void print_usage(const char *prog) typedef struct { pplx_model_t *model; pplx_workspace_t *workspace; - pplx_dist_client_t *dist; #ifdef USE_MLX pplx_mlx_ctx_t *mlx_ctx; #endif @@ -105,9 +91,6 @@ typedef struct { static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, int batch, float *out_embeddings) { - if (e->dist) - return pplx_dist_client_embed_batch(e->dist, inputs, batch, - out_embeddings); #ifdef USE_MLX if (e->mlx_ctx) return pplx_mlx_embed_batch(e->mlx_ctx, inputs, batch, out_embeddings); @@ -120,30 +103,6 @@ static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, inputs, batch, out_embeddings); } -static int parse_layers(const char *arg, int *start, int *end) -{ - int used = 0; - if (!arg || sscanf(arg, "%d:%d%n", start, end, &used) != 2 || - arg[used] != '\0' || *start < 0 || *start >= *end) - return -1; - return 0; -} - -static int parse_endpoint(const char *arg, char *host, size_t hostlen, int *port) -{ - const char *colon = arg ? strrchr(arg, ':') : NULL; - if (!colon || colon == arg || !colon[1]) return -1; - size_t n = (size_t)(colon - arg); - if (n >= hostlen) return -1; - memcpy(host, arg, n); - host[n] = '\0'; - int used = 0; - if (sscanf(colon + 1, "%d%n", port, &used) != 1 || - colon[1 + used] != '\0' || *port <= 0 || *port > 65535) - return -1; - return 0; -} - static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) { memset(out, 0, sizeof(*out)); @@ -474,18 +433,11 @@ int main(int argc, char *argv[]) int use_mlx = 0; int use_cuda = 0; int stdin_mode = 0; - int dist_worker_mode = 0; - const char *dist_remote = NULL; - int layer_start = -1; - int layer_end = -1; int batch_size = 0; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; const char *cuda_fast_gemm = NULL; const char *cuda_weights = NULL; - int dist_activation_bits = 32; - const char *host = "127.0.0.1"; - int port = 8000; int arg_start = 1; while (arg_start < argc && argv[arg_start][0] == '-') { @@ -543,29 +495,12 @@ int main(int argc, char *argv[]) else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg_start]; } - else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } - else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } - else if (!strcmp(f, "--layers")) { - if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) { - fprintf(stderr, "--layers expects START:END with START < END\n"); - return 1; - } - } - else if (!strcmp(f, "--dist-activation-bits")) { - dist_activation_bits = atoi(argv[++arg_start]); - if (dist_activation_bits != 32 && dist_activation_bits != 16) { - fprintf(stderr, "--dist-activation-bits must be 32 or 16\n"); - return 1; - } - } else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { print_embs = 1; } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } else if (!strcmp(f, "--stream")) { stdin_mode = 1; } - else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; } - else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; } else if (!strcmp(f, "--mlx")) { #ifdef USE_MLX use_mlx = 1; @@ -637,28 +572,6 @@ int main(int argc, char *argv[]) return 1; } - if (dist_worker_mode && dist_remote) { - fprintf(stderr, "--dist-worker and --dist-remote are mutually exclusive\n"); - return 1; - } - if ((dist_worker_mode || dist_remote) && - (layer_start < 0 || layer_end < 0)) { - fprintf(stderr, "distributed mode requires --layers START:END\n"); - return 1; - } - if (dist_activation_bits != 32 && !dist_remote) { - fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n"); - return 1; - } - if (dist_remote && layer_start != 0) { - fprintf(stderr, "--dist-remote coordinator layers must start at zero\n"); - return 1; - } - if ((dist_worker_mode || dist_remote) && use_cuda) { - fprintf(stderr, "distributed mode is not available with --cuda yet\n"); - return 1; - } - /* Threads (CPU backend) */ if (!use_mlx && !use_cuda) { if (n_threads <= 0) n_threads = qwen_get_num_cpus(); @@ -674,17 +587,6 @@ int main(int argc, char *argv[]) return 1; } - pplx_dist_options_t dist_opts = { - .mlx_quantize_bits = mlx_quantize_bits, - .mlx_quantize_group_size = mlx_quantize_group_size, - .activation_bits = dist_activation_bits, - }; - - if (dist_worker_mode) - return pplx_dist_run_worker_with_options( - model_dir, host, port, layer_start, layer_end, use_mlx, - &dist_opts); - /* Tokenizer */ char vocab_path[1024]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); @@ -699,25 +601,7 @@ int main(int argc, char *argv[]) t0 = now_ms(); #ifdef USE_MLX - if (dist_remote) { - char remote_host[1024]; - int remote_port = 0; - if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host), - &remote_port) != 0) { - fprintf(stderr, "--dist-remote expects HOST:PORT\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dist = pplx_dist_client_open_with_options( - model_dir, remote_host, remote_port, layer_end, use_mlx, - &dist_opts); - if (!e.dist) { - fprintf(stderr, "failed to connect distributed worker\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_dist_client_config(e.dist)->hidden_size; - } else if (use_mlx) { + if (use_mlx) { pplx_mlx_options_t mlx_opts = { .quantize_bits = mlx_quantize_bits, .quantize_group_size = mlx_quantize_group_size, @@ -743,40 +627,20 @@ int main(int argc, char *argv[]) } else #endif { - if (dist_remote) { - char remote_host[1024]; - int remote_port = 0; - if (parse_endpoint(dist_remote, remote_host, sizeof(remote_host), - &remote_port) != 0) { - fprintf(stderr, "--dist-remote expects HOST:PORT\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dist = pplx_dist_client_open_with_options( - model_dir, remote_host, remote_port, layer_end, use_mlx, - &dist_opts); - if (!e.dist) { - fprintf(stderr, "failed to connect distributed worker\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_dist_client_config(e.dist)->hidden_size; - } else { - e.model = pplx_model_load(model_dir); - if (!e.model) { - fprintf(stderr, "failed to load model\n"); - qwen_tokenizer_free(tok); - return 1; - } - e.workspace = pplx_workspace_new(e.model); - if (!e.workspace) { - fprintf(stderr, "failed to allocate workspace\n"); - pplx_model_free(e.model); - qwen_tokenizer_free(tok); - return 1; - } - e.dim = pplx_model_config(e.model)->hidden_size; + e.model = pplx_model_load(model_dir); + if (!e.model) { + fprintf(stderr, "failed to load model\n"); + qwen_tokenizer_free(tok); + return 1; + } + e.workspace = pplx_workspace_new(e.model); + if (!e.workspace) { + fprintf(stderr, "failed to allocate workspace\n"); + pplx_model_free(e.model); + qwen_tokenizer_free(tok); + return 1; } + e.dim = pplx_model_config(e.model)->hidden_size; } if (verbose >= 1) fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n", @@ -789,7 +653,6 @@ int main(int argc, char *argv[]) fprintf(stderr, "failed to allocate tokenizer workspace\n"); if (e.workspace) pplx_workspace_free(e.workspace); if (e.model) pplx_model_free(e.model); - if (e.dist) pplx_dist_client_free(e.dist); #ifdef USE_MLX if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); #endif @@ -810,7 +673,6 @@ int main(int argc, char *argv[]) /* Cleanup */ if (e.workspace) pplx_workspace_free(e.workspace); if (e.model) pplx_model_free(e.model); - if (e.dist) pplx_dist_client_free(e.dist); #ifdef USE_MLX if (e.mlx_ctx) pplx_mlx_free(e.mlx_ctx); #endif diff --git a/embed_distributed.c b/embed_distributed.c deleted file mode 100644 index 92d22c1..0000000 --- a/embed_distributed.c +++ /dev/null @@ -1,888 +0,0 @@ -/* - * embed_distributed.c - Stateless two-node layer-sharded inference. - * - * This first transport is deliberately synchronous: one coordinator sends a - * packed hidden-state batch to one final worker and receives pooled - * embeddings. The protocol is bounded and versioned so later route discovery, - * retries, and richer transport can extend it without changing model math. - */ - -#include "embed_distributed.h" - -#ifdef USE_MLX -#include "embed_mlx.h" -#endif - -#include <arpa/inet.h> -#include <errno.h> -#include <limits.h> -#include <netdb.h> -#include <netinet/tcp.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <unistd.h> - -#define PPLX_DIST_MAGIC UINT32_C(0x504c5844) /* "PLXD" */ -#define PPLX_DIST_VERSION 2u - -#define PPLX_DIST_MSG_HELLO 1u -#define PPLX_DIST_MSG_WORK 2u -#define PPLX_DIST_MSG_RESULT 3u - -#define PPLX_DIST_MAX_FRAME_BYTES (512u * 1024u * 1024u) -#define PPLX_DIST_MAX_BATCH 512u -#define PPLX_DIST_MAX_TOKENS 120000u -#define PPLX_DIST_MAX_ERROR_BYTES 1024u -#define PPLX_DIST_SOCKET_TIMEOUT_SEC 60 -#define PPLX_DIST_ACTIVATION_BITS_DEFAULT 32u - -typedef struct { - uint32_t magic; - uint32_t version; - uint32_t type; - uint32_t body_bytes; -} dist_frame_header; - -typedef struct { - uint32_t signature_hi; - uint32_t signature_lo; - uint32_t hidden_size; - uint32_t n_layers; - uint32_t vocab_size; - uint32_t n_heads; - uint32_t n_kv_heads; - uint32_t head_dim; - uint32_t intermediate_size; - uint32_t layer_start; - uint32_t layer_end; -} dist_hello; - -typedef struct { - uint32_t request_hi; - uint32_t request_lo; - uint32_t batch; - uint32_t total_tokens; - uint32_t hidden_size; - uint32_t lengths_bytes; - uint32_t states_bytes; - uint32_t states_bits; -} dist_work; - -typedef struct { - uint32_t request_hi; - uint32_t request_lo; - uint32_t status; - uint32_t batch; - uint32_t dimensions; - uint32_t payload_bytes; -} dist_result; - -typedef struct { - pplx_model_t *model; - pplx_workspace_t *workspace; -#ifdef USE_MLX - pplx_mlx_ctx_t *mlx; -#endif - const pplx_config_t *cfg; -} dist_stage; - -struct pplx_dist_client { - dist_stage local; - int fd; - int local_layer_end; - uint64_t next_request_id; - uint32_t activation_bits; -}; - -static int host_is_little_endian(void) -{ - const uint16_t one = 1; - return *(const unsigned char *)&one == 1; -} - -static uint64_t fnv_u32(uint64_t h, uint32_t v) -{ - for (int i = 0; i < 4; i++) { - h ^= (unsigned char)(v >> (i * 8)); - h *= UINT64_C(1099511628211); - } - return h; -} - -static uint64_t config_signature(const pplx_config_t *cfg) -{ - uint64_t h = UINT64_C(1469598103934665603); - uint32_t eps = 0, theta = 0; - memcpy(&eps, &cfg->rms_norm_eps, sizeof(eps)); - memcpy(&theta, &cfg->rope_theta, sizeof(theta)); - h = fnv_u32(h, (uint32_t)cfg->hidden_size); - h = fnv_u32(h, (uint32_t)cfg->n_layers); - h = fnv_u32(h, (uint32_t)cfg->n_heads); - h = fnv_u32(h, (uint32_t)cfg->n_kv_heads); - h = fnv_u32(h, (uint32_t)cfg->head_dim); - h = fnv_u32(h, (uint32_t)cfg->intermediate_size); - h = fnv_u32(h, (uint32_t)cfg->vocab_size); - h = fnv_u32(h, eps); - h = fnv_u32(h, theta); - return h; -} - -static void u64_to_halves(uint64_t v, uint32_t *hi, uint32_t *lo) -{ - *hi = (uint32_t)(v >> 32); - *lo = (uint32_t)v; -} - -static uint64_t u64_from_halves(uint32_t hi, uint32_t lo) -{ - return ((uint64_t)hi << 32) | lo; -} - -static int mul_u32(uint32_t a, uint32_t b, uint32_t *out) -{ - if (a != 0 && b > UINT32_MAX / a) return -1; - *out = a * b; - return 0; -} - -static uint32_t activation_bits_or_default(uint32_t bits) -{ - return bits ? bits : PPLX_DIST_ACTIVATION_BITS_DEFAULT; -} - -static int activation_bits_valid(uint32_t bits) -{ - bits = activation_bits_or_default(bits); - return bits == 32u || bits == 16u; -} - -static int activation_wire_bytes(uint32_t bits, uint64_t values, uint32_t *out) -{ - bits = activation_bits_or_default(bits); - if (!activation_bits_valid(bits)) return -1; - uint64_t bytes = values * (uint64_t)(bits / 8u); - if (bytes > UINT32_MAX) return -1; - if (out) *out = (uint32_t)bytes; - return 0; -} - -static uint16_t f32_to_f16(float f) -{ - uint32_t bits; - memcpy(&bits, &f, sizeof(bits)); - - uint32_t sign = (bits >> 16) & 0x8000u; - int32_t exp = (int32_t)((bits >> 23) & 0xffu) - 127 + 15; - uint32_t mant = bits & 0x7fffffu; - - if (exp <= 0) { - if (exp < -10) return (uint16_t)sign; - mant |= 0x800000u; - uint32_t shift = (uint32_t)(14 - exp); - uint32_t half_mant = mant >> shift; - uint32_t round_bit = (mant >> (shift - 1)) & 1u; - uint32_t sticky = mant & ((1u << (shift - 1)) - 1u); - if (round_bit && (sticky || (half_mant & 1u))) half_mant++; - return (uint16_t)(sign | half_mant); - } - - if (exp >= 31) { - if (((bits >> 23) & 0xffu) == 0xffu && mant != 0) - return (uint16_t)(sign | 0x7e00u); - return (uint16_t)(sign | 0x7c00u); - } - - uint32_t half = sign | ((uint32_t)exp << 10) | (mant >> 13); - uint32_t round = mant & 0x1fffu; - if (round > 0x1000u || (round == 0x1000u && (half & 1u))) half++; - return (uint16_t)half; -} - -static float f16_to_f32(uint16_t h) -{ - uint32_t sign = (uint32_t)(h & 0x8000u) << 16; - int32_t exp = (int32_t)((h >> 10) & 0x1fu); - uint32_t mant = h & 0x03ffu; - uint32_t bits; - - if (exp == 0) { - if (mant == 0) { - bits = sign; - } else { - exp = 1; - while ((mant & 0x0400u) == 0) { - mant <<= 1; - exp--; - } - mant &= 0x03ffu; - bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13); - } - } else if (exp == 31) { - bits = sign | 0x7f800000u | (mant << 13); - } else { - bits = sign | ((uint32_t)(exp + 127 - 15) << 23) | (mant << 13); - } - - float f; - memcpy(&f, &bits, sizeof(f)); - return f; -} - -static int encode_activations(void *dst, const float *src, uint64_t values, - uint32_t bits) -{ - bits = activation_bits_or_default(bits); - if (!dst || !src || !activation_bits_valid(bits)) return -1; - if (bits == 32u) { - memcpy(dst, src, values * sizeof(float)); - return 0; - } - uint16_t *out = dst; - for (uint64_t i = 0; i < values; i++) out[i] = f32_to_f16(src[i]); - return 0; -} - -static int decode_activations(float *dst, const void *wire, uint64_t values, - uint32_t bits) -{ - bits = activation_bits_or_default(bits); - if (!dst || !wire || !activation_bits_valid(bits)) return -1; - if (bits == 32u) { - memcpy(dst, wire, values * sizeof(float)); - return 0; - } - const uint16_t *src = wire; - for (uint64_t i = 0; i < values; i++) dst[i] = f16_to_f32(src[i]); - return 0; -} - -static void frame_to_wire(dist_frame_header *h) -{ - h->magic = htonl(h->magic); - h->version = htonl(h->version); - h->type = htonl(h->type); - h->body_bytes = htonl(h->body_bytes); -} - -static void hello_to_wire(dist_hello *h) -{ - uint32_t *v = (uint32_t *)h; - for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = htonl(v[i]); -} - -static void hello_from_wire(dist_hello *h) -{ - uint32_t *v = (uint32_t *)h; - for (size_t i = 0; i < sizeof(*h) / sizeof(*v); i++) v[i] = ntohl(v[i]); -} - -static void work_to_wire(dist_work *w) -{ - uint32_t *v = (uint32_t *)w; - for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = htonl(v[i]); -} - -static void work_from_wire(dist_work *w) -{ - uint32_t *v = (uint32_t *)w; - for (size_t i = 0; i < sizeof(*w) / sizeof(*v); i++) v[i] = ntohl(v[i]); -} - -static void result_to_wire(dist_result *r) -{ - uint32_t *v = (uint32_t *)r; - for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = htonl(v[i]); -} - -static void result_from_wire(dist_result *r) -{ - uint32_t *v = (uint32_t *)r; - for (size_t i = 0; i < sizeof(*r) / sizeof(*v); i++) v[i] = ntohl(v[i]); -} - -static int write_full(int fd, const void *buf, size_t len) -{ - const unsigned char *p = buf; - while (len) { -#ifdef MSG_NOSIGNAL - ssize_t n = send(fd, p, len, MSG_NOSIGNAL); -#else - ssize_t n = send(fd, p, len, 0); -#endif - if (n < 0 && errno == EINTR) continue; - if (n <= 0) return -1; - p += (size_t)n; - len -= (size_t)n; - } - return 0; -} - -static int read_full(int fd, void *buf, size_t len) -{ - unsigned char *p = buf; - while (len) { - ssize_t n = recv(fd, p, len, 0); - if (n < 0 && errno == EINTR) continue; - if (n <= 0) return -1; - p += (size_t)n; - len -= (size_t)n; - } - return 0; -} - -static int write_frame_header(int fd, uint32_t type, uint32_t body_bytes) -{ - dist_frame_header h = {PPLX_DIST_MAGIC, PPLX_DIST_VERSION, type, body_bytes}; - frame_to_wire(&h); - return write_full(fd, &h, sizeof(h)); -} - -static int read_frame_header(int fd, uint32_t *type, uint32_t *body_bytes) -{ - dist_frame_header h; - if (read_full(fd, &h, sizeof(h)) != 0) return -1; - h.magic = ntohl(h.magic); - h.version = ntohl(h.version); - h.type = ntohl(h.type); - h.body_bytes = ntohl(h.body_bytes); - if (h.magic != PPLX_DIST_MAGIC || h.version != PPLX_DIST_VERSION || - h.body_bytes > PPLX_DIST_MAX_FRAME_BYTES) - return -1; - *type = h.type; - *body_bytes = h.body_bytes; - return 0; -} - -static void set_socket_options(int fd) -{ - int yes = 1; - struct timeval tv = {PPLX_DIST_SOCKET_TIMEOUT_SEC, 0}; - setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); - setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)); - setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); -#ifdef SO_NOSIGPIPE - setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &yes, sizeof(yes)); -#endif -} - -static int open_listener(const char *host, int port) -{ - char service[16]; - snprintf(service, sizeof(service), "%d", port); - struct addrinfo hints; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_PASSIVE; - struct addrinfo *list = NULL; - if (getaddrinfo(host, service, &hints, &list) != 0) return -1; - - int listen_fd = -1; - for (struct addrinfo *ai = list; ai; ai = ai->ai_next) { - int fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) continue; - int yes = 1; - setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); - if (bind(fd, ai->ai_addr, ai->ai_addrlen) == 0 && - listen(fd, 16) == 0) { - listen_fd = fd; - break; - } - close(fd); - } - freeaddrinfo(list); - return listen_fd; -} - -static int connect_endpoint(const char *host, int port) -{ - char service[16]; - snprintf(service, sizeof(service), "%d", port); - struct addrinfo hints; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - struct addrinfo *list = NULL; - if (getaddrinfo(host, service, &hints, &list) != 0) return -1; - - int fd = -1; - for (struct addrinfo *ai = list; ai; ai = ai->ai_next) { - fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); - if (fd < 0) continue; - set_socket_options(fd); - if (connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) break; - close(fd); - fd = -1; - } - freeaddrinfo(list); - return fd; -} - -static int stage_load(dist_stage *stage, const char *model_dir, - int layer_start, int layer_end, int use_mlx, - const pplx_dist_options_t *opts) -{ - memset(stage, 0, sizeof(*stage)); -#ifndef USE_MLX - (void)opts; -#endif -#ifdef USE_MLX - if (use_mlx) { - pplx_mlx_options_t mlx_opts = {0}; - if (opts) { - mlx_opts.quantize_bits = opts->mlx_quantize_bits; - mlx_opts.quantize_group_size = opts->mlx_quantize_group_size; - } - stage->mlx = pplx_mlx_load_slice_with_options( - model_dir, layer_start, layer_end, &mlx_opts); - if (!stage->mlx) return -1; - stage->cfg = pplx_mlx_config(stage->mlx); - return 0; - } -#else - if (use_mlx) return -1; -#endif - stage->model = pplx_model_load_slice(model_dir, layer_start, layer_end); - if (!stage->model) return -1; - stage->workspace = pplx_workspace_new(stage->model); - if (!stage->workspace) { - pplx_model_free(stage->model); - stage->model = NULL; - return -1; - } - stage->cfg = pplx_model_config(stage->model); - return 0; -} - -static void stage_free(dist_stage *stage) -{ - if (!stage) return; - if (stage->workspace) pplx_workspace_free(stage->workspace); - if (stage->model) pplx_model_free(stage->model); -#ifdef USE_MLX - if (stage->mlx) pplx_mlx_free(stage->mlx); -#endif - memset(stage, 0, sizeof(*stage)); -} - -static int stage_forward(dist_stage *stage, const pplx_input_t *inputs, - int batch, const float *input_states, - int layer_start, int layer_end, int apply_final_norm, - float *out_states) -{ -#ifdef USE_MLX - if (stage->mlx) - return pplx_mlx_forward_slice_batch(stage->mlx, inputs, batch, - input_states, layer_start, - layer_end, apply_final_norm, - out_states); -#endif - return pplx_model_forward_slice_batch(stage->model, stage->workspace, - inputs, batch, input_states, - layer_start, layer_end, - apply_final_norm, out_states); -} - -static int send_hello(int fd, const pplx_config_t *cfg, - int layer_start, int layer_end) -{ - uint64_t sig = config_signature(cfg); - dist_hello h; - u64_to_halves(sig, &h.signature_hi, &h.signature_lo); - h.hidden_size = (uint32_t)cfg->hidden_size; - h.n_layers = (uint32_t)cfg->n_layers; - h.vocab_size = (uint32_t)cfg->vocab_size; - h.n_heads = (uint32_t)cfg->n_heads; - h.n_kv_heads = (uint32_t)cfg->n_kv_heads; - h.head_dim = (uint32_t)cfg->head_dim; - h.intermediate_size = (uint32_t)cfg->intermediate_size; - h.layer_start = (uint32_t)layer_start; - h.layer_end = (uint32_t)layer_end; - hello_to_wire(&h); - if (write_frame_header(fd, PPLX_DIST_MSG_HELLO, sizeof(h)) != 0) - return -1; - return write_full(fd, &h, sizeof(h)); -} - -static int recv_hello(int fd, const pplx_config_t *cfg, int expected_start) -{ - uint32_t type = 0, bytes = 0; - dist_hello h; - if (read_frame_header(fd, &type, &bytes) != 0 || - type != PPLX_DIST_MSG_HELLO || bytes != sizeof(h) || - read_full(fd, &h, sizeof(h)) != 0) - return -1; - hello_from_wire(&h); - if (u64_from_halves(h.signature_hi, h.signature_lo) != - config_signature(cfg) || - h.hidden_size != (uint32_t)cfg->hidden_size || - h.n_layers != (uint32_t)cfg->n_layers || - h.layer_start != (uint32_t)expected_start || - h.layer_end != (uint32_t)cfg->n_layers) - return -1; - return 0; -} - -static int send_error_result(int fd, uint64_t request_id, const char *msg) -{ - size_t len = msg ? strlen(msg) : 0; - if (len > PPLX_DIST_MAX_ERROR_BYTES) len = PPLX_DIST_MAX_ERROR_BYTES; - dist_result result; - memset(&result, 0, sizeof(result)); - u64_to_halves(request_id, &result.request_hi, &result.request_lo); - result.status = 1; - result.payload_bytes = (uint32_t)len; - dist_result wire = result; - result_to_wire(&wire); - if (write_frame_header(fd, PPLX_DIST_MSG_RESULT, - (uint32_t)sizeof(wire) + (uint32_t)len) != 0 || - write_full(fd, &wire, sizeof(wire)) != 0) - return -1; - return len ? write_full(fd, msg, len) : 0; -} - -static int worker_process_work(int fd, dist_stage *stage, - int layer_start, int layer_end, - const void *body, uint32_t body_bytes) -{ - uint64_t request_id = 0; - if (body_bytes < sizeof(dist_work)) - return send_error_result(fd, request_id, "truncated WORK frame"); - - dist_work work; - memcpy(&work, body, sizeof(work)); - work_from_wire(&work); - request_id = u64_from_halves(work.request_hi, work.request_lo); - - uint32_t expected_lengths = 0, expected_state_values = 0; - uint32_t expected_state_f32_bytes = 0, expected_state_wire_bytes = 0; - uint32_t states_bits = activation_bits_or_default(work.states_bits); - if (work.batch == 0 || work.batch > PPLX_DIST_MAX_BATCH || - work.total_tokens == 0 || work.total_tokens > PPLX_DIST_MAX_TOKENS || - work.hidden_size != (uint32_t)stage->cfg->hidden_size || - !activation_bits_valid(states_bits) || - mul_u32(work.batch, sizeof(uint32_t), &expected_lengths) != 0 || - mul_u32(work.total_tokens, work.hidden_size, - &expected_state_values) != 0 || - mul_u32(expected_state_values, sizeof(float), - &expected_state_f32_bytes) != 0 || - activation_wire_bytes(states_bits, expected_state_values, - &expected_state_wire_bytes) != 0 || - work.lengths_bytes != expected_lengths || - work.states_bytes != expected_state_wire_bytes) - return send_error_result(fd, request_id, "invalid WORK frame"); - uint64_t expected_body = (uint64_t)sizeof(work) + expected_lengths + - expected_state_wire_bytes; - if (expected_body > UINT32_MAX || body_bytes != (uint32_t)expected_body) - return send_error_result(fd, request_id, "invalid WORK frame"); - - const unsigned char *p = (const unsigned char *)body + sizeof(work); - int *lengths = malloc((size_t)work.batch * sizeof(*lengths)); - pplx_input_t *inputs = calloc(work.batch, sizeof(*inputs)); - float *states = malloc(expected_state_f32_bytes); - float *out_states = malloc(expected_state_f32_bytes); - uint32_t emb_bytes = 0; - if (mul_u32(work.batch, work.hidden_size, &emb_bytes) != 0 || - mul_u32(emb_bytes, sizeof(float), &emb_bytes) != 0) { - free(lengths); free(inputs); free(states); free(out_states); - return send_error_result(fd, request_id, "embedding result is too large"); - } - float *embeddings = malloc(emb_bytes); - if (!lengths || !inputs || !states || !out_states || !embeddings) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "out of memory"); - } - - uint32_t total = 0; - for (uint32_t i = 0; i < work.batch; i++) { - uint32_t n = 0; - memcpy(&n, p + (size_t)i * sizeof(n), sizeof(n)); - n = ntohl(n); - if (n == 0 || n > PPLX_DIST_MAX_TOKENS - total) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "invalid sequence length"); - } - lengths[i] = (int)n; - inputs[i].n_tokens = (int)n; - total += n; - } - if (total != work.total_tokens) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "sequence lengths do not match token total"); - } - if (decode_activations(states, p + expected_lengths, - expected_state_values, states_bits) != 0) { - free(lengths); free(inputs); free(states); free(out_states); - free(embeddings); - return send_error_result(fd, request_id, "invalid activation payload"); - } - - int rc = stage_forward(stage, inputs, (int)work.batch, states, - layer_start, layer_end, 1, out_states); - if (rc == 0) - rc = pplx_pool_batch(stage->cfg, out_states, lengths, - (int)work.batch, embeddings); - free(lengths); free(inputs); free(states); free(out_states); - if (rc != 0) { - free(embeddings); - return send_error_result(fd, request_id, "worker inference failed"); - } - - dist_result result; - memset(&result, 0, sizeof(result)); - u64_to_halves(request_id, &result.request_hi, &result.request_lo); - result.batch = work.batch; - result.dimensions = work.hidden_size; - result.payload_bytes = emb_bytes; - dist_result wire = result; - result_to_wire(&wire); - rc = write_frame_header(fd, PPLX_DIST_MSG_RESULT, - (uint32_t)sizeof(wire) + emb_bytes); - if (rc == 0) rc = write_full(fd, &wire, sizeof(wire)); - if (rc == 0) rc = write_full(fd, embeddings, emb_bytes); - free(embeddings); - return rc; -} - -static int worker_serve_connection(int fd, dist_stage *stage, - int layer_start, int layer_end) -{ - if (send_hello(fd, stage->cfg, layer_start, layer_end) != 0) return -1; - for (;;) { - uint32_t type = 0, bytes = 0; - if (read_frame_header(fd, &type, &bytes) != 0) return -1; - if (type != PPLX_DIST_MSG_WORK || bytes == 0) return -1; - void *body = malloc(bytes); - if (!body) return -1; - int rc = read_full(fd, body, bytes); - if (rc == 0) - rc = worker_process_work(fd, stage, layer_start, layer_end, - body, bytes); - free(body); - if (rc != 0) return -1; - } -} - -int pplx_dist_run_worker(const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx) -{ - return pplx_dist_run_worker_with_options(model_dir, host, port, - layer_start, layer_end, use_mlx, - NULL); -} - -int pplx_dist_run_worker_with_options( - const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx, - const pplx_dist_options_t *opts) -{ - if (!model_dir || !host || port <= 0 || port > 65535 || - !host_is_little_endian()) { - fprintf(stderr, "pplx-dist: invalid worker configuration\n"); - return 1; - } - dist_stage stage; - if (stage_load(&stage, model_dir, layer_start, layer_end, use_mlx, - opts) != 0) { - fprintf(stderr, "pplx-dist: failed to load worker shard\n"); - return 1; - } - if (layer_end != stage.cfg->n_layers) { - fprintf(stderr, "pplx-dist: initial worker must own final layer %d\n", - stage.cfg->n_layers); - stage_free(&stage); - return 1; - } - int listen_fd = open_listener(host, port); - if (listen_fd < 0) { - fprintf(stderr, "pplx-dist: listen failed on %s:%d: %s\n", - host, port, strerror(errno)); - stage_free(&stage); - return 1; - } - fprintf(stderr, "pplx-dist: worker listening on %s:%d layers=[%d,%d)\n", - host, port, layer_start, layer_end); - for (;;) { - int fd = accept(listen_fd, NULL, NULL); - if (fd < 0) { - if (errno == EINTR) continue; - fprintf(stderr, "pplx-dist: accept failed: %s\n", strerror(errno)); - break; - } - set_socket_options(fd); - if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator connected\n"); - worker_serve_connection(fd, &stage, layer_start, layer_end); - close(fd); - if (pplx_verbose >= 1) fprintf(stderr, "pplx-dist: coordinator disconnected\n"); - } - close(listen_fd); - stage_free(&stage); - return 1; -} - -pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, - const char *host, int port, - int local_layer_end, int use_mlx) -{ - return pplx_dist_client_open_with_options(model_dir, host, port, - local_layer_end, use_mlx, NULL); -} - -pplx_dist_client_t *pplx_dist_client_open_with_options( - const char *model_dir, const char *host, int port, - int local_layer_end, int use_mlx, const pplx_dist_options_t *opts) -{ - if (!model_dir || !host || port <= 0 || port > 65535 || - local_layer_end <= 0 || !host_is_little_endian()) - return NULL; - - pplx_dist_client_t *client = calloc(1, sizeof(*client)); - if (!client) return NULL; - client->fd = -1; - if (stage_load(&client->local, model_dir, 0, local_layer_end, use_mlx, - opts) != 0) - goto fail; - client->activation_bits = activation_bits_or_default( - opts ? (uint32_t)opts->activation_bits : 0); - if (!activation_bits_valid(client->activation_bits)) - goto fail; - client->fd = connect_endpoint(host, port); - if (client->fd < 0 || - recv_hello(client->fd, client->local.cfg, local_layer_end) != 0) - goto fail; - client->local_layer_end = local_layer_end; - client->next_request_id = 1; - return client; - -fail: - pplx_dist_client_free(client); - return NULL; -} - -void pplx_dist_client_free(pplx_dist_client_t *client) -{ - if (!client) return; - if (client->fd >= 0) close(client->fd); - stage_free(&client->local); - free(client); -} - -const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client) -{ - return client ? client->local.cfg : NULL; -} - -static int client_transport_error(pplx_dist_client_t *client) -{ - if (client->fd >= 0) { - close(client->fd); - client->fd = -1; - } - return -1; -} - -int pplx_dist_client_embed_batch(pplx_dist_client_t *client, - const pplx_input_t *inputs, int batch, - float *out_embeddings) -{ - if (!client || client->fd < 0 || !inputs || batch <= 0 || - batch > (int)PPLX_DIST_MAX_BATCH || !out_embeddings) - return -1; - - uint32_t total_tokens = 0; - for (int i = 0; i < batch; i++) { - if (!inputs[i].ids || inputs[i].n_tokens <= 0 || - (uint32_t)inputs[i].n_tokens > PPLX_DIST_MAX_TOKENS - total_tokens) - return -1; - total_tokens += (uint32_t)inputs[i].n_tokens; - } - - uint32_t state_values = 0, state_f32_bytes = 0, state_wire_bytes = 0; - uint32_t lengths_bytes = 0; - uint32_t hidden = (uint32_t)client->local.cfg->hidden_size; - uint32_t states_bits = activation_bits_or_default(client->activation_bits); - if (mul_u32(total_tokens, hidden, &state_values) != 0 || - mul_u32(state_values, sizeof(float), &state_f32_bytes) != 0 || - activation_wire_bytes(states_bits, state_values, &state_wire_bytes) != 0 || - mul_u32((uint32_t)batch, sizeof(uint32_t), &lengths_bytes) != 0) - return -1; - uint64_t body_bytes64 = (uint64_t)sizeof(dist_work) + lengths_bytes + - state_wire_bytes; - if (body_bytes64 > PPLX_DIST_MAX_FRAME_BYTES) - return -1; - uint32_t body_bytes = (uint32_t)body_bytes64; - float *states = malloc(state_f32_bytes); - void *wire_states = malloc(state_wire_bytes); - if (!states || !wire_states) { - free(states); free(wire_states); - return -1; - } - int rc = stage_forward(&client->local, inputs, batch, NULL, 0, - client->local_layer_end, 0, states); - if (rc != 0) { - free(states); free(wire_states); - return -1; - } - if (encode_activations(wire_states, states, state_values, states_bits) != 0) { - free(states); free(wire_states); - return -1; - } - - uint64_t request_id = client->next_request_id++; - dist_work work; - u64_to_halves(request_id, &work.request_hi, &work.request_lo); - work.batch = (uint32_t)batch; - work.total_tokens = total_tokens; - work.hidden_size = hidden; - work.lengths_bytes = lengths_bytes; - work.states_bytes = state_wire_bytes; - work.states_bits = states_bits; - dist_work wire = work; - work_to_wire(&wire); - rc = write_frame_header(client->fd, PPLX_DIST_MSG_WORK, body_bytes); - if (rc == 0) rc = write_full(client->fd, &wire, sizeof(wire)); - for (int i = 0; rc == 0 && i < batch; i++) { - uint32_t n = htonl((uint32_t)inputs[i].n_tokens); - rc = write_full(client->fd, &n, sizeof(n)); - } - if (rc == 0) rc = write_full(client->fd, wire_states, state_wire_bytes); - free(states); free(wire_states); - if (rc != 0) return client_transport_error(client); - - uint32_t type = 0, bytes = 0; - dist_result result; - if (read_frame_header(client->fd, &type, &bytes) != 0 || - type != PPLX_DIST_MSG_RESULT || bytes < sizeof(result) || - read_full(client->fd, &result, sizeof(result)) != 0) - return client_transport_error(client); - result_from_wire(&result); - uint32_t payload_bytes = bytes - (uint32_t)sizeof(result); - if (result.payload_bytes != payload_bytes || - u64_from_halves(result.request_hi, result.request_lo) != request_id) - return client_transport_error(client); - if (result.status != 0) { - char error[PPLX_DIST_MAX_ERROR_BYTES + 1]; - if (payload_bytes > PPLX_DIST_MAX_ERROR_BYTES || - read_full(client->fd, error, payload_bytes) != 0) - return client_transport_error(client); - error[payload_bytes] = '\0'; - fprintf(stderr, "pplx-dist: worker error: %s\n", error); - return client_transport_error(client); - } - uint32_t expected = 0; - if (result.batch != (uint32_t)batch || result.dimensions != hidden || - mul_u32((uint32_t)batch, hidden, &expected) != 0 || - mul_u32(expected, sizeof(float), &expected) != 0 || - payload_bytes != expected || - read_full(client->fd, out_embeddings, payload_bytes) != 0) - return client_transport_error(client); - return 0; -} diff --git a/embed_distributed.h b/embed_distributed.h deleted file mode 100644 index 3bd3e48..0000000 --- a/embed_distributed.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * embed_distributed.h - Stateless two-node layer-sharded inference. - */ - -#ifndef PPLX_DISTRIBUTED_H -#define PPLX_DISTRIBUTED_H - -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying PPLX_API are exported - * from the shared library; everything else stays internal. */ -#ifndef PPLX_API -#if defined(__GNUC__) -#define PPLX_API __attribute__((visibility("default"))) -#else -#define PPLX_API -#endif -#endif - -#include "embed.h" - -typedef struct pplx_dist_client pplx_dist_client_t; - -typedef struct { - int mlx_quantize_bits; - int mlx_quantize_group_size; - int activation_bits; /* 0 defaults to fp32; supported: 32, 16 */ -} pplx_dist_options_t; - -/* - * Open a coordinator-side connection. The coordinator owns layers - * [0, local_layer_end); the remote worker must own [local_layer_end, N). - */ -PPLX_API pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, - const char *host, int port, - int local_layer_end, int use_mlx); -PPLX_API pplx_dist_client_t *pplx_dist_client_open_with_options( - const char *model_dir, const char *host, int port, - int local_layer_end, int use_mlx, const pplx_dist_options_t *opts); -PPLX_API void pplx_dist_client_free(pplx_dist_client_t *client); - -PPLX_API const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client); - -PPLX_API int pplx_dist_client_embed_batch(pplx_dist_client_t *client, - const pplx_input_t *inputs, int batch, - float *out_embeddings); - -/* - * Run a final-stage worker until the process is terminated. The worker owns - * [layer_start, layer_end), and layer_end must be the model's final layer. - */ -PPLX_API int pplx_dist_run_worker(const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx); -PPLX_API int pplx_dist_run_worker_with_options( - const char *model_dir, const char *host, int port, - int layer_start, int layer_end, int use_mlx, - const pplx_dist_options_t *opts); - -#endif /* PPLX_DISTRIBUTED_H */ diff --git a/embed_mlx.c b/embed_mlx.c index 4ef283c..aa0d1ed 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -654,19 +654,6 @@ pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, return mlx_load_range_ex(model_dir, 0, -1, opts, 0); } -pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, - int layer_start, int layer_end) -{ - return mlx_load_range_ex(model_dir, layer_start, layer_end, NULL, 0); -} - -pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( - const char *model_dir, int layer_start, int layer_end, - const pplx_mlx_options_t *opts) -{ - return mlx_load_range_ex(model_dir, layer_start, layer_end, opts, 0); -} - /* ======================================================================== * Free * ======================================================================== */ @@ -1674,179 +1661,6 @@ int pplx_mlx_embed_into(pplx_mlx_ctx_t *ctx, const int *token_ids, return pplx_mlx_embed_batch(ctx, &input, 1, out_embedding); } -int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states) -{ - if (!ctx || !inputs || batch <= 0 || !out_states || - layer_start < 0 || layer_start > layer_end || - layer_end > ctx->config.n_layers || - layer_start < ctx->layer_start || layer_end > ctx->layer_end || - (layer_start == 0 && input_states) || - (layer_start != 0 && !input_states) || - (layer_start == 0 && !arr_ok(ctx->embed_tokens)) || - (apply_final_norm && - (layer_end != ctx->config.n_layers || !arr_ok(ctx->norm)))) - return -1; - - const pplx_config_t *c = &ctx->config; - mlx_stream S = ctx->stream; - int hidden = c->hidden_size; - int max_seq = 0; - int total_seq = 0; - int has_padding = 0; - for (int b = 0; b < batch; b++) { - if ((layer_start == 0 && !inputs[b].ids) || inputs[b].n_tokens <= 0 || - total_seq > INT_MAX - inputs[b].n_tokens) - return -1; - total_seq += inputs[b].n_tokens; - if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; - } - for (int b = 0; b < batch; b++) { - if (inputs[b].n_tokens != max_seq) has_padding = 1; - } - - size_t rows, ids_bytes, mask_bytes, state_bytes, row_bytes; - if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 || - mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 || - mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 || - mlx_mul_size((size_t)hidden, sizeof(float), &row_bytes) != 0) - return -1; - if (rows == 0 || row_bytes == 0 || (has_padding && mask_bytes == 0)) - return -1; - if (layer_start != 0 && - mlx_mul_size(rows, (size_t)hidden, &state_bytes) != 0) - return -1; - if (layer_start == 0) - state_bytes = 0; - - int *padded_ids = layer_start == 0 - ? (int *)malloc(ids_bytes) : NULL; - float *padded_states = layer_start != 0 - ? (float *)calloc(1, state_bytes) : NULL; - float *attn_mask_data = has_padding - ? (float *)malloc(mask_bytes) : NULL; - if ((layer_start == 0 && !padded_ids) || - (layer_start != 0 && !padded_states) || - (has_padding && !attn_mask_data)) { - free(padded_ids); free(padded_states); free(attn_mask_data); - return -1; - } - - int packed_offset = 0; - for (int b = 0; b < batch; b++) { - for (int t = 0; t < max_seq; t++) { - size_t dense_row = (size_t)b * max_seq + t; - if (t < inputs[b].n_tokens) { - if (layer_start == 0) { - int id = inputs[b].ids[t]; - if (id < 0 || id >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", - id, b, t); - free(padded_ids); free(padded_states); - free(attn_mask_data); - return -1; - } - padded_ids[dense_row] = id; - } else { - memcpy(padded_states + dense_row * hidden, - input_states + (size_t)(packed_offset + t) * hidden, - row_bytes); - } - if (has_padding) attn_mask_data[dense_row] = 0.0f; - } else { - if (layer_start == 0) padded_ids[dense_row] = 0; - if (has_padding) attn_mask_data[dense_row] = -1.0e9f; - } - } - packed_offset += inputs[b].n_tokens; - } - - int attn_mask_shape[] = {batch, 1, 1, max_seq}; - mlx_array attn_mask = has_padding - ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) - : (mlx_array){0}; - free(attn_mask_data); - if (has_padding && !arr_ok(attn_mask)) { - free(padded_ids); free(padded_states); - return -1; - } - if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - mlx_array attn_mask_bf16 = mlx_array_new(); - mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); - mlx_array_free(attn_mask); - attn_mask = attn_mask_bf16; - if (!arr_ok(attn_mask)) { - free(padded_ids); free(padded_states); - return -1; - } - } - - mlx_array x; - if (layer_start == 0) { - int ids_shape[] = {batch, max_seq}; - mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); - free(padded_ids); - if (!arr_ok(ids)) { - if (has_padding) mlx_array_free(attn_mask); - return -1; - } - x = mlx_array_new(); - mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); - mlx_array_free(ids); - } else { - int states_shape[] = {batch, max_seq, hidden}; - mlx_array states = mlx_array_new_data(padded_states, states_shape, 3, - MLX_FLOAT32); - free(padded_states); - if (!arr_ok(states)) { - if (has_padding) mlx_array_free(attn_mask); - return -1; - } - if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) { - x = mlx_array_new(); - mlx_astype(&x, states, MLX_BFLOAT16, S); - mlx_array_free(states); - } else { - x = states; - } - } - - x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, - layer_start, layer_end); - if (has_padding) mlx_array_free(attn_mask); - - if (apply_final_norm) { - mlx_array x_normed = mlx_array_new(); - mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S); - mlx_array_free(x); - x = x_normed; - } - - mlx_array x_f32 = mlx_array_new(); - mlx_astype(&x_f32, x, MLX_FLOAT32, S); - mlx_array_free(x); - mlx_array_eval(x_f32); - - const float *data = mlx_array_data_float32(x_f32); - if (!data) { - mlx_array_free(x_f32); - return -1; - } - packed_offset = 0; - for (int b = 0; b < batch; b++) { - memcpy(out_states + (size_t)packed_offset * hidden, - data + (size_t)b * max_seq * hidden, - (size_t)inputs[b].n_tokens * row_bytes); - packed_offset += inputs[b].n_tokens; - } - mlx_array_free(x_f32); - return 0; -} - typedef struct { int index; int span_offset; diff --git a/embed_mlx.h b/embed_mlx.h index 52a8801..ccc20c5 100644 --- a/embed_mlx.h +++ b/embed_mlx.h @@ -40,17 +40,6 @@ PPLX_API pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); PPLX_API pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, const pplx_mlx_options_t *opts); -/* - * Load one contiguous transformer layer range for distributed execution. - * Uses the same half-open range and endpoint ownership rules as - * pplx_model_load_slice(). - */ -PPLX_API pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, - int layer_start, int layer_end); -PPLX_API pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( - const char *model_dir, int layer_start, int layer_end, - const pplx_mlx_options_t *opts); - PPLX_API void pplx_mlx_free(pplx_mlx_ctx_t *ctx); /* @@ -126,19 +115,6 @@ PPLX_API int pplx_mlx_embed_batch(pplx_mlx_ctx_t *ctx, const pplx_input_t *input int batch, float *out_embeddings); /* - * MLX equivalent of pplx_model_forward_slice_batch(). - * - * Hidden states cross the API boundary as packed float32 rows even though MLX - * executes a padded dense batch internally. - */ -PPLX_API int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, - const pplx_input_t *inputs, int batch, - const float *input_states, - int layer_start, int layer_end, - int apply_final_norm, - float *out_states); - -/* * Run one contextual sequence and pool selected token spans. * Returns 0 on success, -1 on error. */ diff --git a/tests/test_cli.c b/tests/test_cli.c index fd464f0..0a5212a 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -167,9 +167,6 @@ int main(int argc, char **argv) {"--cuda", "--cuda without CUDA build"}, {"--mlx-quant-bits 4", "--mlx-quant-bits 4"}, {"--mlx-quant-group-size 0", "--mlx-quant-group-size 0"}, - {"--layers 5:2", "inverted --layers range"}, - {"--layers nope", "malformed --layers"}, - {"--dist-activation-bits 8", "--dist-activation-bits 8"}, {"--cuda-gemm-mode f32 -d X t", "--cuda-gemm-mode without CUDA"}, {"--cuda-weight-dtype bf16 -d X t", "--cuda-weight-dtype without CUDA"}, }; @@ -179,15 +176,6 @@ int main(int argc, char **argv) /* Validation that runs after the model dir is accepted. */ snprintf(args, sizeof(args), "-d '%s' --mlx-quant-bits 8 t", g_dir); expect(run_cli(args, NULL) == 1, "--mlx-quant-bits without --mlx"); - snprintf(args, sizeof(args), "-d '%s' --dist-worker", g_dir); - expect(run_cli(args, NULL) == 1, "--dist-worker without --layers"); - snprintf(args, sizeof(args), - "-d '%s' --dist-worker --dist-remote h:1 --layers 0:1", g_dir); - expect(run_cli(args, NULL) == 1, "--dist-worker with --dist-remote"); - snprintf(args, sizeof(args), - "-d '%s' --dist-activation-bits 16 t", g_dir); - expect(run_cli(args, NULL) == 1, - "--dist-activation-bits without --dist-remote"); if (g_failures) { fprintf(stderr, "%d CLI check(s) failed\n", g_failures); |
