diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-01 04:22:49 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 17a81db4313e6242f233a3300f6ffefc95b50e66 (patch) | |
| tree | 2b81258626b73432a30ff27ce79a45a42d48a8be | |
| parent | 7014b630c628ba78920aa27a7eae5ead3f2f10f9 (diff) | |
| download | ffwd-17a81db4313e6242f233a3300f6ffefc95b50e66.tar.gz ffwd-17a81db4313e6242f233a3300f6ffefc95b50e66.zip | |
server: reject unsafe mlx model overcommit
| -rw-r--r-- | main.c | 7 | ||||
| -rw-r--r-- | pplx_server.c | 103 | ||||
| -rw-r--r-- | pplx_server.h | 1 | ||||
| -rw-r--r-- | qwen_asr_safetensors.c | 16 | ||||
| -rw-r--r-- | qwen_asr_safetensors.h | 2 |
5 files changed, 129 insertions, 0 deletions
@@ -42,6 +42,8 @@ static void print_usage(const char *prog) " Load model ID from directory for --serve (repeatable)\n" " --backend cpu|mlx\n" " Backend for --serve (default: cpu)\n" + " --allow-memory-overcommit\n" + " Allow an MLX server model set above the host-memory safety budget\n" " --host HOST Server bind host (default: 127.0.0.1)\n" " --port N Server bind port (default: 8000)\n" " --cors Enable CORS headers in --serve mode\n" @@ -507,6 +509,7 @@ int main(int argc, char *argv[]) int stdin_mode = 0; int serve_mode = 0; int dist_worker_mode = 0; + int allow_memory_overcommit = 0; const char *dist_remote = NULL; int layer_start = -1; int layer_end = -1; @@ -567,6 +570,9 @@ int main(int argc, char *argv[]) } else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } + else if (!strcmp(f, "--allow-memory-overcommit")) { + allow_memory_overcommit = 1; + } else if (!strcmp(f, "--layers")) { if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) { fprintf(stderr, "--layers expects START:END with START < END\n"); @@ -633,6 +639,7 @@ int main(int argc, char *argv[]) .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, .use_mlx = use_mlx, + .allow_memory_overcommit = allow_memory_overcommit, .enable_cors = cors, .api_key = api_key, }; diff --git a/pplx_server.c b/pplx_server.c index 6bbe778..e981321 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -4,6 +4,7 @@ #include "pplx_server.h" #include "pplx_embed.h" +#include "qwen_asr_safetensors.h" #include "qwen_asr_tokenizer.h" #ifdef USE_MLX @@ -35,6 +36,10 @@ #include <time.h> #include <unistd.h> +#ifdef __APPLE__ +#include <sys/sysctl.h> +#endif + #include <cjson/cJSON.h> #define PPLX_HTTP_IO_BUF 8192 @@ -55,6 +60,8 @@ #define PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 #define PPLX_SERVER_BATCH_WAIT_US 1000 #define PPLX_SERVER_MICROBATCH_MAX_JOBS 128 +#define PPLX_MLX_MEMORY_BUDGET_PERCENT 90 +#define PPLX_MLX_RESIDENT_MULTIPLIER 2 typedef struct { char *ptr; @@ -1884,11 +1891,107 @@ static void free_models(http_server *s) { } } +static uint64_t physical_memory_nbytes(void) { +#ifdef __APPLE__ + uint64_t bytes = 0; + size_t len = sizeof(bytes); + if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && + len == sizeof(bytes) && bytes > 0) + return bytes; +#endif + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + if (pages <= 0 || page_size <= 0 || + (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) + return 0; + return (uint64_t)pages * (uint64_t)page_size; +} + +static int validate_model_specs(const pplx_server_config_t *cfg) { + int seen[4] = {0}; + for (int i = 0; i < cfg->n_models; i++) { + const pplx_server_model_spec_t *spec = &cfg->models[i]; + model_slot slot = model_slot_for_id(spec->id); + if (slot == MODEL_UNKNOWN) { + fprintf(stderr, "pplx-serve: unknown model id: %s\n", + spec->id ? spec->id : "<null>"); + return -1; + } + if (!spec->path || !spec->path[0]) { + fprintf(stderr, "pplx-serve: model %s has an empty path\n", + spec->id); + return -1; + } + if (seen[slot]) { + fprintf(stderr, "pplx-serve: duplicate model id: %s\n", spec->id); + return -1; + } + seen[slot] = 1; + } + return 0; +} + +static int mlx_memory_preflight(const pplx_server_config_t *cfg) { + if (!cfg->use_mlx) return 0; + + uint64_t payload = 0; + for (int i = 0; i < cfg->n_models; i++) { + multi_safetensors_t *ms = + multi_safetensors_open(cfg->models[i].path); + size_t model_payload = 0; + if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { + multi_safetensors_close(ms); + fprintf(stderr, "pplx-serve: failed to inspect MLX model: %s\n", + cfg->models[i].path); + return -1; + } + multi_safetensors_close(ms); + if ((uint64_t)model_payload > UINT64_MAX - payload) { + fprintf(stderr, "pplx-serve: MLX model payload size overflow\n"); + return -1; + } + payload += (uint64_t)model_payload; + } + + if (payload > UINT64_MAX / PPLX_MLX_RESIDENT_MULTIPLIER) { + fprintf(stderr, "pplx-serve: MLX resident-memory estimate overflow\n"); + return -1; + } + uint64_t estimated = payload * PPLX_MLX_RESIDENT_MULTIPLIER; + uint64_t physical = physical_memory_nbytes(); + if (physical == 0) { + server_log("pplx-serve: warning: could not determine physical memory; " + "skipping MLX memory preflight"); + return 0; + } + uint64_t budget = + physical / 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT + + physical % 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT / 100; + const double gib = 1024.0 * 1024.0 * 1024.0; + server_log("pplx-serve: MLX memory preflight: %.1f GiB tensors, " + "%.1f GiB estimated resident, %.1f GiB safety budget", + (double)payload / gib, (double)estimated / gib, + (double)budget / gib); + if (estimated <= budget) return 0; + if (cfg->allow_memory_overcommit) { + server_log("pplx-serve: warning: MLX memory estimate exceeds safety " + "budget; continuing due to --allow-memory-overcommit"); + return 0; + } + server_log("pplx-serve: refusing MLX model set above host-memory safety " + "budget"); + server_log("pplx-serve: use BF16 artifacts, load fewer models, or pass " + "--allow-memory-overcommit"); + return -1; +} + int pplx_run_server(const pplx_server_config_t *cfg) { if (!cfg || !cfg->models || cfg->n_models <= 0) { fprintf(stderr, "pplx-serve: at least one --model MODEL_ID=PATH is required\n"); return 1; } + if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0) + return 1; http_server s; memset(&s, 0, sizeof(s)); diff --git a/pplx_server.h b/pplx_server.h index 08a3a0d..188d467 100644 --- a/pplx_server.h +++ b/pplx_server.h @@ -17,6 +17,7 @@ typedef struct { int max_batch_tokens; int batch_wait_us; int use_mlx; + int allow_memory_overcommit; int enable_cors; const char *api_key; } pplx_server_config_t; diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c index 86136f7..726ee9c 100644 --- a/qwen_asr_safetensors.c +++ b/qwen_asr_safetensors.c @@ -449,6 +449,22 @@ void multi_safetensors_close(multi_safetensors_t *ms) { free(ms); } +int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, + size_t *out_nbytes) { + if (!ms || !out_nbytes) return -1; + size_t total = 0; + for (int s = 0; s < ms->num_shards; s++) { + const safetensors_file_t *sf = ms->shards[s]; + for (int i = 0; i < sf->num_tensors; i++) { + size_t n = sf->tensors[i].data_size; + if (n > SIZE_MAX - total) return -1; + total += n; + } + } + *out_nbytes = total; + return 0; +} + const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) { diff --git a/qwen_asr_safetensors.h b/qwen_asr_safetensors.h index 4cb09f7..7bd1846 100644 --- a/qwen_asr_safetensors.h +++ b/qwen_asr_safetensors.h @@ -54,6 +54,8 @@ void safetensors_close(safetensors_file_t *sf); /* Open model from directory (auto-detects single file or multi-shard) */ multi_safetensors_t *multi_safetensors_open(const char *model_dir); void multi_safetensors_close(multi_safetensors_t *ms); +int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, + size_t *out_nbytes); /* Find a tensor by name across all shards */ const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, |
