aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-01 04:22:49 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit17a81db4313e6242f233a3300f6ffefc95b50e66 (patch)
tree2b81258626b73432a30ff27ce79a45a42d48a8be
parent7014b630c628ba78920aa27a7eae5ead3f2f10f9 (diff)
downloadffwd-17a81db4313e6242f233a3300f6ffefc95b50e66.tar.gz
ffwd-17a81db4313e6242f233a3300f6ffefc95b50e66.zip
server: reject unsafe mlx model overcommit
-rw-r--r--main.c7
-rw-r--r--pplx_server.c103
-rw-r--r--pplx_server.h1
-rw-r--r--qwen_asr_safetensors.c16
-rw-r--r--qwen_asr_safetensors.h2
5 files changed, 129 insertions, 0 deletions
diff --git a/main.c b/main.c
index 8bcf405..ab5eb3e 100644
--- a/main.c
+++ b/main.c
@@ -42,6 +42,8 @@ static void print_usage(const char *prog)
" Load model ID from directory for --serve (repeatable)\n"
" --backend cpu|mlx\n"
" Backend for --serve (default: cpu)\n"
+ " --allow-memory-overcommit\n"
+ " Allow an MLX server model set above the host-memory safety budget\n"
" --host HOST Server bind host (default: 127.0.0.1)\n"
" --port N Server bind port (default: 8000)\n"
" --cors Enable CORS headers in --serve mode\n"
@@ -507,6 +509,7 @@ int main(int argc, char *argv[])
int stdin_mode = 0;
int serve_mode = 0;
int dist_worker_mode = 0;
+ int allow_memory_overcommit = 0;
const char *dist_remote = NULL;
int layer_start = -1;
int layer_end = -1;
@@ -567,6 +570,9 @@ int main(int argc, char *argv[])
}
else if (!strcmp(f, "--host")) { host = argv[++arg_start]; }
else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); }
+ else if (!strcmp(f, "--allow-memory-overcommit")) {
+ allow_memory_overcommit = 1;
+ }
else if (!strcmp(f, "--layers")) {
if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) {
fprintf(stderr, "--layers expects START:END with START < END\n");
@@ -633,6 +639,7 @@ int main(int argc, char *argv[])
.max_batch_tokens = max_batch_tokens,
.batch_wait_us = batch_wait_us,
.use_mlx = use_mlx,
+ .allow_memory_overcommit = allow_memory_overcommit,
.enable_cors = cors,
.api_key = api_key,
};
diff --git a/pplx_server.c b/pplx_server.c
index 6bbe778..e981321 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -4,6 +4,7 @@
#include "pplx_server.h"
#include "pplx_embed.h"
+#include "qwen_asr_safetensors.h"
#include "qwen_asr_tokenizer.h"
#ifdef USE_MLX
@@ -35,6 +36,10 @@
#include <time.h>
#include <unistd.h>
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+#endif
+
#include <cjson/cJSON.h>
#define PPLX_HTTP_IO_BUF 8192
@@ -55,6 +60,8 @@
#define PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
#define PPLX_SERVER_BATCH_WAIT_US 1000
#define PPLX_SERVER_MICROBATCH_MAX_JOBS 128
+#define PPLX_MLX_MEMORY_BUDGET_PERCENT 90
+#define PPLX_MLX_RESIDENT_MULTIPLIER 2
typedef struct {
char *ptr;
@@ -1884,11 +1891,107 @@ static void free_models(http_server *s) {
}
}
+static uint64_t physical_memory_nbytes(void) {
+#ifdef __APPLE__
+ uint64_t bytes = 0;
+ size_t len = sizeof(bytes);
+ if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 &&
+ len == sizeof(bytes) && bytes > 0)
+ return bytes;
+#endif
+ long pages = sysconf(_SC_PHYS_PAGES);
+ long page_size = sysconf(_SC_PAGESIZE);
+ if (pages <= 0 || page_size <= 0 ||
+ (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
+ return 0;
+ return (uint64_t)pages * (uint64_t)page_size;
+}
+
+static int validate_model_specs(const pplx_server_config_t *cfg) {
+ int seen[4] = {0};
+ for (int i = 0; i < cfg->n_models; i++) {
+ const pplx_server_model_spec_t *spec = &cfg->models[i];
+ model_slot slot = model_slot_for_id(spec->id);
+ if (slot == MODEL_UNKNOWN) {
+ fprintf(stderr, "pplx-serve: unknown model id: %s\n",
+ spec->id ? spec->id : "<null>");
+ return -1;
+ }
+ if (!spec->path || !spec->path[0]) {
+ fprintf(stderr, "pplx-serve: model %s has an empty path\n",
+ spec->id);
+ return -1;
+ }
+ if (seen[slot]) {
+ fprintf(stderr, "pplx-serve: duplicate model id: %s\n", spec->id);
+ return -1;
+ }
+ seen[slot] = 1;
+ }
+ return 0;
+}
+
+static int mlx_memory_preflight(const pplx_server_config_t *cfg) {
+ if (!cfg->use_mlx) return 0;
+
+ uint64_t payload = 0;
+ for (int i = 0; i < cfg->n_models; i++) {
+ multi_safetensors_t *ms =
+ multi_safetensors_open(cfg->models[i].path);
+ size_t model_payload = 0;
+ if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) {
+ multi_safetensors_close(ms);
+ fprintf(stderr, "pplx-serve: failed to inspect MLX model: %s\n",
+ cfg->models[i].path);
+ return -1;
+ }
+ multi_safetensors_close(ms);
+ if ((uint64_t)model_payload > UINT64_MAX - payload) {
+ fprintf(stderr, "pplx-serve: MLX model payload size overflow\n");
+ return -1;
+ }
+ payload += (uint64_t)model_payload;
+ }
+
+ if (payload > UINT64_MAX / PPLX_MLX_RESIDENT_MULTIPLIER) {
+ fprintf(stderr, "pplx-serve: MLX resident-memory estimate overflow\n");
+ return -1;
+ }
+ uint64_t estimated = payload * PPLX_MLX_RESIDENT_MULTIPLIER;
+ uint64_t physical = physical_memory_nbytes();
+ if (physical == 0) {
+ server_log("pplx-serve: warning: could not determine physical memory; "
+ "skipping MLX memory preflight");
+ return 0;
+ }
+ uint64_t budget =
+ physical / 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT +
+ physical % 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT / 100;
+ const double gib = 1024.0 * 1024.0 * 1024.0;
+ server_log("pplx-serve: MLX memory preflight: %.1f GiB tensors, "
+ "%.1f GiB estimated resident, %.1f GiB safety budget",
+ (double)payload / gib, (double)estimated / gib,
+ (double)budget / gib);
+ if (estimated <= budget) return 0;
+ if (cfg->allow_memory_overcommit) {
+ server_log("pplx-serve: warning: MLX memory estimate exceeds safety "
+ "budget; continuing due to --allow-memory-overcommit");
+ return 0;
+ }
+ server_log("pplx-serve: refusing MLX model set above host-memory safety "
+ "budget");
+ server_log("pplx-serve: use BF16 artifacts, load fewer models, or pass "
+ "--allow-memory-overcommit");
+ return -1;
+}
+
int pplx_run_server(const pplx_server_config_t *cfg) {
if (!cfg || !cfg->models || cfg->n_models <= 0) {
fprintf(stderr, "pplx-serve: at least one --model MODEL_ID=PATH is required\n");
return 1;
}
+ if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0)
+ return 1;
http_server s;
memset(&s, 0, sizeof(s));
diff --git a/pplx_server.h b/pplx_server.h
index 08a3a0d..188d467 100644
--- a/pplx_server.h
+++ b/pplx_server.h
@@ -17,6 +17,7 @@ typedef struct {
int max_batch_tokens;
int batch_wait_us;
int use_mlx;
+ int allow_memory_overcommit;
int enable_cors;
const char *api_key;
} pplx_server_config_t;
diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c
index 86136f7..726ee9c 100644
--- a/qwen_asr_safetensors.c
+++ b/qwen_asr_safetensors.c
@@ -449,6 +449,22 @@ void multi_safetensors_close(multi_safetensors_t *ms) {
free(ms);
}
+int multi_safetensors_data_nbytes(const multi_safetensors_t *ms,
+ size_t *out_nbytes) {
+ if (!ms || !out_nbytes) return -1;
+ size_t total = 0;
+ for (int s = 0; s < ms->num_shards; s++) {
+ const safetensors_file_t *sf = ms->shards[s];
+ for (int i = 0; i < sf->num_tensors; i++) {
+ size_t n = sf->tensors[i].data_size;
+ if (n > SIZE_MAX - total) return -1;
+ total += n;
+ }
+ }
+ *out_nbytes = total;
+ return 0;
+}
+
const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
const char *name,
safetensors_file_t **out_sf) {
diff --git a/qwen_asr_safetensors.h b/qwen_asr_safetensors.h
index 4cb09f7..7bd1846 100644
--- a/qwen_asr_safetensors.h
+++ b/qwen_asr_safetensors.h
@@ -54,6 +54,8 @@ void safetensors_close(safetensors_file_t *sf);
/* Open model from directory (auto-detects single file or multi-shard) */
multi_safetensors_t *multi_safetensors_open(const char *model_dir);
void multi_safetensors_close(multi_safetensors_t *ms);
+int multi_safetensors_data_nbytes(const multi_safetensors_t *ms,
+ size_t *out_nbytes);
/* Find a tensor by name across all shards */
const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,