From 9d69c13793c4c69c080ed4273bcd36ded69de2bb Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 3 Jun 2026 00:30:12 +0200 Subject: mlx: add opt-in 8-bit weight quantization --- main.c | 60 ++++++++++++-- pplx_distributed.c | 38 ++++++++- pplx_distributed.h | 12 +++ pplx_embed_mlx.c | 227 ++++++++++++++++++++++++++++++++++++++++++++++++----- pplx_embed_mlx.h | 10 +++ pplx_server.c | 16 +++- pplx_server.h | 2 + 7 files changed, 332 insertions(+), 33 deletions(-) diff --git a/main.c b/main.c index ab5eb3e..86ee2e8 100644 --- a/main.c +++ b/main.c @@ -42,6 +42,10 @@ static void print_usage(const char *prog) " Load model ID from directory for --serve (repeatable)\n" " --backend cpu|mlx\n" " Backend for --serve (default: cpu)\n" + " --mlx-quantize-bits N\n" + " Quantize MLX linear weights to 8 bits at load time\n" + " --mlx-quantize-group-size N\n" + " MLX quantization group size (default: 64)\n" " --allow-memory-overcommit\n" " Allow an MLX server model set above the host-memory safety budget\n" " --host HOST Server bind host (default: 127.0.0.1)\n" @@ -516,6 +520,8 @@ int main(int argc, char *argv[]) int batch_size = 0; int max_batch_tokens = 0; int batch_wait_us = -1; + int mlx_quantize_bits = 0; + int mlx_quantize_group_size = 64; const char *backend = "cpu"; const char *host = "127.0.0.1"; const char *api_key = NULL; @@ -568,6 +574,22 @@ int main(int argc, char *argv[]) return 1; } } + else if (!strcmp(f, "--mlx-quantize-bits")) { + mlx_quantize_bits = atoi(argv[++arg_start]); + if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { + fprintf(stderr, "--mlx-quantize-bits must be 0 or 8\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx-quantize-group-size")) { + mlx_quantize_group_size = atoi(argv[++arg_start]); + if (mlx_quantize_group_size <= 0) { + fprintf(stderr, "--mlx-quantize-group-size must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } else if (!strcmp(f, "--allow-memory-overcommit")) { @@ -623,6 +645,11 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } + if (mlx_quantize_bits && !use_mlx) { + fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n"); + free_model_specs(&model_specs); + return 1; + } if (!use_mlx) { if (n_threads <= 0) n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); @@ -639,6 +666,8 @@ int main(int argc, char *argv[]) .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, .use_mlx = use_mlx, + .mlx_quantize_bits = mlx_quantize_bits, + .mlx_quantize_group_size = mlx_quantize_group_size, .allow_memory_overcommit = allow_memory_overcommit, .enable_cors = cors, .api_key = api_key, @@ -684,9 +713,20 @@ int main(int argc, char *argv[]) if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); } + if (mlx_quantize_bits && !use_mlx) { + fprintf(stderr, "--mlx-quantize-bits requires --mlx or --backend mlx\n"); + return 1; + } + + pplx_dist_options_t dist_opts = { + .mlx_quantize_bits = mlx_quantize_bits, + .mlx_quantize_group_size = mlx_quantize_group_size, + }; + if (dist_worker_mode) - return pplx_dist_run_worker(model_dir, host, port, layer_start, - layer_end, use_mlx); + return pplx_dist_run_worker_with_options( + model_dir, host, port, layer_start, layer_end, use_mlx, + &dist_opts); /* Tokenizer */ char vocab_path[1024]; @@ -711,8 +751,9 @@ int main(int argc, char *argv[]) qwen_tokenizer_free(tok); return 1; } - e.dist = pplx_dist_client_open(model_dir, remote_host, remote_port, - layer_end, use_mlx); + e.dist = pplx_dist_client_open_with_options( + model_dir, remote_host, remote_port, layer_end, use_mlx, + &dist_opts); if (!e.dist) { fprintf(stderr, "failed to connect distributed worker\n"); qwen_tokenizer_free(tok); @@ -720,7 +761,11 @@ int main(int argc, char *argv[]) } e.dim = pplx_dist_client_config(e.dist)->hidden_size; } else if (use_mlx) { - e.mlx_ctx = pplx_mlx_load(model_dir); + pplx_mlx_options_t mlx_opts = { + .quantize_bits = mlx_quantize_bits, + .quantize_group_size = mlx_quantize_group_size, + }; + e.mlx_ctx = pplx_mlx_load_with_options(model_dir, &mlx_opts); if (!e.mlx_ctx) { fprintf(stderr, "failed to load model\n"); qwen_tokenizer_free(tok); @@ -739,8 +784,9 @@ int main(int argc, char *argv[]) qwen_tokenizer_free(tok); return 1; } - e.dist = pplx_dist_client_open(model_dir, remote_host, remote_port, - layer_end, use_mlx); + e.dist = pplx_dist_client_open_with_options( + model_dir, remote_host, remote_port, layer_end, use_mlx, + &dist_opts); if (!e.dist) { fprintf(stderr, "failed to connect distributed worker\n"); qwen_tokenizer_free(tok); diff --git a/pplx_distributed.c b/pplx_distributed.c index f65fc55..79eae70 100644 --- a/pplx_distributed.c +++ b/pplx_distributed.c @@ -310,12 +310,22 @@ static int connect_endpoint(const char *host, int port) } static int stage_load(dist_stage *stage, const char *model_dir, - int layer_start, int layer_end, int use_mlx) + int layer_start, int layer_end, int use_mlx, + const pplx_dist_options_t *opts) { memset(stage, 0, sizeof(*stage)); +#ifndef USE_MLX + (void)opts; +#endif #ifdef USE_MLX if (use_mlx) { - stage->mlx = pplx_mlx_load_slice(model_dir, layer_start, layer_end); + pplx_mlx_options_t mlx_opts = {0}; + if (opts) { + mlx_opts.quantize_bits = opts->mlx_quantize_bits; + mlx_opts.quantize_group_size = opts->mlx_quantize_group_size; + } + stage->mlx = pplx_mlx_load_slice_with_options( + model_dir, layer_start, layer_end, &mlx_opts); if (!stage->mlx) return -1; stage->cfg = pplx_mlx_config(stage->mlx); return 0; @@ -534,6 +544,16 @@ static int worker_serve_connection(int fd, dist_stage *stage, int pplx_dist_run_worker(const char *model_dir, const char *host, int port, int layer_start, int layer_end, int use_mlx) +{ + return pplx_dist_run_worker_with_options(model_dir, host, port, + layer_start, layer_end, use_mlx, + NULL); +} + +int pplx_dist_run_worker_with_options( + const char *model_dir, const char *host, int port, + int layer_start, int layer_end, int use_mlx, + const pplx_dist_options_t *opts) { if (!model_dir || !host || port <= 0 || port > 65535 || !host_is_little_endian()) { @@ -541,7 +561,8 @@ int pplx_dist_run_worker(const char *model_dir, const char *host, int port, return 1; } dist_stage stage; - if (stage_load(&stage, model_dir, layer_start, layer_end, use_mlx) != 0) { + if (stage_load(&stage, model_dir, layer_start, layer_end, use_mlx, + opts) != 0) { fprintf(stderr, "pplx-dist: failed to load worker shard\n"); return 1; } @@ -581,6 +602,14 @@ int pplx_dist_run_worker(const char *model_dir, const char *host, int port, pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, const char *host, int port, int local_layer_end, int use_mlx) +{ + return pplx_dist_client_open_with_options(model_dir, host, port, + local_layer_end, use_mlx, NULL); +} + +pplx_dist_client_t *pplx_dist_client_open_with_options( + const char *model_dir, const char *host, int port, + int local_layer_end, int use_mlx, const pplx_dist_options_t *opts) { if (!model_dir || !host || port <= 0 || port > 65535 || local_layer_end <= 0 || !host_is_little_endian()) @@ -589,7 +618,8 @@ pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, pplx_dist_client_t *client = calloc(1, sizeof(*client)); if (!client) return NULL; client->fd = -1; - if (stage_load(&client->local, model_dir, 0, local_layer_end, use_mlx) != 0) + if (stage_load(&client->local, model_dir, 0, local_layer_end, use_mlx, + opts) != 0) goto fail; client->fd = connect_endpoint(host, port); if (client->fd < 0 || diff --git a/pplx_distributed.h b/pplx_distributed.h index 2979ede..f84056c 100644 --- a/pplx_distributed.h +++ b/pplx_distributed.h @@ -9,6 +9,11 @@ typedef struct pplx_dist_client pplx_dist_client_t; +typedef struct { + int mlx_quantize_bits; + int mlx_quantize_group_size; +} pplx_dist_options_t; + /* * Open a coordinator-side connection. The coordinator owns layers * [0, local_layer_end); the remote worker must own [local_layer_end, N). @@ -16,6 +21,9 @@ typedef struct pplx_dist_client pplx_dist_client_t; pplx_dist_client_t *pplx_dist_client_open(const char *model_dir, const char *host, int port, int local_layer_end, int use_mlx); +pplx_dist_client_t *pplx_dist_client_open_with_options( + const char *model_dir, const char *host, int port, + int local_layer_end, int use_mlx, const pplx_dist_options_t *opts); void pplx_dist_client_free(pplx_dist_client_t *client); const pplx_config_t *pplx_dist_client_config(const pplx_dist_client_t *client); @@ -30,5 +38,9 @@ int pplx_dist_client_embed_batch(pplx_dist_client_t *client, */ int pplx_dist_run_worker(const char *model_dir, const char *host, int port, int layer_start, int layer_end, int use_mlx); +int pplx_dist_run_worker_with_options( + const char *model_dir, const char *host, int port, + int layer_start, int layer_end, int use_mlx, + const pplx_dist_options_t *opts); #endif /* PPLX_DISTRIBUTED_H */ diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index 873e355..7bda478 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -15,10 +15,19 @@ * ======================================================================== */ typedef struct { - mlx_array wq, wk, wv, wo; + mlx_array w; + mlx_array scales; + mlx_array biases; + int quantized; + int bits; + int group_size; +} mlx_linear_t; + +typedef struct { + mlx_linear_t wq, wk, wv, wo; mlx_array q_norm, k_norm; mlx_array input_norm, post_attn_norm; - mlx_array gate_proj, up_proj, down_proj; + mlx_linear_t gate_proj, up_proj, down_proj; } mlx_layer_t; struct pplx_mlx_ctx { @@ -30,6 +39,9 @@ struct pplx_mlx_ctx { multi_safetensors_t *ms; int layer_start; /* loaded transformer range, inclusive */ int layer_end; /* loaded transformer range, exclusive */ + int quantize_bits; + int quantize_group_size; + mlx_dtype weight_dtype; }; /* ======================================================================== @@ -47,7 +59,7 @@ static int mlx_mul_size(size_t a, size_t b, size_t *out) static mlx_dtype mlx_weight_dtype(const pplx_mlx_ctx_t *ctx) { - return mlx_array_dtype(ctx->layers[ctx->layer_start].wq); + return ctx->weight_dtype; } static size_t safetensor_dtype_size(safetensor_dtype_t dtype) @@ -131,11 +143,95 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); } +static mlx_linear_t load_linear_tensor(multi_safetensors_t *ms, + const char *name, + const int *shape, int ndim) +{ + mlx_linear_t l = {0}; + l.w = load_tensor(ms, name, shape, ndim); + return l; +} + +static int linear_ok(mlx_linear_t l) +{ + return arr_ok(l.w); +} + +static void free_linear(mlx_linear_t *l) +{ + mlx_array_free(l->w); + mlx_array_free(l->scales); + mlx_array_free(l->biases); + memset(l, 0, sizeof(*l)); +} + +static int quantize_linear(mlx_linear_t *l, int bits, int group_size, + mlx_stream s) +{ + if (!l || !arr_ok(l->w) || bits == 0) return 0; + if (bits != 8) { + fprintf(stderr, "mlx: quantized weights support only 8 bits\n"); + return -1; + } + if (group_size <= 0) group_size = 64; + + mlx_optional_int gs = {.value = group_size, .has_value = true}; + mlx_optional_int qb = {.value = bits, .has_value = true}; + mlx_array null_arr = (mlx_array){0}; + mlx_vector_array q = mlx_vector_array_new(); + mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s); + if (!q.ctx || mlx_vector_array_size(q) < 2) { + mlx_vector_array_free(q); + fprintf(stderr, "mlx: quantize failed\n"); + return -1; + } + + mlx_array qw = (mlx_array){0}; + mlx_array scales = (mlx_array){0}; + mlx_array biases = (mlx_array){0}; + mlx_vector_array_get(&qw, q, 0); + mlx_vector_array_get(&scales, q, 1); + if (mlx_vector_array_size(q) >= 3) + mlx_vector_array_get(&biases, q, 2); + mlx_vector_array_free(q); + + if (!arr_ok(qw) || !arr_ok(scales)) { + mlx_array_free(qw); + mlx_array_free(scales); + mlx_array_free(biases); + fprintf(stderr, "mlx: quantize returned invalid arrays\n"); + return -1; + } + mlx_array_eval(qw); + mlx_array_eval(scales); + if (arr_ok(biases)) mlx_array_eval(biases); + mlx_synchronize(s); + + mlx_array_free(l->w); + l->w = qw; + l->scales = scales; + l->biases = biases; + l->quantized = 1; + l->bits = bits; + l->group_size = group_size; + return 0; +} + /* y = x @ W^T */ -static mlx_array linear(mlx_array x, mlx_array W, mlx_stream s) +static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) { + if (W->quantized) { + mlx_array res = mlx_array_new(); + mlx_optional_int gs = {.value = W->group_size, .has_value = true}; + mlx_optional_int bits = {.value = W->bits, .has_value = true}; + mlx_array null_arr = (mlx_array){0}; + mlx_quantized_matmul(&res, x, W->w, W->scales, + arr_ok(W->biases) ? W->biases : null_arr, + true, gs, bits, "affine", s); + return res; + } mlx_array Wt = mlx_array_new(); - mlx_transpose(&Wt, W, s); + mlx_transpose(&Wt, W->w, s); mlx_array res = mlx_array_new(); mlx_matmul(&res, x, Wt, s); mlx_array_free(Wt); @@ -245,11 +341,44 @@ static int mlx_parse_config(pplx_config_t *cfg, const char *model_dir) return 0; } +static pplx_mlx_options_t normalize_mlx_options(const pplx_mlx_options_t *opts) +{ + pplx_mlx_options_t out = {0}; + if (opts) out = *opts; + if (out.quantize_bits != 0 && + out.quantize_bits != 8) { + fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); + out.quantize_bits = -1; + } + if (out.quantize_bits && out.quantize_group_size <= 0) + out.quantize_group_size = 64; + return out; +} + +static int quantize_layer(mlx_layer_t *l, int bits, int group_size, + mlx_stream s) +{ + if (bits == 0) return 0; + return quantize_linear(&l->wq, bits, group_size, s) || + quantize_linear(&l->wk, bits, group_size, s) || + quantize_linear(&l->wv, bits, group_size, s) || + quantize_linear(&l->wo, bits, group_size, s) || + quantize_linear(&l->gate_proj, bits, group_size, s) || + quantize_linear(&l->up_proj, bits, group_size, s) || + quantize_linear(&l->down_proj, bits, group_size, s); +} + static pplx_mlx_ctx_t *mlx_load_range(const char *model_dir, - int layer_start, int layer_end) + int layer_start, int layer_end, + const pplx_mlx_options_t *opts) { + pplx_mlx_options_t options = normalize_mlx_options(opts); + if (options.quantize_bits < 0) return NULL; + pplx_mlx_ctx_t *ctx = calloc(1, sizeof(pplx_mlx_ctx_t)); if (!ctx) return NULL; + ctx->quantize_bits = options.quantize_bits; + ctx->quantize_group_size = options.quantize_group_size; if (mlx_parse_config(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } @@ -294,18 +423,36 @@ static pplx_mlx_ctx_t *mlx_load_range(const char *model_dir, #define LD(fld, fmt, ...) do { \ snprintf(name, sizeof(name), fmt, i); \ int sh[] = {__VA_ARGS__}; \ - l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ - if (!arr_ok(l->fld)) goto fail; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ + if (!linear_ok(l->fld)) goto fail; \ } while (0) LD(wq, "layers.%d.self_attn.q_proj.weight", qd, h); LD(wk, "layers.%d.self_attn.k_proj.weight", kvd, h); LD(wv, "layers.%d.self_attn.v_proj.weight", kvd, h); LD(wo, "layers.%d.self_attn.o_proj.weight", h, qd); +#undef LD + +#define LD(fld, fmt, ...) do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ + if (!arr_ok(l->fld)) goto fail; \ +} while (0) + LD(q_norm, "layers.%d.self_attn.q_norm.weight", hd); LD(k_norm, "layers.%d.self_attn.k_norm.weight", hd); LD(input_norm, "layers.%d.input_layernorm.weight", h); LD(post_attn_norm, "layers.%d.post_attention_layernorm.weight", h); +#undef LD + +#define LD(fld, fmt, ...) do { \ + snprintf(name, sizeof(name), fmt, i); \ + int sh[] = {__VA_ARGS__}; \ + l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh)/sizeof(sh[0])); \ + if (!linear_ok(l->fld)) goto fail; \ +} while (0) + LD(gate_proj, "layers.%d.mlp.gate_proj.weight", inter, h); LD(up_proj, "layers.%d.mlp.up_proj.weight", inter, h); LD(down_proj, "layers.%d.mlp.down_proj.weight", h, inter); @@ -321,6 +468,31 @@ static pplx_mlx_ctx_t *mlx_load_range(const char *model_dir, } ctx->stream = mlx_default_gpu_stream_new(); + ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w); + if (ctx->quantize_bits) { + for (int i = layer_start; i < layer_end; i++) { + if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, + ctx->quantize_group_size, ctx->stream) != 0) + goto fail; + } + mlx_clear_cache(); + if (pplx_verbose >= 1) + fprintf(stderr, "mlx: quantized linear weights to %d-bit " + "(group_size=%d)\n", + ctx->quantize_bits, ctx->quantize_group_size); + } + + if (pplx_verbose >= 2) { + size_t active = 0, cache = 0, peak = 0; + mlx_get_active_memory(&active); + mlx_get_cache_memory(&cache); + mlx_get_peak_memory(&peak); + fprintf(stderr, "mlx: memory active=%.2f GiB cache=%.2f GiB " + "peak=%.2f GiB\n", + (double)active / (1024.0 * 1024.0 * 1024.0), + (double)cache / (1024.0 * 1024.0 * 1024.0), + (double)peak / (1024.0 * 1024.0 * 1024.0)); + } if (pplx_verbose >= 1) fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", @@ -335,13 +507,26 @@ fail: pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir) { - return mlx_load_range(model_dir, 0, -1); + return mlx_load_range(model_dir, 0, -1, NULL); +} + +pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, + const pplx_mlx_options_t *opts) +{ + return mlx_load_range(model_dir, 0, -1, opts); } pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, int layer_start, int layer_end) { - return mlx_load_range(model_dir, layer_start, layer_end); + return mlx_load_range(model_dir, layer_start, layer_end, NULL); +} + +pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( + const char *model_dir, int layer_start, int layer_end, + const pplx_mlx_options_t *opts) +{ + return mlx_load_range(model_dir, layer_start, layer_end, opts); } /* ======================================================================== @@ -350,12 +535,12 @@ pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, static void free_mlx_layer(mlx_layer_t *l) { - mlx_array_free(l->wq); mlx_array_free(l->wk); - mlx_array_free(l->wv); mlx_array_free(l->wo); + free_linear(&l->wq); free_linear(&l->wk); + free_linear(&l->wv); free_linear(&l->wo); mlx_array_free(l->q_norm); mlx_array_free(l->k_norm); mlx_array_free(l->input_norm); mlx_array_free(l->post_attn_norm); - mlx_array_free(l->gate_proj); mlx_array_free(l->up_proj); - mlx_array_free(l->down_proj); + free_linear(&l->gate_proj); free_linear(&l->up_proj); + free_linear(&l->down_proj); } void pplx_mlx_free(pplx_mlx_ctx_t *ctx) @@ -397,9 +582,9 @@ static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, mlx_array xn = mlx_array_new(); mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S); - mlx_array q_flat = linear(xn, l->wq, S); - mlx_array k_flat = linear(xn, l->wk, S); - mlx_array v_flat = linear(xn, l->wv, S); + mlx_array q_flat = linear(xn, &l->wq, S); + mlx_array k_flat = linear(xn, &l->wk, S); + mlx_array v_flat = linear(xn, &l->wv, S); mlx_array_free(xn); int q_shape[] = {batch, max_seq, n_heads, head_dim}; @@ -447,7 +632,7 @@ static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S); mlx_array_free(attn_t); - mlx_array proj = linear(attn_flat, l->wo, S); + mlx_array proj = linear(attn_flat, &l->wo, S); mlx_array_free(attn_flat); mlx_array x2 = mlx_array_new(); mlx_add(&x2, x, proj, S); @@ -457,8 +642,8 @@ static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, mlx_array xn2 = mlx_array_new(); mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S); - mlx_array gate = linear(xn2, l->gate_proj, S); - mlx_array up = linear(xn2, l->up_proj, S); + mlx_array gate = linear(xn2, &l->gate_proj, S); + mlx_array up = linear(xn2, &l->up_proj, S); mlx_array_free(xn2); mlx_array gate_sig = mlx_array_new(); @@ -471,7 +656,7 @@ static mlx_array mlx_forward_layers(pplx_mlx_ctx_t *ctx, mlx_array x, mlx_multiply(&mid, silu_gate, up, S); mlx_array_free(silu_gate); mlx_array_free(up); - mlx_array ffn = linear(mid, l->down_proj, S); + mlx_array ffn = linear(mid, &l->down_proj, S); mlx_array_free(mid); mlx_array x3 = mlx_array_new(); diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h index 2b7164d..9b341e0 100644 --- a/pplx_embed_mlx.h +++ b/pplx_embed_mlx.h @@ -14,11 +14,18 @@ /* Opaque MLX model context */ typedef struct pplx_mlx_ctx pplx_mlx_ctx_t; +typedef struct { + int quantize_bits; /* 0 disables; currently 8 */ + int quantize_group_size; /* default 64 when quantize_bits is set */ +} pplx_mlx_options_t; + /* * Load model into MLX arrays from safetensors + config.json. * Returns NULL on error. */ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); +pplx_mlx_ctx_t *pplx_mlx_load_with_options(const char *model_dir, + const pplx_mlx_options_t *opts); /* * Load one contiguous transformer layer range for distributed execution. @@ -27,6 +34,9 @@ pplx_mlx_ctx_t *pplx_mlx_load(const char *model_dir); */ pplx_mlx_ctx_t *pplx_mlx_load_slice(const char *model_dir, int layer_start, int layer_end); +pplx_mlx_ctx_t *pplx_mlx_load_slice_with_options( + const char *model_dir, int layer_start, int layer_end, + const pplx_mlx_options_t *opts); void pplx_mlx_free(pplx_mlx_ctx_t *ctx); diff --git a/pplx_server.c b/pplx_server.c index 2bee40c..231e959 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -230,6 +230,8 @@ typedef struct { char *api_key; loaded_model models[4]; int use_mlx; + int mlx_quantize_bits; + int mlx_quantize_group_size; pthread_t worker; pthread_mutex_t mu; @@ -1777,7 +1779,11 @@ static void *worker_main(void *arg) { for (int i = 0; i < 4; i++) { loaded_model *m = &s->models[i]; if (!m->info) continue; - m->mlx_ctx = pplx_mlx_load(m->path); + pplx_mlx_options_t mlx_opts = { + .quantize_bits = s->mlx_quantize_bits, + .quantize_group_size = s->mlx_quantize_group_size, + }; + m->mlx_ctx = pplx_mlx_load_with_options(m->path, &mlx_opts); if (!m->mlx_ctx) { server_log("pplx-serve: failed to load MLX model on worker: %s", m->path); @@ -1995,6 +2001,11 @@ static int mlx_memory_preflight(const pplx_server_config_t *cfg) { "%.1f GiB estimated resident, %.1f GiB safety budget", (double)payload / gib, (double)estimated / gib, (double)budget / gib); + if (cfg->mlx_quantize_bits) { + server_log("pplx-serve: MLX %d-bit quantization enabled; preflight " + "uses source payload as a conservative peak estimate", + cfg->mlx_quantize_bits); + } if (estimated <= budget) return 0; if (cfg->allow_memory_overcommit) { server_log("pplx-serve: warning: MLX memory estimate exceeds safety " @@ -2028,6 +2039,9 @@ int pplx_run_server(const pplx_server_config_t *cfg) { ? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US; s.enable_cors = cfg->enable_cors; s.use_mlx = cfg->use_mlx; + s.mlx_quantize_bits = cfg->mlx_quantize_bits; + s.mlx_quantize_group_size = cfg->mlx_quantize_group_size > 0 + ? cfg->mlx_quantize_group_size : 64; if (cfg->api_key && cfg->api_key[0]) s.api_key = xstrdup(cfg->api_key); else { diff --git a/pplx_server.h b/pplx_server.h index 188d467..72d4a6c 100644 --- a/pplx_server.h +++ b/pplx_server.h @@ -17,6 +17,8 @@ typedef struct { int max_batch_tokens; int batch_wait_us; int use_mlx; + int mlx_quantize_bits; + int mlx_quantize_group_size; int allow_memory_overcommit; int enable_cors; const char *api_key; -- cgit v1.3.1