aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 08:28:13 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitac994a45e239bdbb9aa5c34cbab2de71295ba420 (patch)
tree98c7adf49d156f641d5f0b1e02bcc79aea75991f
parent2c37727e653d2340d8977dd5db4addc767dd81ea (diff)
downloadffwd-ac994a45e239bdbb9aa5c34cbab2de71295ba420.tar.gz
ffwd-ac994a45e239bdbb9aa5c34cbab2de71295ba420.zip
chore: rename sources; binaries pplx-embed, pplx-embed-server
-rw-r--r--Makefile49
-rw-r--r--embed.c4
-rw-r--r--embed.h2
-rw-r--r--embed_cli.c10
-rw-r--r--embed_cuda.cu4
-rw-r--r--embed_cuda.h4
-rw-r--r--embed_distributed.c6
-rw-r--r--embed_distributed.h4
-rw-r--r--embed_internal.h2
-rw-r--r--embed_mlx.c6
-rw-r--r--embed_mlx.h6
-rw-r--r--embed_server.c343
12 files changed, 386 insertions, 54 deletions
diff --git a/Makefile b/Makefile
index 890fc7b..2e6d54d 100644
--- a/Makefile
+++ b/Makefile
@@ -16,9 +16,8 @@ endif
UNAME_S := $(shell uname -s)
# Source files
-SRCS = pplx_embed.c \
- pplx_distributed.c \
- pplx_server.c \
+SRCS = embed.c \
+ embed_distributed.c \
qwen_kernels.c \
qwen_kernels_generic.c \
qwen_kernels_neon.c \
@@ -29,12 +28,12 @@ SRCS = pplx_embed.c \
deps/ae/anet.c \
deps/ae/monotonic.c
-MLX_SRCS = pplx_embed_mlx.c
-CUDA_SRCS = pplx_embed_cuda.cu
+MLX_SRCS = embed_mlx.c
+CUDA_SRCS = embed_cuda.cu
OBJS = $(SRCS:.c=.o)
MLX_OBJS = $(MLX_SRCS:.c=.o)
-TARGET = pplx_embed
+TARGET = pplx-embed
SERVER_TARGET = pplx-embed-server
LIB = libpplxembed.a
@@ -54,7 +53,7 @@ endif
all: help
help:
- @echo "pplx_embed - Inference for pplx-embed-v1 and pplx-embed-context-v1 embedding models"
+ @echo "pplx-embed - Inference for pplx-embed-v1 and pplx-embed-context-v1 embedding models"
@echo ""
@echo "Build targets:"
@echo " make blas Build with BLAS acceleration (CPU)"
@@ -69,7 +68,7 @@ help:
@echo "(HTTP API), and the static library $(LIB)."
@echo ""
@echo "Usage:"
- @echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\""
+ @echo " ./pplx-embed -d /path/to/model-dir \"text1\" \"text2\""
@echo " ./pplx-embed-server --model pplx-embed-v1-0.6b=/path/to/model-dir"
# =============================================================================
@@ -99,8 +98,8 @@ blas:
MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null)
MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null)
-mlx: SRCS += pplx_embed_mlx.c
-mlx: OBJS = $(SRCS:.c=.o) pplx_embed_mlx.o
+mlx: SRCS += embed_mlx.c
+mlx: OBJS = $(SRCS:.c=.o) embed_mlx.o
ifeq ($(UNAME_S),Darwin)
mlx: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX \
-I$(MLXC_PREFIX)/include
@@ -111,7 +110,7 @@ mlx: LDFLAGS += -framework Accelerate -framework Metal -framework Foundation \
endif
mlx:
$(MAKE) clean
- $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o"
+ $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="embed_mlx.o"
# =============================================================================
# CUDA build (Linux NVIDIA GPU - uses CUDA Runtime + cuBLAS)
@@ -123,10 +122,10 @@ cuda: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/inclu
cuda: LDFLAGS += -L$(CUDA_HOME)/lib64 -lopenblas -lcudart -lcublas
cuda:
$(MAKE) clean
- $(MAKE) pplx_embed_cuda.o
- $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o"
+ $(MAKE) embed_cuda.o
+ $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="embed_cuda.o"
-pplx_embed_cuda.o: pplx_embed_cuda.cu pplx_embed_cuda.h pplx_embed_internal.h pplx_embed.h
+embed_cuda.o: embed_cuda.cu embed_cuda.h embed_internal.h embed.h
$(NVCC) -O3 -std=c++17 -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $<
# =============================================================================
@@ -183,32 +182,30 @@ debug:
$(LIB): $(OBJS) $(EXTRA_OBJS)
ar rcs $@ $^
-$(TARGET): $(LIB) main.o
- $(CC) $(CFLAGS) -o $@ main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS)
+$(TARGET): $(LIB) embed_cli.o
+ $(CC) $(CFLAGS) -o $@ embed_cli.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS)
-$(SERVER_TARGET): $(LIB) server_main.o
- $(CC) $(CFLAGS) -o $@ server_main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS)
+$(SERVER_TARGET): $(LIB) embed_server.o
+ $(CC) $(CFLAGS) -o $@ embed_server.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS)
# =============================================================================
# Compile rules
# =============================================================================
-pplx_embed.o: pplx_embed.c pplx_embed.h qwen_kernels.h qwen_safetensors.h
+embed.o: embed.c embed.h qwen_kernels.h qwen_safetensors.h
$(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
-pplx_distributed.o: pplx_distributed.c pplx_distributed.h pplx_embed.h pplx_embed_mlx.h
+embed_distributed.o: embed_distributed.c embed_distributed.h embed.h embed_mlx.h
$(CC) $(CFLAGS) -c -o $@ $<
-pplx_server.o: pplx_server.c pplx_server.h pplx_embed.h pplx_embed_mlx.h qwen_tokenizer.h deps/ae/ae.h deps/ae/anet.h
+embed_server.o: embed_server.c embed_server.h embed.h embed_mlx.h qwen_tokenizer.h deps/ae/ae.h deps/ae/anet.h
$(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
-pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_safetensors.h
+embed_mlx.o: embed_mlx.c embed_mlx.h embed.h qwen_safetensors.h
$(CC) $(CFLAGS) -c -o $@ $<
-main.o: main.c pplx_embed.h pplx_distributed.h qwen_kernels.h qwen_tokenizer.h
+embed_cli.o: embed_cli.c embed.h embed_distributed.h qwen_kernels.h qwen_tokenizer.h
$(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
-server_main.o: server_main.c pplx_embed.h pplx_server.h qwen_kernels.h
- $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
qwen_kernels.o: qwen_kernels.c qwen_kernels.h qwen_kernels_impl.h
$(CC) $(CFLAGS) -c -o $@ $<
@@ -233,6 +230,6 @@ deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h
# =============================================================================
clean:
- rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o server_main.o \
+ rm -f $(OBJS) embed_mlx.o embed_cuda.o embed_cli.o embed_server.o \
$(TARGET) $(SERVER_TARGET) $(LIB) libpplxembed.dylib libpplxembed.so \
tests/test_kernels_generic tests/test_kernels_blas
diff --git a/embed.c b/embed.c
index 996be9c..59ccc10 100644
--- a/embed.c
+++ b/embed.c
@@ -1,5 +1,5 @@
-#include "pplx_embed.h"
-#include "pplx_embed_internal.h"
+#include "embed.h"
+#include "embed_internal.h"
#include "qwen_kernels.h"
#include "qwen_safetensors.h"
diff --git a/embed.h b/embed.h
index 5a2501b..9b905e5 100644
--- a/embed.h
+++ b/embed.h
@@ -1,5 +1,5 @@
/*
- * pplx_embed.h - Pure C inference for Perplexity AI's pplx-embed models.
+ * embed.h - Pure C inference for Perplexity AI's pplx-embed models.
*
* Supports all pplx-embed-v1 and pplx-embed-context-v1 variants (0.6B, 4B).
*/
diff --git a/embed_cli.c b/embed_cli.c
index 140494b..b9dc311 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -1,15 +1,15 @@
-/* main.c - pplx_embed command-line tool */
+/* embed_cli.c - pplx-embed command-line tool */
-#include "pplx_embed.h"
-#include "pplx_distributed.h"
+#include "embed.h"
+#include "embed_distributed.h"
#include "qwen_kernels.h"
#include "qwen_tokenizer.h"
#ifdef USE_MLX
-#include "pplx_embed_mlx.h"
+#include "embed_mlx.h"
#endif
#ifdef USE_CUDA
-#include "pplx_embed_cuda.h"
+#include "embed_cuda.h"
#endif
#include <stdio.h>
diff --git a/embed_cuda.cu b/embed_cuda.cu
index a00fc84..c1406b4 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -1,6 +1,6 @@
extern "C" {
-#include "pplx_embed_cuda.h"
-#include "pplx_embed_internal.h"
+#include "embed_cuda.h"
+#include "embed_internal.h"
}
#include <cuda_runtime.h>
diff --git a/embed_cuda.h b/embed_cuda.h
index badff9c..8ebe6e2 100644
--- a/embed_cuda.h
+++ b/embed_cuda.h
@@ -1,11 +1,11 @@
/*
- * pplx_embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference.
+ * embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference.
*/
#ifndef PPLX_EMBED_CUDA_H
#define PPLX_EMBED_CUDA_H
-#include "pplx_embed.h"
+#include "embed.h"
typedef struct pplx_cuda_ctx pplx_cuda_ctx_t;
diff --git a/embed_distributed.c b/embed_distributed.c
index d4254f7..92d22c1 100644
--- a/embed_distributed.c
+++ b/embed_distributed.c
@@ -1,5 +1,5 @@
/*
- * pplx_distributed.c - Stateless two-node layer-sharded inference.
+ * embed_distributed.c - Stateless two-node layer-sharded inference.
*
* This first transport is deliberately synchronous: one coordinator sends a
* packed hidden-state batch to one final worker and receives pooled
@@ -7,10 +7,10 @@
* retries, and richer transport can extend it without changing model math.
*/
-#include "pplx_distributed.h"
+#include "embed_distributed.h"
#ifdef USE_MLX
-#include "pplx_embed_mlx.h"
+#include "embed_mlx.h"
#endif
#include <arpa/inet.h>
diff --git a/embed_distributed.h b/embed_distributed.h
index cd97147..4cd32e4 100644
--- a/embed_distributed.h
+++ b/embed_distributed.h
@@ -1,11 +1,11 @@
/*
- * pplx_distributed.h - Stateless two-node layer-sharded inference.
+ * embed_distributed.h - Stateless two-node layer-sharded inference.
*/
#ifndef PPLX_DISTRIBUTED_H
#define PPLX_DISTRIBUTED_H
-#include "pplx_embed.h"
+#include "embed.h"
typedef struct pplx_dist_client pplx_dist_client_t;
diff --git a/embed_internal.h b/embed_internal.h
index 1e7e4d0..7d6f398 100644
--- a/embed_internal.h
+++ b/embed_internal.h
@@ -1,7 +1,7 @@
#ifndef PPLX_EMBED_INTERNAL_H
#define PPLX_EMBED_INTERNAL_H
-#include "pplx_embed.h"
+#include "embed.h"
#include "qwen_safetensors.h"
typedef struct {
diff --git a/embed_mlx.c b/embed_mlx.c
index 602571d..4ef283c 100644
--- a/embed_mlx.c
+++ b/embed_mlx.c
@@ -1,5 +1,5 @@
-#include "pplx_embed_mlx.h"
-#include "pplx_embed.h"
+#include "embed_mlx.h"
+#include "embed.h"
#include "qwen_safetensors.h"
#include <mlx/c/mlx.h>
@@ -333,7 +333,7 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s)
* ======================================================================== */
/* We duplicate a minimal config parser here so MLX can load standalone.
- * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */
+ * (Same logic as embed.c:parse_config, kept inline for simplicity.) */
static const char *skip_ws_m(const char *p)
{
diff --git a/embed_mlx.h b/embed_mlx.h
index f2ad136..25376e9 100644
--- a/embed_mlx.h
+++ b/embed_mlx.h
@@ -1,5 +1,5 @@
/*
- * pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference
+ * embed_mlx.h - MLX GPU backend for pplx-embed inference
*
* Uses Apple's mlx-c to run the transformer on Metal.
*/
@@ -9,7 +9,7 @@
#include <stddef.h>
#include <stdint.h>
-#include "pplx_embed.h"
+#include "embed.h"
/* Opaque MLX model context */
typedef struct pplx_mlx_ctx pplx_mlx_ctx_t;
@@ -143,7 +143,7 @@ int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx,
const pplx_context_input_t *inputs, int batch,
float *out_embeddings);
-/* Get the config (so main.c can read hidden_size etc.) */
+/* Get the config (so embed_cli.c can read hidden_size etc.) */
const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx);
#endif /* PPLX_EMBED_MLX_H */
diff --git a/embed_server.c b/embed_server.c
index d831062..f74bdad 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -2,16 +2,17 @@
#define _GNU_SOURCE
#endif
-#include "pplx_server.h"
-#include "pplx_embed.h"
+#include "embed_server.h"
+#include "embed.h"
#include "qwen_safetensors.h"
+#include "qwen_kernels.h"
#include "qwen_tokenizer.h"
#ifdef USE_MLX
-#include "pplx_embed_mlx.h"
+#include "embed_mlx.h"
#endif
#ifdef USE_CUDA
-#include "pplx_embed_cuda.h"
+#include "embed_cuda.h"
#endif
#include "deps/ae/ae.h"
@@ -2429,3 +2430,337 @@ int pplx_run_server(const pplx_server_config_t *cfg) {
pthread_mutex_destroy(&s.mu);
return 0;
}
+
+/* ====================================================================
+ * pplx-embed-server entry point
+ * ==================================================================== */
+#ifndef EMBED_SERVER_TEST
+
+static void print_usage(const char *prog)
+{
+ fprintf(stderr,
+ "Usage: %s --model ID=DIR [options]\n"
+ "\n"
+ "Serves POST /v1/embeddings and /v1/contextualizedembeddings.\n"
+ "\n"
+ "Options:\n"
+ " --model ID=DIR Model to serve (repeatable)\n"
+ " --backend cpu|mlx|cuda Execution backend (default: cpu); --mlx and\n"
+ " --cuda are shorthands\n"
+ " --host HOST Bind host (default: 127.0.0.1)\n"
+ " --port N Bind port (default: 8000)\n"
+ " --api-key K Require Authorization: Bearer K\n"
+ " --cors Enable CORS headers\n"
+ " -b, --batch-size N Max texts or documents per inference batch\n"
+ " (default: 8)\n"
+ " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --batch-wait-us N Micro-batch wait in microseconds\n"
+ " (default: 1000; 0 drains only queued work)\n"
+ " --memory-utilization F Fraction of physical memory the MLX model-set\n"
+ " preflight may plan for (default: 0.90; values\n"
+ " above 1.0 overcommit)\n"
+ " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
+ " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
+ " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
+ " (default: f32, exact)\n"
+ " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
+ " f32); bf16 halves weight memory\n"
+ " -t, --threads N CPU threads (default: all cores)\n"
+ " -v, --verbose Verbose (-vv for debug)\n"
+ " -h, --help Show this help\n"
+ "\n"
+ "Examples:\n"
+ " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
+ " %s --backend mlx --mlx-quant-bits 8 \\\n"
+ " --model pplx-embed-v1-4b=./model-4b-bf16\n",
+ prog, prog, prog);
+}
+
+typedef struct {
+ pplx_server_model_spec_t *v;
+ int n;
+ int cap;
+} model_specs_t;
+
+static int append_model_spec(model_specs_t *specs, const char *arg)
+{
+ const char *eq = strchr(arg, '=');
+ if (!eq || eq == arg || !eq[1]) {
+ fprintf(stderr, "--model expects MODEL_ID=DIR\n");
+ return -1;
+ }
+ if (specs->n == specs->cap) {
+ int new_cap = specs->cap ? specs->cap * 2 : 4;
+ void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
+ if (!p) return -1;
+ specs->v = p;
+ specs->cap = new_cap;
+ }
+ size_t id_len = (size_t)(eq - arg);
+ char *id = (char *)malloc(id_len + 1);
+ char *path = strdup(eq + 1);
+ if (id) {
+ memcpy(id, arg, id_len);
+ id[id_len] = '\0';
+ }
+ specs->v[specs->n].id = id;
+ specs->v[specs->n].path = path;
+ if (!specs->v[specs->n].id || !specs->v[specs->n].path)
+ return -1;
+ specs->n++;
+ return 0;
+}
+
+static void free_model_specs(model_specs_t *specs)
+{
+ if (!specs) return;
+ for (int i = 0; i < specs->n; i++) {
+ free((char *)specs->v[i].id);
+ free((char *)specs->v[i].path);
+ }
+ free(specs->v);
+}
+
+int main(int argc, char *argv[])
+{
+ int n_threads = 0;
+ int verbose = 0;
+ int use_mlx = 0;
+ int use_cuda = 0;
+ int batch_size = 0;
+ int max_batch_tokens = 0;
+ int batch_wait_us = -1;
+ int mlx_quantize_bits = 0;
+ int mlx_quantize_group_size = 64;
+ const char *cuda_fast_gemm = NULL;
+ const char *cuda_weights = NULL;
+ double memory_utilization = 0.0;
+ const char *host = "127.0.0.1";
+ const char *api_key = NULL;
+ int port = 8000;
+ int cors = 0;
+ model_specs_t model_specs = {0};
+
+ int arg = 1;
+ while (arg < argc && argv[arg][0] == '-') {
+ const char *f = argv[arg];
+ if (!strcmp(f, "--model")) {
+ if (append_model_spec(&model_specs, argv[++arg]) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--backend")) {
+ const char *backend = argv[++arg];
+ if (!strcmp(backend, "mlx")) {
+#ifdef USE_MLX
+ use_mlx = 1;
+ use_cuda = 0;
+#else
+ fprintf(stderr, "mlx backend not available (build with: make mlx)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ } else if (!strcmp(backend, "cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "cuda backend not available (build with: make cuda)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ } else if (!strcmp(backend, "cpu")) {
+ use_mlx = 0;
+ use_cuda = 0;
+ } else {
+ fprintf(stderr, "invalid --backend: %s\n", backend);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx")) {
+#ifdef USE_MLX
+ use_mlx = 1;
+ use_cuda = 0;
+#else
+ fprintf(stderr, "--mlx not available (build with: make mlx)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+ else if (!strcmp(f, "--cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "--cuda not available (build with: make cuda)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+ else if (!strcmp(f, "--host")) { host = argv[++arg]; }
+ else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); }
+ else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; }
+ else if (!strcmp(f, "--cors")) { cors = 1; }
+ else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ batch_size = atoi(argv[++arg]);
+ }
+ else if (!strcmp(f, "--max-batch-tokens")) {
+ max_batch_tokens = atoi(argv[++arg]);
+ if (max_batch_tokens <= 0) {
+ fprintf(stderr, "--max-batch-tokens must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--batch-wait-us")) {
+ batch_wait_us = atoi(argv[++arg]);
+ if (batch_wait_us < 0) {
+ fprintf(stderr, "--batch-wait-us must be >= 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--memory-utilization")) {
+ memory_utilization = atof(argv[++arg]);
+ if (memory_utilization <= 0.0) {
+ fprintf(stderr, "--memory-utilization must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx-quant-bits")) {
+ mlx_quantize_bits = atoi(argv[++arg]);
+ if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
+ fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx-quant-group-size")) {
+ mlx_quantize_group_size = atoi(argv[++arg]);
+ if (mlx_quantize_group_size <= 0) {
+ fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--cuda-gemm-mode")) {
+ cuda_fast_gemm = argv[++arg];
+ }
+ else if (!strcmp(f, "--cuda-weight-dtype")) {
+ cuda_weights = argv[++arg];
+ }
+ else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ n_threads = atoi(argv[++arg]);
+ }
+ else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
+ else if (!strcmp(f, "-vv")) { verbose = 2; }
+ else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 0;
+ }
+ else {
+ fprintf(stderr, "unknown option: %s\n", f);
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ arg++;
+ }
+
+ pplx_verbose = verbose;
+ qwen_verbose = verbose;
+
+ if (arg < argc) {
+ fprintf(stderr, "unexpected argument: %s\n", argv[arg]);
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (model_specs.n == 0) {
+ fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n");
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (mlx_quantize_bits && !use_mlx) {
+ fprintf(stderr, "--mlx-quant-bits requires --backend mlx\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+
+ if (cuda_fast_gemm) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-gemm-mode requires --backend cuda\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
+ fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+
+ if (cuda_weights) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-weight-dtype requires --backend cuda\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (!strcmp(cuda_weights, "bf16")) {
+ pplx_cuda_set_weights_bf16(1);
+ } else if (!strcmp(cuda_weights, "f32")) {
+ pplx_cuda_set_weights_bf16(0);
+ } else {
+ fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+
+ if (!use_mlx && !use_cuda) {
+ if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ qwen_set_threads(n_threads);
+ if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ } else if (verbose >= 1) {
+ fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
+ }
+
+ pplx_server_config_t scfg = {
+ .models = model_specs.v,
+ .n_models = model_specs.n,
+ .host = host,
+ .port = port,
+ .batch_size = batch_size > 0 ? batch_size : 8,
+ .max_batch_tokens = max_batch_tokens,
+ .batch_wait_us = batch_wait_us,
+ .use_mlx = use_mlx,
+ .use_cuda = use_cuda,
+ .mlx_quantize_bits = mlx_quantize_bits,
+ .mlx_quantize_group_size = mlx_quantize_group_size,
+ .memory_utilization = memory_utilization,
+ .enable_cors = cors,
+ .api_key = api_key,
+ };
+ int rc = pplx_run_server(&scfg);
+ free_model_specs(&model_specs);
+ return rc;
+}
+
+#endif /* EMBED_SERVER_TEST */