aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile55
-rw-r--r--main.c245
-rw-r--r--pplx_embed_cuda.cu4
-rw-r--r--pplx_server.c23
-rw-r--r--pplx_server.h4
-rw-r--r--server_main.c340
6 files changed, 441 insertions, 230 deletions
diff --git a/Makefile b/Makefile
index c31f9e1..2451926 100644
--- a/Makefile
+++ b/Makefile
@@ -34,8 +34,12 @@ CUDA_SRCS = pplx_embed_cuda.cu
OBJS = $(SRCS:.c=.o)
MLX_OBJS = $(MLX_SRCS:.c=.o)
-TARGET = pplx_embed
-LIB = libpplxembed.a
+TARGET = pplx_embed
+SERVER_TARGET = pplx-embed-server
+LIB = libpplxembed.a
+
+KERNEL_SRCS = qwen_asr_kernels.c qwen_asr_kernels_generic.c \
+ qwen_asr_kernels_neon.c qwen_asr_kernels_avx.c
ifeq ($(shell uname -s),Darwin)
SHARED_LIB = libpplxembed.dylib
@@ -45,7 +49,7 @@ SHARED_LIB = libpplxembed.so
SHARED_FLAGS = -shared
endif
-.PHONY: all blas mlx cuda shared debug clean help
+.PHONY: all blas mlx cuda shared test debug clean help
all: help
@@ -57,14 +61,16 @@ help:
@echo " make mlx Build with Apple MLX GPU backend (recommended on Apple Silicon)"
@echo " make cuda Build with CUDA/cuBLAS backend (Linux/NVIDIA)"
@echo " make shared Build the shared library ($(SHARED_LIB), BLAS backend)"
+ @echo " make test Build and run the C test suite (no model files needed)"
@echo " make debug Debug build with AddressSanitizer"
@echo " make clean Remove build artifacts"
@echo ""
- @echo "Every backend target also produces the static library $(LIB)."
+ @echo "Each backend target builds ./$(TARGET) (CLI), ./$(SERVER_TARGET)"
+ @echo "(HTTP API), and the static library $(LIB)."
@echo ""
@echo "Usage:"
@echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\""
- @echo " ./pplx_embed -d /path/to/model-dir --mlx \"text1\" \"text2\" (if built with make mlx)"
+ @echo " ./pplx-embed-server --model pplx-embed-v1-0.6b=/path/to/model-dir"
# =============================================================================
# BLAS build (Apple Accelerate on macOS, OpenBLAS on Linux)
@@ -84,7 +90,7 @@ blas: LDFLAGS += -lopenblas
endif
blas:
$(MAKE) clean
- $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
+ $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)"
# =============================================================================
# MLX build (Apple Silicon GPU - uses mlx-c pure C API)
@@ -105,7 +111,7 @@ mlx: LDFLAGS += -framework Accelerate -framework Metal -framework Foundation \
endif
mlx:
$(MAKE) clean
- $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o"
+ $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o"
# =============================================================================
# CUDA build (Linux NVIDIA GPU - uses CUDA Runtime + cuBLAS)
@@ -118,7 +124,7 @@ cuda: LDFLAGS += -L$(CUDA_HOME)/lib64 -lopenblas -lcudart -lcublas
cuda:
$(MAKE) clean
$(MAKE) pplx_embed_cuda.o
- $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o"
+ $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o"
pplx_embed_cuda.o: pplx_embed_cuda.cu pplx_embed_cuda.h pplx_embed_internal.h pplx_embed.h
$(NVCC) -O3 -std=c++17 -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $<
@@ -140,6 +146,26 @@ shared:
$(SHARED_LIB): $(OBJS) $(EXTRA_OBJS)
$(CC) $(CFLAGS) $(SHARED_FLAGS) -o $@ $^ $(LDFLAGS) $(CJSON_LDFLAGS)
+
+# =============================================================================
+# Tests (no model files required)
+# =============================================================================
+ifeq ($(UNAME_S),Darwin)
+TEST_BLAS_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK
+TEST_BLAS_LDFLAGS = -framework Accelerate
+else
+TEST_BLAS_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -I/usr/include/openblas
+TEST_BLAS_LDFLAGS = -lopenblas
+endif
+
+test:
+ $(CC) -Wall -Wextra -O2 -I. -o tests/test_kernels_generic \
+ tests/test_kernels.c $(KERNEL_SRCS) -lm -lpthread
+ ./tests/test_kernels_generic
+ $(CC) -Wall -Wextra -O2 $(TEST_BLAS_CFLAGS) -I. -o tests/test_kernels_blas \
+ tests/test_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ ./tests/test_kernels_blas
+
# =============================================================================
# Debug build
# =============================================================================
@@ -160,6 +186,9 @@ $(LIB): $(OBJS) $(EXTRA_OBJS)
$(TARGET): $(LIB) main.o
$(CC) $(CFLAGS) -o $@ main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS)
+$(SERVER_TARGET): $(LIB) server_main.o
+ $(CC) $(CFLAGS) -o $@ server_main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS)
+
# =============================================================================
# Compile rules
# =============================================================================
@@ -175,7 +204,10 @@ pplx_server.o: pplx_server.c pplx_server.h pplx_embed.h pplx_embed_mlx.h qwen_as
pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_asr_safetensors.h
$(CC) $(CFLAGS) -c -o $@ $<
-main.o: main.c pplx_embed.h pplx_distributed.h pplx_server.h qwen_asr_kernels.h qwen_asr_tokenizer.h
+main.o: main.c pplx_embed.h pplx_distributed.h qwen_asr_kernels.h qwen_asr_tokenizer.h
+ $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
+
+server_main.o: server_main.c pplx_embed.h pplx_server.h qwen_asr_kernels.h
$(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $<
qwen_asr_kernels.o: qwen_asr_kernels.c qwen_asr_kernels.h qwen_asr_kernels_impl.h
@@ -201,5 +233,6 @@ deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h
# =============================================================================
clean:
- rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o $(TARGET) \
- $(LIB) libpplxembed.dylib libpplxembed.so
+ rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o server_main.o \
+ $(TARGET) $(SERVER_TARGET) $(LIB) libpplxembed.dylib libpplxembed.so \
+ tests/test_kernels_generic tests/test_kernels_blas
diff --git a/main.c b/main.c
index 566e6fe..ef1c91a 100644
--- a/main.c
+++ b/main.c
@@ -2,7 +2,6 @@
#include "pplx_embed.h"
#include "pplx_distributed.h"
-#include "pplx_server.h"
#include "qwen_asr_kernels.h"
#include "qwen_asr_tokenizer.h"
@@ -35,59 +34,50 @@ static void print_usage(const char *prog)
" -d <dir> Model directory (required)\n"
" --mlx Use Apple MLX GPU backend\n"
" --cuda Use CUDA/cuBLAS GPU backend\n"
- " --stdin Read lines from stdin, write JSON embeddings to stdout\n"
- " --serve Serve the Perplexity-compatible HTTP API\n"
+ " --stream Read lines from stdin, write one JSON embedding per line\n"
" --dist-worker\n"
" Serve one final distributed layer shard\n"
" --dist-remote HOST:PORT\n"
- " Use one remote final shard for CLI or --stdin inference\n"
+ " Use one remote final shard for CLI or --stream inference\n"
" --layers A:B Distributed half-open layer range, for example 0:14\n"
" --dist-activation-bits N\n"
" Distributed activation transport width: 32 or 16 (default: 32)\n"
- " --model ID=DIR\n"
- " Load model ID from directory for --serve (repeatable)\n"
- " --backend cpu|mlx|cuda\n"
- " Backend for --serve (default: cpu)\n"
- " --mlx-quantize-bits N\n"
+ " --mlx-quant-bits N\n"
" Quantize MLX linear weights to 8 bits at load time\n"
- " --mlx-quantize-group-size N\n"
+ " --mlx-quant-group-size N\n"
" MLX quantization group size (default: 64)\n"
- " --cuda-fast-gemm MODE\n"
+ " --cuda-gemm-mode MODE\n"
" CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
- " --cuda-weights DTYPE\n"
+ " --cuda-weight-dtype DTYPE\n"
" CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
" weight memory and uses BF16 tensor cores\n"
- " --allow-memory-overcommit\n"
- " Allow an MLX server model set above the host-memory safety budget\n"
- " --host HOST Server bind host (default: 127.0.0.1)\n"
- " --port N Server bind port (default: 8000)\n"
- " --cors Enable CORS headers in --serve mode\n"
- " --api-key K Require Authorization: Bearer K in --serve mode\n"
- " -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n"
- " --max-batch-tokens N\n"
- " Server max tokens per standard inference batch (default: 16384)\n"
- " --batch-wait-us N\n"
- " Server micro-batch wait in microseconds (default: 1000; 0 disables)\n"
- " -t <n> CPU threads (default: all cores)\n"
- " -e Print raw embeddings (with multiple texts)\n"
- " -v Verbose (-vv for debug)\n"
+ " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n"
+ " --port N Bind port for --dist-worker (default: 8000)\n"
+ " -b, --batch-size N\n"
+ " Max texts per engine batch (default: all; --stream: 1)\n"
+ " -t, --threads N\n"
+ " CPU threads (default: all cores)\n"
+ " -e, --embeddings\n"
+ " Print raw embeddings (with multiple texts)\n"
+ " -v, --verbose\n"
+ " Verbose (-vv for debug)\n"
" -h Show this help\n"
"\n"
"Modes:\n"
" 1 text arg Print embedding as space-separated floats\n"
" 2+ text args Print cosine similarity matrix\n"
" no args Batch: read all stdin lines, then similarity matrix\n"
- " --stdin Streaming: read stdin lines, write JSON per line\n"
- " --serve HTTP server for /v1/embeddings and /v1/contextualizedembeddings\n"
+ " --stream Streaming: read stdin lines, write JSON per line\n"
" --dist-worker Final layer-shard worker\n"
"\n"
+ "The HTTP API is served by the separate pplx-embed-server binary.\n"
+ "\n"
"Examples:\n"
" %s -d ./model \"query: what is AI?\"\n"
- " %s -d ./model --mlx --stdin < texts.txt\n"
- " %s --serve --model pplx-embed-v1-0.6b=./model --port 8000\n"
+ " %s -d ./model --mlx --stream < texts.txt\n"
" %s -d ./model --dist-worker --layers 14:28 --port 9000\n"
" %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n",
- prog, prog, prog, prog, prog, prog);
+ prog, prog, prog, prog, prog);
}
/* ========================================================================
@@ -114,12 +104,6 @@ typedef struct {
int n_tokens;
} token_buf_t;
-typedef struct {
- pplx_server_model_spec_t *v;
- int n;
- int cap;
-} model_specs_t;
-
static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs,
int batch, float *out_embeddings)
{
@@ -192,45 +176,6 @@ static void free_tokens(token_buf_t *tokens, int n)
free(tokens[i].ids);
}
-static int append_model_spec(model_specs_t *specs, const char *arg)
-{
- const char *eq = strchr(arg, '=');
- if (!eq || eq == arg || !eq[1]) {
- fprintf(stderr, "--model expects MODEL_ID=DIR\n");
- return -1;
- }
- if (specs->n == specs->cap) {
- int new_cap = specs->cap ? specs->cap * 2 : 4;
- void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
- if (!p) return -1;
- specs->v = p;
- specs->cap = new_cap;
- }
- size_t id_len = (size_t)(eq - arg);
- char *id = (char *)malloc(id_len + 1);
- char *path = strdup(eq + 1);
- if (id) {
- memcpy(id, arg, id_len);
- id[id_len] = '\0';
- }
- specs->v[specs->n].id = id;
- specs->v[specs->n].path = path;
- if (!specs->v[specs->n].id || !specs->v[specs->n].path)
- return -1;
- specs->n++;
- return 0;
-}
-
-static void free_model_specs(model_specs_t *specs)
-{
- if (!specs) return;
- for (int i = 0; i < specs->n; i++) {
- free((char *)specs->v[i].id);
- free((char *)specs->v[i].path);
- }
- free(specs->v);
-}
-
/* ========================================================================
* Output helpers
* ======================================================================== */
@@ -531,56 +476,29 @@ int main(int argc, char *argv[])
int use_mlx = 0;
int use_cuda = 0;
int stdin_mode = 0;
- int serve_mode = 0;
int dist_worker_mode = 0;
- int allow_memory_overcommit = 0;
const char *dist_remote = NULL;
int layer_start = -1;
int layer_end = -1;
int batch_size = 0;
- int max_batch_tokens = 0;
- int batch_wait_us = -1;
int mlx_quantize_bits = 0;
int mlx_quantize_group_size = 64;
const char *cuda_fast_gemm = NULL;
const char *cuda_weights = NULL;
int dist_activation_bits = 32;
const char *host = "127.0.0.1";
- const char *api_key = NULL;
int port = 8000;
- int cors = 0;
- model_specs_t model_specs = {0};
int arg_start = 1;
while (arg_start < argc && argv[arg_start][0] == '-') {
const char *f = argv[arg_start];
if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; }
- else if (!strcmp(f, "-t")) { n_threads = atoi(argv[++arg_start]); }
+ else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ n_threads = atoi(argv[++arg_start]);
+ }
else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg_start]);
}
- else if (!strcmp(f, "--batch-wait-us")) {
- batch_wait_us = atoi(argv[++arg_start]);
- if (batch_wait_us < 0) {
- fprintf(stderr, "--batch-wait-us must be >= 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--max-batch-tokens")) {
- max_batch_tokens = atoi(argv[++arg_start]);
- if (max_batch_tokens <= 0) {
- fprintf(stderr, "--max-batch-tokens must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--model")) {
- if (append_model_spec(&model_specs, argv[++arg_start]) != 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- }
else if (!strcmp(f, "--backend")) {
const char *backend = argv[++arg_start];
if (!strcmp(backend, "mlx")) {
@@ -589,7 +507,6 @@ int main(int argc, char *argv[])
use_cuda = 0;
#else
fprintf(stderr, "mlx backend not available (build with: make mlx)\n");
- free_model_specs(&model_specs);
return 1;
#endif
} else if (!strcmp(backend, "cuda")) {
@@ -598,7 +515,6 @@ int main(int argc, char *argv[])
use_mlx = 0;
#else
fprintf(stderr, "cuda backend not available (build with: make cuda)\n");
- free_model_specs(&model_specs);
return 1;
#endif
} else if (!strcmp(backend, "cpu")) {
@@ -606,41 +522,34 @@ int main(int argc, char *argv[])
use_cuda = 0;
} else {
fprintf(stderr, "invalid --backend: %s\n", backend);
- free_model_specs(&model_specs);
return 1;
}
}
- else if (!strcmp(f, "--mlx-quantize-bits")) {
+ else if (!strcmp(f, "--mlx-quant-bits")) {
mlx_quantize_bits = atoi(argv[++arg_start]);
if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quantize-bits must be 0 or 8\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
return 1;
}
}
- else if (!strcmp(f, "--mlx-quantize-group-size")) {
+ else if (!strcmp(f, "--mlx-quant-group-size")) {
mlx_quantize_group_size = atoi(argv[++arg_start]);
if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quantize-group-size must be > 0\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
return 1;
}
}
- else if (!strcmp(f, "--cuda-fast-gemm")) {
+ else if (!strcmp(f, "--cuda-gemm-mode")) {
cuda_fast_gemm = argv[++arg_start];
}
- else if (!strcmp(f, "--cuda-weights")) {
+ else if (!strcmp(f, "--cuda-weight-dtype")) {
cuda_weights = argv[++arg_start];
}
else if (!strcmp(f, "--host")) { host = argv[++arg_start]; }
else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); }
- else if (!strcmp(f, "--allow-memory-overcommit")) {
- allow_memory_overcommit = 1;
- }
else if (!strcmp(f, "--layers")) {
if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) {
fprintf(stderr, "--layers expects START:END with START < END\n");
- free_model_specs(&model_specs);
return 1;
}
}
@@ -648,17 +557,15 @@ int main(int argc, char *argv[])
dist_activation_bits = atoi(argv[++arg_start]);
if (dist_activation_bits != 32 && dist_activation_bits != 16) {
fprintf(stderr, "--dist-activation-bits must be 32 or 16\n");
- free_model_specs(&model_specs);
return 1;
}
}
- else if (!strcmp(f, "--cors")) { cors = 1; }
- else if (!strcmp(f, "--api-key")){ api_key = argv[++arg_start]; }
- else if (!strcmp(f, "-e")) { print_embs = 1; }
- else if (!strcmp(f, "-v")) { verbose++; }
+ else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
+ print_embs = 1;
+ }
+ else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
else if (!strcmp(f, "-vv")) { verbose = 2; }
- else if (!strcmp(f, "--stdin")) { stdin_mode = 1; }
- else if (!strcmp(f, "--serve")) { serve_mode = 1; }
+ else if (!strcmp(f, "--stream")) { stdin_mode = 1; }
else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; }
else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; }
else if (!strcmp(f, "--mlx")) {
@@ -667,7 +574,6 @@ int main(int argc, char *argv[])
use_cuda = 0;
#else
fprintf(stderr, "--mlx not available (build with: make mlx)\n");
- free_model_specs(&model_specs);
return 1;
#endif
}
@@ -677,13 +583,11 @@ int main(int argc, char *argv[])
use_mlx = 0;
#else
fprintf(stderr, "--cuda not available (build with: make cuda)\n");
- free_model_specs(&model_specs);
return 1;
#endif
}
else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
print_usage(argv[0]);
- free_model_specs(&model_specs);
return 0;
}
else break;
@@ -696,18 +600,15 @@ int main(int argc, char *argv[])
if (cuda_fast_gemm) {
#ifdef USE_CUDA
if (!use_cuda) {
- fprintf(stderr, "--cuda-fast-gemm requires --cuda or --backend cuda\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--cuda-gemm-mode requires --cuda or --backend cuda\n");
return 1;
}
if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
- fprintf(stderr, "--cuda-fast-gemm must be f32, tf32, bf16, or 16f\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
return 1;
}
#else
- fprintf(stderr, "--cuda-fast-gemm requires a CUDA build\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
return 1;
#endif
}
@@ -715,8 +616,7 @@ int main(int argc, char *argv[])
if (cuda_weights) {
#ifdef USE_CUDA
if (!use_cuda) {
- fprintf(stderr, "--cuda-weights requires --cuda or --backend cuda\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--cuda-weight-dtype requires --cuda or --backend cuda\n");
return 1;
}
if (!strcmp(cuda_weights, "bf16")) {
@@ -724,77 +624,18 @@ int main(int argc, char *argv[])
} else if (!strcmp(cuda_weights, "f32")) {
pplx_cuda_set_weights_bf16(0);
} else {
- fprintf(stderr, "--cuda-weights must be f32 or bf16\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
return 1;
}
#else
- fprintf(stderr, "--cuda-weights requires a CUDA build\n");
- free_model_specs(&model_specs);
+ fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
return 1;
#endif
}
- if (serve_mode) {
- if (stdin_mode || dist_worker_mode || dist_remote) {
- fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (model_specs.n == 0) {
- fprintf(stderr, "--serve requires at least one --model MODEL_ID=DIR\n");
- print_usage(argv[0]);
- free_model_specs(&model_specs);
- return 1;
- }
- if (dist_activation_bits != 32) {
- fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (!use_mlx && !use_cuda) {
- if (n_threads <= 0) n_threads = qwen_get_num_cpus();
- qwen_set_threads(n_threads);
- if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- } else if (verbose >= 1) {
- fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
- }
- pplx_server_config_t scfg = {
- .models = model_specs.v,
- .n_models = model_specs.n,
- .host = host,
- .port = port,
- .batch_size = batch_size > 0 ? batch_size : 8,
- .max_batch_tokens = max_batch_tokens,
- .batch_wait_us = batch_wait_us,
- .use_mlx = use_mlx,
- .use_cuda = use_cuda,
- .mlx_quantize_bits = mlx_quantize_bits,
- .mlx_quantize_group_size = mlx_quantize_group_size,
- .allow_memory_overcommit = allow_memory_overcommit,
- .enable_cors = cors,
- .api_key = api_key,
- };
- int rc = pplx_run_server(&scfg);
- free_model_specs(&model_specs);
- return rc;
- }
-
- if (model_specs.n != 0) {
- fprintf(stderr, "--model is only valid with --serve\n");
- free_model_specs(&model_specs);
- return 1;
- }
-
if (!model_dir || !model_dir[0]) {
fprintf(stderr, "model directory required (-d <dir>)\n");
print_usage(argv[0]);
- free_model_specs(&model_specs);
return 1;
}
@@ -831,7 +672,7 @@ int main(int argc, char *argv[])
}
if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quantize-bits requires --mlx or --backend mlx\n");
+ fprintf(stderr, "--mlx-quant-bits requires --mlx or --backend mlx\n");
return 1;
}
@@ -958,7 +799,6 @@ int main(int argc, char *argv[])
if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx);
#endif
qwen_tokenizer_free(tok);
- free_model_specs(&model_specs);
return 1;
}
@@ -981,6 +821,5 @@ int main(int argc, char *argv[])
#endif
qwen_tokenizer_workspace_free(e.tok_ws);
qwen_tokenizer_free(tok);
- free_model_specs(&model_specs);
return rc;
}
diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu
index bda1be2..a00fc84 100644
--- a/pplx_embed_cuda.cu
+++ b/pplx_embed_cuda.cu
@@ -537,14 +537,14 @@ static int launch_check(void)
}
// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm).
-// PPLX_CUDA_FAST_GEMM={bf16,tf32,16f} selects a reduced-precision tensor-core
+// PPLX_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core
// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands
// internally.
static cublasComputeType_t gemm_compute(void)
{
static int env_init = 0;
if (!g_gemm_compute_set && !env_init) {
- const char *e = getenv("PPLX_CUDA_FAST_GEMM");
+ const char *e = getenv("PPLX_CUDA_GEMM_MODE");
if (e && parse_gemm_compute(e, &g_gemm_compute) == 0)
g_gemm_compute_set = 1;
env_init = 1;
diff --git a/pplx_server.c b/pplx_server.c
index 6e7bb22..f8ad5e1 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -2266,29 +2266,26 @@ static int mlx_memory_preflight(const pplx_server_config_t *cfg) {
"skipping MLX memory preflight");
return 0;
}
- uint64_t budget =
- physical / 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT +
- physical % 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT / 100;
+ double utilization = cfg->memory_utilization > 0.0
+ ? cfg->memory_utilization
+ : PPLX_MLX_MEMORY_BUDGET_PERCENT / 100.0;
+ uint64_t budget = (uint64_t)((double)physical * utilization);
const double gib = 1024.0 * 1024.0 * 1024.0;
server_log("pplx-serve: MLX memory preflight: %.1f GiB tensors, "
- "%.1f GiB estimated resident, %.1f GiB safety budget",
+ "%.1f GiB estimated resident, %.1f GiB budget "
+ "(%.2f of physical memory)",
(double)payload / gib, (double)estimated / gib,
- (double)budget / gib);
+ (double)budget / gib, utilization);
if (cfg->mlx_quantize_bits) {
server_log("pplx-serve: MLX %d-bit quantization enabled; preflight "
"uses source payload as a conservative peak estimate",
cfg->mlx_quantize_bits);
}
if (estimated <= budget) return 0;
- if (cfg->allow_memory_overcommit) {
- server_log("pplx-serve: warning: MLX memory estimate exceeds safety "
- "budget; continuing due to --allow-memory-overcommit");
- return 0;
- }
- server_log("pplx-serve: refusing MLX model set above host-memory safety "
+ server_log("pplx-serve: refusing MLX model set above the host-memory "
"budget");
- server_log("pplx-serve: use BF16 artifacts, load fewer models, or pass "
- "--allow-memory-overcommit");
+ server_log("pplx-serve: use BF16 artifacts, load fewer models, or raise "
+ "--memory-utilization (values above 1.0 overcommit)");
return -1;
}
diff --git a/pplx_server.h b/pplx_server.h
index 225a7ac..c38f0ca 100644
--- a/pplx_server.h
+++ b/pplx_server.h
@@ -20,7 +20,9 @@ typedef struct {
int use_cuda;
int mlx_quantize_bits;
int mlx_quantize_group_size;
- int allow_memory_overcommit;
+ double memory_utilization; /* fraction of physical memory the MLX
+ preflight may plan for; 0 = default 0.90,
+ values above 1.0 allow overcommit */
int enable_cors;
const char *api_key;
} pplx_server_config_t;
diff --git a/server_main.c b/server_main.c
new file mode 100644
index 0000000..b4c7a34
--- /dev/null
+++ b/server_main.c
@@ -0,0 +1,340 @@
+/* server_main.c - pplx-embed-server: Perplexity-compatible embeddings API */
+
+#include "pplx_embed.h"
+#include "pplx_server.h"
+#include "qwen_asr_kernels.h"
+
+#ifdef USE_CUDA
+#include "pplx_embed_cuda.h"
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static void print_usage(const char *prog)
+{
+ fprintf(stderr,
+ "Usage: %s --model ID=DIR [options]\n"
+ "\n"
+ "Serves POST /v1/embeddings and /v1/contextualizedembeddings.\n"
+ "\n"
+ "Options:\n"
+ " --model ID=DIR Model to serve (repeatable)\n"
+ " --backend cpu|mlx|cuda Execution backend (default: cpu); --mlx and\n"
+ " --cuda are shorthands\n"
+ " --host HOST Bind host (default: 127.0.0.1)\n"
+ " --port N Bind port (default: 8000)\n"
+ " --api-key K Require Authorization: Bearer K\n"
+ " --cors Enable CORS headers\n"
+ " -b, --batch-size N Max texts or documents per inference batch\n"
+ " (default: 8)\n"
+ " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --batch-wait-us N Micro-batch wait in microseconds\n"
+ " (default: 1000; 0 drains only queued work)\n"
+ " --memory-utilization F Fraction of physical memory the MLX model-set\n"
+ " preflight may plan for (default: 0.90; values\n"
+ " above 1.0 overcommit)\n"
+ " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
+ " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
+ " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
+ " (default: f32, exact)\n"
+ " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
+ " f32); bf16 halves weight memory\n"
+ " -t, --threads N CPU threads (default: all cores)\n"
+ " -v, --verbose Verbose (-vv for debug)\n"
+ " -h, --help Show this help\n"
+ "\n"
+ "Examples:\n"
+ " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
+ " %s --backend mlx --mlx-quant-bits 8 \\\n"
+ " --model pplx-embed-v1-4b=./model-4b-bf16\n",
+ prog, prog, prog);
+}
+
+typedef struct {
+ pplx_server_model_spec_t *v;
+ int n;
+ int cap;
+} model_specs_t;
+
+static int append_model_spec(model_specs_t *specs, const char *arg)
+{
+ const char *eq = strchr(arg, '=');
+ if (!eq || eq == arg || !eq[1]) {
+ fprintf(stderr, "--model expects MODEL_ID=DIR\n");
+ return -1;
+ }
+ if (specs->n == specs->cap) {
+ int new_cap = specs->cap ? specs->cap * 2 : 4;
+ void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
+ if (!p) return -1;
+ specs->v = p;
+ specs->cap = new_cap;
+ }
+ size_t id_len = (size_t)(eq - arg);
+ char *id = (char *)malloc(id_len + 1);
+ char *path = strdup(eq + 1);
+ if (id) {
+ memcpy(id, arg, id_len);
+ id[id_len] = '\0';
+ }
+ specs->v[specs->n].id = id;
+ specs->v[specs->n].path = path;
+ if (!specs->v[specs->n].id || !specs->v[specs->n].path)
+ return -1;
+ specs->n++;
+ return 0;
+}
+
+static void free_model_specs(model_specs_t *specs)
+{
+ if (!specs) return;
+ for (int i = 0; i < specs->n; i++) {
+ free((char *)specs->v[i].id);
+ free((char *)specs->v[i].path);
+ }
+ free(specs->v);
+}
+
+int main(int argc, char *argv[])
+{
+ int n_threads = 0;
+ int verbose = 0;
+ int use_mlx = 0;
+ int use_cuda = 0;
+ int batch_size = 0;
+ int max_batch_tokens = 0;
+ int batch_wait_us = -1;
+ int mlx_quantize_bits = 0;
+ int mlx_quantize_group_size = 64;
+ const char *cuda_fast_gemm = NULL;
+ const char *cuda_weights = NULL;
+ double memory_utilization = 0.0;
+ const char *host = "127.0.0.1";
+ const char *api_key = NULL;
+ int port = 8000;
+ int cors = 0;
+ model_specs_t model_specs = {0};
+
+ int arg = 1;
+ while (arg < argc && argv[arg][0] == '-') {
+ const char *f = argv[arg];
+ if (!strcmp(f, "--model")) {
+ if (append_model_spec(&model_specs, argv[++arg]) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--backend")) {
+ const char *backend = argv[++arg];
+ if (!strcmp(backend, "mlx")) {
+#ifdef USE_MLX
+ use_mlx = 1;
+ use_cuda = 0;
+#else
+ fprintf(stderr, "mlx backend not available (build with: make mlx)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ } else if (!strcmp(backend, "cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "cuda backend not available (build with: make cuda)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ } else if (!strcmp(backend, "cpu")) {
+ use_mlx = 0;
+ use_cuda = 0;
+ } else {
+ fprintf(stderr, "invalid --backend: %s\n", backend);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx")) {
+#ifdef USE_MLX
+ use_mlx = 1;
+ use_cuda = 0;
+#else
+ fprintf(stderr, "--mlx not available (build with: make mlx)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+ else if (!strcmp(f, "--cuda")) {
+#ifdef USE_CUDA
+ use_cuda = 1;
+ use_mlx = 0;
+#else
+ fprintf(stderr, "--cuda not available (build with: make cuda)\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+ else if (!strcmp(f, "--host")) { host = argv[++arg]; }
+ else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); }
+ else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; }
+ else if (!strcmp(f, "--cors")) { cors = 1; }
+ else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ batch_size = atoi(argv[++arg]);
+ }
+ else if (!strcmp(f, "--max-batch-tokens")) {
+ max_batch_tokens = atoi(argv[++arg]);
+ if (max_batch_tokens <= 0) {
+ fprintf(stderr, "--max-batch-tokens must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--batch-wait-us")) {
+ batch_wait_us = atoi(argv[++arg]);
+ if (batch_wait_us < 0) {
+ fprintf(stderr, "--batch-wait-us must be >= 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--memory-utilization")) {
+ memory_utilization = atof(argv[++arg]);
+ if (memory_utilization <= 0.0) {
+ fprintf(stderr, "--memory-utilization must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx-quant-bits")) {
+ mlx_quantize_bits = atoi(argv[++arg]);
+ if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
+ fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx-quant-group-size")) {
+ mlx_quantize_group_size = atoi(argv[++arg]);
+ if (mlx_quantize_group_size <= 0) {
+ fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--cuda-gemm-mode")) {
+ cuda_fast_gemm = argv[++arg];
+ }
+ else if (!strcmp(f, "--cuda-weight-dtype")) {
+ cuda_weights = argv[++arg];
+ }
+ else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ n_threads = atoi(argv[++arg]);
+ }
+ else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
+ else if (!strcmp(f, "-vv")) { verbose = 2; }
+ else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 0;
+ }
+ else {
+ fprintf(stderr, "unknown option: %s\n", f);
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ arg++;
+ }
+
+ pplx_verbose = verbose;
+ qwen_verbose = verbose;
+
+ if (arg < argc) {
+ fprintf(stderr, "unexpected argument: %s\n", argv[arg]);
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (model_specs.n == 0) {
+ fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n");
+ print_usage(argv[0]);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (mlx_quantize_bits && !use_mlx) {
+ fprintf(stderr, "--mlx-quant-bits requires --backend mlx\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+
+ if (cuda_fast_gemm) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-gemm-mode requires --backend cuda\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
+ fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+
+ if (cuda_weights) {
+#ifdef USE_CUDA
+ if (!use_cuda) {
+ fprintf(stderr, "--cuda-weight-dtype requires --backend cuda\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (!strcmp(cuda_weights, "bf16")) {
+ pplx_cuda_set_weights_bf16(1);
+ } else if (!strcmp(cuda_weights, "f32")) {
+ pplx_cuda_set_weights_bf16(0);
+ } else {
+ fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+#else
+ fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
+ free_model_specs(&model_specs);
+ return 1;
+#endif
+ }
+
+ if (!use_mlx && !use_cuda) {
+ if (n_threads <= 0) n_threads = qwen_get_num_cpus();
+ qwen_set_threads(n_threads);
+ if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
+ } else if (verbose >= 1) {
+ fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
+ }
+
+ pplx_server_config_t scfg = {
+ .models = model_specs.v,
+ .n_models = model_specs.n,
+ .host = host,
+ .port = port,
+ .batch_size = batch_size > 0 ? batch_size : 8,
+ .max_batch_tokens = max_batch_tokens,
+ .batch_wait_us = batch_wait_us,
+ .use_mlx = use_mlx,
+ .use_cuda = use_cuda,
+ .mlx_quantize_bits = mlx_quantize_bits,
+ .mlx_quantize_group_size = mlx_quantize_group_size,
+ .memory_utilization = memory_utilization,
+ .enable_cors = cors,
+ .api_key = api_key,
+ };
+ int rc = pplx_run_server(&scfg);
+ free_model_specs(&model_specs);
+ return rc;
+}