diff options
| -rw-r--r-- | Makefile | 55 | ||||
| -rw-r--r-- | main.c | 245 | ||||
| -rw-r--r-- | pplx_embed_cuda.cu | 4 | ||||
| -rw-r--r-- | pplx_server.c | 23 | ||||
| -rw-r--r-- | pplx_server.h | 4 | ||||
| -rw-r--r-- | server_main.c | 340 |
6 files changed, 441 insertions, 230 deletions
@@ -34,8 +34,12 @@ CUDA_SRCS = pplx_embed_cuda.cu OBJS = $(SRCS:.c=.o) MLX_OBJS = $(MLX_SRCS:.c=.o) -TARGET = pplx_embed -LIB = libpplxembed.a +TARGET = pplx_embed +SERVER_TARGET = pplx-embed-server +LIB = libpplxembed.a + +KERNEL_SRCS = qwen_asr_kernels.c qwen_asr_kernels_generic.c \ + qwen_asr_kernels_neon.c qwen_asr_kernels_avx.c ifeq ($(shell uname -s),Darwin) SHARED_LIB = libpplxembed.dylib @@ -45,7 +49,7 @@ SHARED_LIB = libpplxembed.so SHARED_FLAGS = -shared endif -.PHONY: all blas mlx cuda shared debug clean help +.PHONY: all blas mlx cuda shared test debug clean help all: help @@ -57,14 +61,16 @@ help: @echo " make mlx Build with Apple MLX GPU backend (recommended on Apple Silicon)" @echo " make cuda Build with CUDA/cuBLAS backend (Linux/NVIDIA)" @echo " make shared Build the shared library ($(SHARED_LIB), BLAS backend)" + @echo " make test Build and run the C test suite (no model files needed)" @echo " make debug Debug build with AddressSanitizer" @echo " make clean Remove build artifacts" @echo "" - @echo "Every backend target also produces the static library $(LIB)." + @echo "Each backend target builds ./$(TARGET) (CLI), ./$(SERVER_TARGET)" + @echo "(HTTP API), and the static library $(LIB)." @echo "" @echo "Usage:" @echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\"" - @echo " ./pplx_embed -d /path/to/model-dir --mlx \"text1\" \"text2\" (if built with make mlx)" + @echo " ./pplx-embed-server --model pplx-embed-v1-0.6b=/path/to/model-dir" # ============================================================================= # BLAS build (Apple Accelerate on macOS, OpenBLAS on Linux) @@ -84,7 +90,7 @@ blas: LDFLAGS += -lopenblas endif blas: $(MAKE) clean - $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" + $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" # ============================================================================= # MLX build (Apple Silicon GPU - uses mlx-c pure C API) @@ -105,7 +111,7 @@ mlx: LDFLAGS += -framework Accelerate -framework Metal -framework Foundation \ endif mlx: $(MAKE) clean - $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o" + $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o" # ============================================================================= # CUDA build (Linux NVIDIA GPU - uses CUDA Runtime + cuBLAS) @@ -118,7 +124,7 @@ cuda: LDFLAGS += -L$(CUDA_HOME)/lib64 -lopenblas -lcudart -lcublas cuda: $(MAKE) clean $(MAKE) pplx_embed_cuda.o - $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o" + $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o" pplx_embed_cuda.o: pplx_embed_cuda.cu pplx_embed_cuda.h pplx_embed_internal.h pplx_embed.h $(NVCC) -O3 -std=c++17 -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $< @@ -140,6 +146,26 @@ shared: $(SHARED_LIB): $(OBJS) $(EXTRA_OBJS) $(CC) $(CFLAGS) $(SHARED_FLAGS) -o $@ $^ $(LDFLAGS) $(CJSON_LDFLAGS) + +# ============================================================================= +# Tests (no model files required) +# ============================================================================= +ifeq ($(UNAME_S),Darwin) +TEST_BLAS_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK +TEST_BLAS_LDFLAGS = -framework Accelerate +else +TEST_BLAS_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -I/usr/include/openblas +TEST_BLAS_LDFLAGS = -lopenblas +endif + +test: + $(CC) -Wall -Wextra -O2 -I. -o tests/test_kernels_generic \ + tests/test_kernels.c $(KERNEL_SRCS) -lm -lpthread + ./tests/test_kernels_generic + $(CC) -Wall -Wextra -O2 $(TEST_BLAS_CFLAGS) -I. -o tests/test_kernels_blas \ + tests/test_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + ./tests/test_kernels_blas + # ============================================================================= # Debug build # ============================================================================= @@ -160,6 +186,9 @@ $(LIB): $(OBJS) $(EXTRA_OBJS) $(TARGET): $(LIB) main.o $(CC) $(CFLAGS) -o $@ main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) +$(SERVER_TARGET): $(LIB) server_main.o + $(CC) $(CFLAGS) -o $@ server_main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) + # ============================================================================= # Compile rules # ============================================================================= @@ -175,7 +204,10 @@ pplx_server.o: pplx_server.c pplx_server.h pplx_embed.h pplx_embed_mlx.h qwen_as pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_asr_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< -main.o: main.c pplx_embed.h pplx_distributed.h pplx_server.h qwen_asr_kernels.h qwen_asr_tokenizer.h +main.o: main.c pplx_embed.h pplx_distributed.h qwen_asr_kernels.h qwen_asr_tokenizer.h + $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< + +server_main.o: server_main.c pplx_embed.h pplx_server.h qwen_asr_kernels.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< qwen_asr_kernels.o: qwen_asr_kernels.c qwen_asr_kernels.h qwen_asr_kernels_impl.h @@ -201,5 +233,6 @@ deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h # ============================================================================= clean: - rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o $(TARGET) \ - $(LIB) libpplxembed.dylib libpplxembed.so + rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o server_main.o \ + $(TARGET) $(SERVER_TARGET) $(LIB) libpplxembed.dylib libpplxembed.so \ + tests/test_kernels_generic tests/test_kernels_blas @@ -2,7 +2,6 @@ #include "pplx_embed.h" #include "pplx_distributed.h" -#include "pplx_server.h" #include "qwen_asr_kernels.h" #include "qwen_asr_tokenizer.h" @@ -35,59 +34,50 @@ static void print_usage(const char *prog) " -d <dir> Model directory (required)\n" " --mlx Use Apple MLX GPU backend\n" " --cuda Use CUDA/cuBLAS GPU backend\n" - " --stdin Read lines from stdin, write JSON embeddings to stdout\n" - " --serve Serve the Perplexity-compatible HTTP API\n" + " --stream Read lines from stdin, write one JSON embedding per line\n" " --dist-worker\n" " Serve one final distributed layer shard\n" " --dist-remote HOST:PORT\n" - " Use one remote final shard for CLI or --stdin inference\n" + " Use one remote final shard for CLI or --stream inference\n" " --layers A:B Distributed half-open layer range, for example 0:14\n" " --dist-activation-bits N\n" " Distributed activation transport width: 32 or 16 (default: 32)\n" - " --model ID=DIR\n" - " Load model ID from directory for --serve (repeatable)\n" - " --backend cpu|mlx|cuda\n" - " Backend for --serve (default: cpu)\n" - " --mlx-quantize-bits N\n" + " --mlx-quant-bits N\n" " Quantize MLX linear weights to 8 bits at load time\n" - " --mlx-quantize-group-size N\n" + " --mlx-quant-group-size N\n" " MLX quantization group size (default: 64)\n" - " --cuda-fast-gemm MODE\n" + " --cuda-gemm-mode MODE\n" " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" - " --cuda-weights DTYPE\n" + " --cuda-weight-dtype DTYPE\n" " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" " weight memory and uses BF16 tensor cores\n" - " --allow-memory-overcommit\n" - " Allow an MLX server model set above the host-memory safety budget\n" - " --host HOST Server bind host (default: 127.0.0.1)\n" - " --port N Server bind port (default: 8000)\n" - " --cors Enable CORS headers in --serve mode\n" - " --api-key K Require Authorization: Bearer K in --serve mode\n" - " -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n" - " --max-batch-tokens N\n" - " Server max tokens per standard inference batch (default: 16384)\n" - " --batch-wait-us N\n" - " Server micro-batch wait in microseconds (default: 1000; 0 disables)\n" - " -t <n> CPU threads (default: all cores)\n" - " -e Print raw embeddings (with multiple texts)\n" - " -v Verbose (-vv for debug)\n" + " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n" + " --port N Bind port for --dist-worker (default: 8000)\n" + " -b, --batch-size N\n" + " Max texts per engine batch (default: all; --stream: 1)\n" + " -t, --threads N\n" + " CPU threads (default: all cores)\n" + " -e, --embeddings\n" + " Print raw embeddings (with multiple texts)\n" + " -v, --verbose\n" + " Verbose (-vv for debug)\n" " -h Show this help\n" "\n" "Modes:\n" " 1 text arg Print embedding as space-separated floats\n" " 2+ text args Print cosine similarity matrix\n" " no args Batch: read all stdin lines, then similarity matrix\n" - " --stdin Streaming: read stdin lines, write JSON per line\n" - " --serve HTTP server for /v1/embeddings and /v1/contextualizedembeddings\n" + " --stream Streaming: read stdin lines, write JSON per line\n" " --dist-worker Final layer-shard worker\n" "\n" + "The HTTP API is served by the separate pplx-embed-server binary.\n" + "\n" "Examples:\n" " %s -d ./model \"query: what is AI?\"\n" - " %s -d ./model --mlx --stdin < texts.txt\n" - " %s --serve --model pplx-embed-v1-0.6b=./model --port 8000\n" + " %s -d ./model --mlx --stream < texts.txt\n" " %s -d ./model --dist-worker --layers 14:28 --port 9000\n" " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n", - prog, prog, prog, prog, prog, prog); + prog, prog, prog, prog, prog); } /* ======================================================================== @@ -114,12 +104,6 @@ typedef struct { int n_tokens; } token_buf_t; -typedef struct { - pplx_server_model_spec_t *v; - int n; - int cap; -} model_specs_t; - static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, int batch, float *out_embeddings) { @@ -192,45 +176,6 @@ static void free_tokens(token_buf_t *tokens, int n) free(tokens[i].ids); } -static int append_model_spec(model_specs_t *specs, const char *arg) -{ - const char *eq = strchr(arg, '='); - if (!eq || eq == arg || !eq[1]) { - fprintf(stderr, "--model expects MODEL_ID=DIR\n"); - return -1; - } - if (specs->n == specs->cap) { - int new_cap = specs->cap ? specs->cap * 2 : 4; - void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); - if (!p) return -1; - specs->v = p; - specs->cap = new_cap; - } - size_t id_len = (size_t)(eq - arg); - char *id = (char *)malloc(id_len + 1); - char *path = strdup(eq + 1); - if (id) { - memcpy(id, arg, id_len); - id[id_len] = '\0'; - } - specs->v[specs->n].id = id; - specs->v[specs->n].path = path; - if (!specs->v[specs->n].id || !specs->v[specs->n].path) - return -1; - specs->n++; - return 0; -} - -static void free_model_specs(model_specs_t *specs) -{ - if (!specs) return; - for (int i = 0; i < specs->n; i++) { - free((char *)specs->v[i].id); - free((char *)specs->v[i].path); - } - free(specs->v); -} - /* ======================================================================== * Output helpers * ======================================================================== */ @@ -531,56 +476,29 @@ int main(int argc, char *argv[]) int use_mlx = 0; int use_cuda = 0; int stdin_mode = 0; - int serve_mode = 0; int dist_worker_mode = 0; - int allow_memory_overcommit = 0; const char *dist_remote = NULL; int layer_start = -1; int layer_end = -1; int batch_size = 0; - int max_batch_tokens = 0; - int batch_wait_us = -1; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; const char *cuda_fast_gemm = NULL; const char *cuda_weights = NULL; int dist_activation_bits = 32; const char *host = "127.0.0.1"; - const char *api_key = NULL; int port = 8000; - int cors = 0; - model_specs_t model_specs = {0}; int arg_start = 1; while (arg_start < argc && argv[arg_start][0] == '-') { const char *f = argv[arg_start]; if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; } - else if (!strcmp(f, "-t")) { n_threads = atoi(argv[++arg_start]); } + else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + n_threads = atoi(argv[++arg_start]); + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg_start]); } - else if (!strcmp(f, "--batch-wait-us")) { - batch_wait_us = atoi(argv[++arg_start]); - if (batch_wait_us < 0) { - fprintf(stderr, "--batch-wait-us must be >= 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--max-batch-tokens")) { - max_batch_tokens = atoi(argv[++arg_start]); - if (max_batch_tokens <= 0) { - fprintf(stderr, "--max-batch-tokens must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--model")) { - if (append_model_spec(&model_specs, argv[++arg_start]) != 0) { - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--backend")) { const char *backend = argv[++arg_start]; if (!strcmp(backend, "mlx")) { @@ -589,7 +507,6 @@ int main(int argc, char *argv[]) use_cuda = 0; #else fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); - free_model_specs(&model_specs); return 1; #endif } else if (!strcmp(backend, "cuda")) { @@ -598,7 +515,6 @@ int main(int argc, char *argv[]) use_mlx = 0; #else fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); - free_model_specs(&model_specs); return 1; #endif } else if (!strcmp(backend, "cpu")) { @@ -606,41 +522,34 @@ int main(int argc, char *argv[]) use_cuda = 0; } else { fprintf(stderr, "invalid --backend: %s\n", backend); - free_model_specs(&model_specs); return 1; } } - else if (!strcmp(f, "--mlx-quantize-bits")) { + else if (!strcmp(f, "--mlx-quant-bits")) { mlx_quantize_bits = atoi(argv[++arg_start]); if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quantize-bits must be 0 or 8\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); return 1; } } - else if (!strcmp(f, "--mlx-quantize-group-size")) { + else if (!strcmp(f, "--mlx-quant-group-size")) { mlx_quantize_group_size = atoi(argv[++arg_start]); if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quantize-group-size must be > 0\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); return 1; } } - else if (!strcmp(f, "--cuda-fast-gemm")) { + else if (!strcmp(f, "--cuda-gemm-mode")) { cuda_fast_gemm = argv[++arg_start]; } - else if (!strcmp(f, "--cuda-weights")) { + else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg_start]; } else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } - else if (!strcmp(f, "--allow-memory-overcommit")) { - allow_memory_overcommit = 1; - } else if (!strcmp(f, "--layers")) { if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) { fprintf(stderr, "--layers expects START:END with START < END\n"); - free_model_specs(&model_specs); return 1; } } @@ -648,17 +557,15 @@ int main(int argc, char *argv[]) dist_activation_bits = atoi(argv[++arg_start]); if (dist_activation_bits != 32 && dist_activation_bits != 16) { fprintf(stderr, "--dist-activation-bits must be 32 or 16\n"); - free_model_specs(&model_specs); return 1; } } - else if (!strcmp(f, "--cors")) { cors = 1; } - else if (!strcmp(f, "--api-key")){ api_key = argv[++arg_start]; } - else if (!strcmp(f, "-e")) { print_embs = 1; } - else if (!strcmp(f, "-v")) { verbose++; } + else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { + print_embs = 1; + } + else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "--stdin")) { stdin_mode = 1; } - else if (!strcmp(f, "--serve")) { serve_mode = 1; } + else if (!strcmp(f, "--stream")) { stdin_mode = 1; } else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; } else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; } else if (!strcmp(f, "--mlx")) { @@ -667,7 +574,6 @@ int main(int argc, char *argv[]) use_cuda = 0; #else fprintf(stderr, "--mlx not available (build with: make mlx)\n"); - free_model_specs(&model_specs); return 1; #endif } @@ -677,13 +583,11 @@ int main(int argc, char *argv[]) use_mlx = 0; #else fprintf(stderr, "--cuda not available (build with: make cuda)\n"); - free_model_specs(&model_specs); return 1; #endif } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(argv[0]); - free_model_specs(&model_specs); return 0; } else break; @@ -696,18 +600,15 @@ int main(int argc, char *argv[]) if (cuda_fast_gemm) { #ifdef USE_CUDA if (!use_cuda) { - fprintf(stderr, "--cuda-fast-gemm requires --cuda or --backend cuda\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-gemm-mode requires --cuda or --backend cuda\n"); return 1; } if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { - fprintf(stderr, "--cuda-fast-gemm must be f32, tf32, bf16, or 16f\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); return 1; } #else - fprintf(stderr, "--cuda-fast-gemm requires a CUDA build\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); return 1; #endif } @@ -715,8 +616,7 @@ int main(int argc, char *argv[]) if (cuda_weights) { #ifdef USE_CUDA if (!use_cuda) { - fprintf(stderr, "--cuda-weights requires --cuda or --backend cuda\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-weight-dtype requires --cuda or --backend cuda\n"); return 1; } if (!strcmp(cuda_weights, "bf16")) { @@ -724,77 +624,18 @@ int main(int argc, char *argv[]) } else if (!strcmp(cuda_weights, "f32")) { pplx_cuda_set_weights_bf16(0); } else { - fprintf(stderr, "--cuda-weights must be f32 or bf16\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); return 1; } #else - fprintf(stderr, "--cuda-weights requires a CUDA build\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); return 1; #endif } - if (serve_mode) { - if (stdin_mode || dist_worker_mode || dist_remote) { - fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n"); - free_model_specs(&model_specs); - return 1; - } - if (model_specs.n == 0) { - fprintf(stderr, "--serve requires at least one --model MODEL_ID=DIR\n"); - print_usage(argv[0]); - free_model_specs(&model_specs); - return 1; - } - if (dist_activation_bits != 32) { - fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n"); - free_model_specs(&model_specs); - return 1; - } - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n"); - free_model_specs(&model_specs); - return 1; - } - if (!use_mlx && !use_cuda) { - if (n_threads <= 0) n_threads = qwen_get_num_cpus(); - qwen_set_threads(n_threads); - if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } else if (verbose >= 1) { - fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); - } - pplx_server_config_t scfg = { - .models = model_specs.v, - .n_models = model_specs.n, - .host = host, - .port = port, - .batch_size = batch_size > 0 ? batch_size : 8, - .max_batch_tokens = max_batch_tokens, - .batch_wait_us = batch_wait_us, - .use_mlx = use_mlx, - .use_cuda = use_cuda, - .mlx_quantize_bits = mlx_quantize_bits, - .mlx_quantize_group_size = mlx_quantize_group_size, - .allow_memory_overcommit = allow_memory_overcommit, - .enable_cors = cors, - .api_key = api_key, - }; - int rc = pplx_run_server(&scfg); - free_model_specs(&model_specs); - return rc; - } - - if (model_specs.n != 0) { - fprintf(stderr, "--model is only valid with --serve\n"); - free_model_specs(&model_specs); - return 1; - } - if (!model_dir || !model_dir[0]) { fprintf(stderr, "model directory required (-d <dir>)\n"); print_usage(argv[0]); - free_model_specs(&model_specs); return 1; } @@ -831,7 +672,7 @@ int main(int argc, char *argv[]) } if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quantize-bits requires --mlx or --backend mlx\n"); + fprintf(stderr, "--mlx-quant-bits requires --mlx or --backend mlx\n"); return 1; } @@ -958,7 +799,6 @@ int main(int argc, char *argv[]) if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); #endif qwen_tokenizer_free(tok); - free_model_specs(&model_specs); return 1; } @@ -981,6 +821,5 @@ int main(int argc, char *argv[]) #endif qwen_tokenizer_workspace_free(e.tok_ws); qwen_tokenizer_free(tok); - free_model_specs(&model_specs); return rc; } diff --git a/pplx_embed_cuda.cu b/pplx_embed_cuda.cu index bda1be2..a00fc84 100644 --- a/pplx_embed_cuda.cu +++ b/pplx_embed_cuda.cu @@ -537,14 +537,14 @@ static int launch_check(void) } // Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm). -// PPLX_CUDA_FAST_GEMM={bf16,tf32,16f} selects a reduced-precision tensor-core +// PPLX_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core // matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands // internally. static cublasComputeType_t gemm_compute(void) { static int env_init = 0; if (!g_gemm_compute_set && !env_init) { - const char *e = getenv("PPLX_CUDA_FAST_GEMM"); + const char *e = getenv("PPLX_CUDA_GEMM_MODE"); if (e && parse_gemm_compute(e, &g_gemm_compute) == 0) g_gemm_compute_set = 1; env_init = 1; diff --git a/pplx_server.c b/pplx_server.c index 6e7bb22..f8ad5e1 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -2266,29 +2266,26 @@ static int mlx_memory_preflight(const pplx_server_config_t *cfg) { "skipping MLX memory preflight"); return 0; } - uint64_t budget = - physical / 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT + - physical % 100 * PPLX_MLX_MEMORY_BUDGET_PERCENT / 100; + double utilization = cfg->memory_utilization > 0.0 + ? cfg->memory_utilization + : PPLX_MLX_MEMORY_BUDGET_PERCENT / 100.0; + uint64_t budget = (uint64_t)((double)physical * utilization); const double gib = 1024.0 * 1024.0 * 1024.0; server_log("pplx-serve: MLX memory preflight: %.1f GiB tensors, " - "%.1f GiB estimated resident, %.1f GiB safety budget", + "%.1f GiB estimated resident, %.1f GiB budget " + "(%.2f of physical memory)", (double)payload / gib, (double)estimated / gib, - (double)budget / gib); + (double)budget / gib, utilization); if (cfg->mlx_quantize_bits) { server_log("pplx-serve: MLX %d-bit quantization enabled; preflight " "uses source payload as a conservative peak estimate", cfg->mlx_quantize_bits); } if (estimated <= budget) return 0; - if (cfg->allow_memory_overcommit) { - server_log("pplx-serve: warning: MLX memory estimate exceeds safety " - "budget; continuing due to --allow-memory-overcommit"); - return 0; - } - server_log("pplx-serve: refusing MLX model set above host-memory safety " + server_log("pplx-serve: refusing MLX model set above the host-memory " "budget"); - server_log("pplx-serve: use BF16 artifacts, load fewer models, or pass " - "--allow-memory-overcommit"); + server_log("pplx-serve: use BF16 artifacts, load fewer models, or raise " + "--memory-utilization (values above 1.0 overcommit)"); return -1; } diff --git a/pplx_server.h b/pplx_server.h index 225a7ac..c38f0ca 100644 --- a/pplx_server.h +++ b/pplx_server.h @@ -20,7 +20,9 @@ typedef struct { int use_cuda; int mlx_quantize_bits; int mlx_quantize_group_size; - int allow_memory_overcommit; + double memory_utilization; /* fraction of physical memory the MLX + preflight may plan for; 0 = default 0.90, + values above 1.0 allow overcommit */ int enable_cors; const char *api_key; } pplx_server_config_t; diff --git a/server_main.c b/server_main.c new file mode 100644 index 0000000..b4c7a34 --- /dev/null +++ b/server_main.c @@ -0,0 +1,340 @@ +/* server_main.c - pplx-embed-server: Perplexity-compatible embeddings API */ + +#include "pplx_embed.h" +#include "pplx_server.h" +#include "qwen_asr_kernels.h" + +#ifdef USE_CUDA +#include "pplx_embed_cuda.h" +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static void print_usage(const char *prog) +{ + fprintf(stderr, + "Usage: %s --model ID=DIR [options]\n" + "\n" + "Serves POST /v1/embeddings and /v1/contextualizedembeddings.\n" + "\n" + "Options:\n" + " --model ID=DIR Model to serve (repeatable)\n" + " --backend cpu|mlx|cuda Execution backend (default: cpu); --mlx and\n" + " --cuda are shorthands\n" + " --host HOST Bind host (default: 127.0.0.1)\n" + " --port N Bind port (default: 8000)\n" + " --api-key K Require Authorization: Bearer K\n" + " --cors Enable CORS headers\n" + " -b, --batch-size N Max texts or documents per inference batch\n" + " (default: 8)\n" + " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --batch-wait-us N Micro-batch wait in microseconds\n" + " (default: 1000; 0 drains only queued work)\n" + " --memory-utilization F Fraction of physical memory the MLX model-set\n" + " preflight may plan for (default: 0.90; values\n" + " above 1.0 overcommit)\n" + " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" + " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" + " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" + " (default: f32, exact)\n" + " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" + " f32); bf16 halves weight memory\n" + " -t, --threads N CPU threads (default: all cores)\n" + " -v, --verbose Verbose (-vv for debug)\n" + " -h, --help Show this help\n" + "\n" + "Examples:\n" + " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" + " %s --backend mlx --mlx-quant-bits 8 \\\n" + " --model pplx-embed-v1-4b=./model-4b-bf16\n", + prog, prog, prog); +} + +typedef struct { + pplx_server_model_spec_t *v; + int n; + int cap; +} model_specs_t; + +static int append_model_spec(model_specs_t *specs, const char *arg) +{ + const char *eq = strchr(arg, '='); + if (!eq || eq == arg || !eq[1]) { + fprintf(stderr, "--model expects MODEL_ID=DIR\n"); + return -1; + } + if (specs->n == specs->cap) { + int new_cap = specs->cap ? specs->cap * 2 : 4; + void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); + if (!p) return -1; + specs->v = p; + specs->cap = new_cap; + } + size_t id_len = (size_t)(eq - arg); + char *id = (char *)malloc(id_len + 1); + char *path = strdup(eq + 1); + if (id) { + memcpy(id, arg, id_len); + id[id_len] = '\0'; + } + specs->v[specs->n].id = id; + specs->v[specs->n].path = path; + if (!specs->v[specs->n].id || !specs->v[specs->n].path) + return -1; + specs->n++; + return 0; +} + +static void free_model_specs(model_specs_t *specs) +{ + if (!specs) return; + for (int i = 0; i < specs->n; i++) { + free((char *)specs->v[i].id); + free((char *)specs->v[i].path); + } + free(specs->v); +} + +int main(int argc, char *argv[]) +{ + int n_threads = 0; + int verbose = 0; + int use_mlx = 0; + int use_cuda = 0; + int batch_size = 0; + int max_batch_tokens = 0; + int batch_wait_us = -1; + int mlx_quantize_bits = 0; + int mlx_quantize_group_size = 64; + const char *cuda_fast_gemm = NULL; + const char *cuda_weights = NULL; + double memory_utilization = 0.0; + const char *host = "127.0.0.1"; + const char *api_key = NULL; + int port = 8000; + int cors = 0; + model_specs_t model_specs = {0}; + + int arg = 1; + while (arg < argc && argv[arg][0] == '-') { + const char *f = argv[arg]; + if (!strcmp(f, "--model")) { + if (append_model_spec(&model_specs, argv[++arg]) != 0) { + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--backend")) { + const char *backend = argv[++arg]; + if (!strcmp(backend, "mlx")) { +#ifdef USE_MLX + use_mlx = 1; + use_cuda = 0; +#else + fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } else if (!strcmp(backend, "cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } else if (!strcmp(backend, "cpu")) { + use_mlx = 0; + use_cuda = 0; + } else { + fprintf(stderr, "invalid --backend: %s\n", backend); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx")) { +#ifdef USE_MLX + use_mlx = 1; + use_cuda = 0; +#else + fprintf(stderr, "--mlx not available (build with: make mlx)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + else if (!strcmp(f, "--cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "--cuda not available (build with: make cuda)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + else if (!strcmp(f, "--host")) { host = argv[++arg]; } + else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); } + else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; } + else if (!strcmp(f, "--cors")) { cors = 1; } + else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + batch_size = atoi(argv[++arg]); + } + else if (!strcmp(f, "--max-batch-tokens")) { + max_batch_tokens = atoi(argv[++arg]); + if (max_batch_tokens <= 0) { + fprintf(stderr, "--max-batch-tokens must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--batch-wait-us")) { + batch_wait_us = atoi(argv[++arg]); + if (batch_wait_us < 0) { + fprintf(stderr, "--batch-wait-us must be >= 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--memory-utilization")) { + memory_utilization = atof(argv[++arg]); + if (memory_utilization <= 0.0) { + fprintf(stderr, "--memory-utilization must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx-quant-bits")) { + mlx_quantize_bits = atoi(argv[++arg]); + if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { + fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx-quant-group-size")) { + mlx_quantize_group_size = atoi(argv[++arg]); + if (mlx_quantize_group_size <= 0) { + fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--cuda-gemm-mode")) { + cuda_fast_gemm = argv[++arg]; + } + else if (!strcmp(f, "--cuda-weight-dtype")) { + cuda_weights = argv[++arg]; + } + else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + n_threads = atoi(argv[++arg]); + } + else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } + else if (!strcmp(f, "-vv")) { verbose = 2; } + else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + print_usage(argv[0]); + free_model_specs(&model_specs); + return 0; + } + else { + fprintf(stderr, "unknown option: %s\n", f); + print_usage(argv[0]); + free_model_specs(&model_specs); + return 1; + } + arg++; + } + + pplx_verbose = verbose; + qwen_verbose = verbose; + + if (arg < argc) { + fprintf(stderr, "unexpected argument: %s\n", argv[arg]); + print_usage(argv[0]); + free_model_specs(&model_specs); + return 1; + } + if (model_specs.n == 0) { + fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n"); + print_usage(argv[0]); + free_model_specs(&model_specs); + return 1; + } + if (mlx_quantize_bits && !use_mlx) { + fprintf(stderr, "--mlx-quant-bits requires --backend mlx\n"); + free_model_specs(&model_specs); + return 1; + } + + if (cuda_fast_gemm) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-gemm-mode requires --backend cuda\n"); + free_model_specs(&model_specs); + return 1; + } + if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { + fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); + free_model_specs(&model_specs); + return 1; + } +#else + fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + + if (cuda_weights) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-weight-dtype requires --backend cuda\n"); + free_model_specs(&model_specs); + return 1; + } + if (!strcmp(cuda_weights, "bf16")) { + pplx_cuda_set_weights_bf16(1); + } else if (!strcmp(cuda_weights, "f32")) { + pplx_cuda_set_weights_bf16(0); + } else { + fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); + free_model_specs(&model_specs); + return 1; + } +#else + fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + + if (!use_mlx && !use_cuda) { + if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + qwen_set_threads(n_threads); + if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + } else if (verbose >= 1) { + fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); + } + + pplx_server_config_t scfg = { + .models = model_specs.v, + .n_models = model_specs.n, + .host = host, + .port = port, + .batch_size = batch_size > 0 ? batch_size : 8, + .max_batch_tokens = max_batch_tokens, + .batch_wait_us = batch_wait_us, + .use_mlx = use_mlx, + .use_cuda = use_cuda, + .mlx_quantize_bits = mlx_quantize_bits, + .mlx_quantize_group_size = mlx_quantize_group_size, + .memory_utilization = memory_utilization, + .enable_cors = cors, + .api_key = api_key, + }; + int rc = pplx_run_server(&scfg); + free_model_specs(&model_specs); + return rc; +} |
