diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 08:28:13 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | ac994a45e239bdbb9aa5c34cbab2de71295ba420 (patch) | |
| tree | 98c7adf49d156f641d5f0b1e02bcc79aea75991f | |
| parent | 2c37727e653d2340d8977dd5db4addc767dd81ea (diff) | |
| download | ffwd-ac994a45e239bdbb9aa5c34cbab2de71295ba420.tar.gz ffwd-ac994a45e239bdbb9aa5c34cbab2de71295ba420.zip | |
chore: rename sources; binaries pplx-embed, pplx-embed-server
| -rw-r--r-- | Makefile | 49 | ||||
| -rw-r--r-- | embed.c | 4 | ||||
| -rw-r--r-- | embed.h | 2 | ||||
| -rw-r--r-- | embed_cli.c | 10 | ||||
| -rw-r--r-- | embed_cuda.cu | 4 | ||||
| -rw-r--r-- | embed_cuda.h | 4 | ||||
| -rw-r--r-- | embed_distributed.c | 6 | ||||
| -rw-r--r-- | embed_distributed.h | 4 | ||||
| -rw-r--r-- | embed_internal.h | 2 | ||||
| -rw-r--r-- | embed_mlx.c | 6 | ||||
| -rw-r--r-- | embed_mlx.h | 6 | ||||
| -rw-r--r-- | embed_server.c | 343 |
12 files changed, 386 insertions, 54 deletions
@@ -16,9 +16,8 @@ endif UNAME_S := $(shell uname -s) # Source files -SRCS = pplx_embed.c \ - pplx_distributed.c \ - pplx_server.c \ +SRCS = embed.c \ + embed_distributed.c \ qwen_kernels.c \ qwen_kernels_generic.c \ qwen_kernels_neon.c \ @@ -29,12 +28,12 @@ SRCS = pplx_embed.c \ deps/ae/anet.c \ deps/ae/monotonic.c -MLX_SRCS = pplx_embed_mlx.c -CUDA_SRCS = pplx_embed_cuda.cu +MLX_SRCS = embed_mlx.c +CUDA_SRCS = embed_cuda.cu OBJS = $(SRCS:.c=.o) MLX_OBJS = $(MLX_SRCS:.c=.o) -TARGET = pplx_embed +TARGET = pplx-embed SERVER_TARGET = pplx-embed-server LIB = libpplxembed.a @@ -54,7 +53,7 @@ endif all: help help: - @echo "pplx_embed - Inference for pplx-embed-v1 and pplx-embed-context-v1 embedding models" + @echo "pplx-embed - Inference for pplx-embed-v1 and pplx-embed-context-v1 embedding models" @echo "" @echo "Build targets:" @echo " make blas Build with BLAS acceleration (CPU)" @@ -69,7 +68,7 @@ help: @echo "(HTTP API), and the static library $(LIB)." @echo "" @echo "Usage:" - @echo " ./pplx_embed -d /path/to/model-dir \"text1\" \"text2\"" + @echo " ./pplx-embed -d /path/to/model-dir \"text1\" \"text2\"" @echo " ./pplx-embed-server --model pplx-embed-v1-0.6b=/path/to/model-dir" # ============================================================================= @@ -99,8 +98,8 @@ blas: MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null) MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null) -mlx: SRCS += pplx_embed_mlx.c -mlx: OBJS = $(SRCS:.c=.o) pplx_embed_mlx.o +mlx: SRCS += embed_mlx.c +mlx: OBJS = $(SRCS:.c=.o) embed_mlx.o ifeq ($(UNAME_S),Darwin) mlx: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX \ -I$(MLXC_PREFIX)/include @@ -111,7 +110,7 @@ mlx: LDFLAGS += -framework Accelerate -framework Metal -framework Foundation \ endif mlx: $(MAKE) clean - $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_mlx.o" + $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="embed_mlx.o" # ============================================================================= # CUDA build (Linux NVIDIA GPU - uses CUDA Runtime + cuBLAS) @@ -123,10 +122,10 @@ cuda: CFLAGS = $(CFLAGS_BASE) -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/inclu cuda: LDFLAGS += -L$(CUDA_HOME)/lib64 -lopenblas -lcudart -lcublas cuda: $(MAKE) clean - $(MAKE) pplx_embed_cuda.o - $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="pplx_embed_cuda.o" + $(MAKE) embed_cuda.o + $(MAKE) $(TARGET) $(SERVER_TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_OBJS="embed_cuda.o" -pplx_embed_cuda.o: pplx_embed_cuda.cu pplx_embed_cuda.h pplx_embed_internal.h pplx_embed.h +embed_cuda.o: embed_cuda.cu embed_cuda.h embed_internal.h embed.h $(NVCC) -O3 -std=c++17 -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $< # ============================================================================= @@ -183,32 +182,30 @@ debug: $(LIB): $(OBJS) $(EXTRA_OBJS) ar rcs $@ $^ -$(TARGET): $(LIB) main.o - $(CC) $(CFLAGS) -o $@ main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) +$(TARGET): $(LIB) embed_cli.o + $(CC) $(CFLAGS) -o $@ embed_cli.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) -$(SERVER_TARGET): $(LIB) server_main.o - $(CC) $(CFLAGS) -o $@ server_main.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) +$(SERVER_TARGET): $(LIB) embed_server.o + $(CC) $(CFLAGS) -o $@ embed_server.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) # ============================================================================= # Compile rules # ============================================================================= -pplx_embed.o: pplx_embed.c pplx_embed.h qwen_kernels.h qwen_safetensors.h +embed.o: embed.c embed.h qwen_kernels.h qwen_safetensors.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< -pplx_distributed.o: pplx_distributed.c pplx_distributed.h pplx_embed.h pplx_embed_mlx.h +embed_distributed.o: embed_distributed.c embed_distributed.h embed.h embed_mlx.h $(CC) $(CFLAGS) -c -o $@ $< -pplx_server.o: pplx_server.c pplx_server.h pplx_embed.h pplx_embed_mlx.h qwen_tokenizer.h deps/ae/ae.h deps/ae/anet.h +embed_server.o: embed_server.c embed_server.h embed.h embed_mlx.h qwen_tokenizer.h deps/ae/ae.h deps/ae/anet.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< -pplx_embed_mlx.o: pplx_embed_mlx.c pplx_embed_mlx.h pplx_embed.h qwen_safetensors.h +embed_mlx.o: embed_mlx.c embed_mlx.h embed.h qwen_safetensors.h $(CC) $(CFLAGS) -c -o $@ $< -main.o: main.c pplx_embed.h pplx_distributed.h qwen_kernels.h qwen_tokenizer.h +embed_cli.o: embed_cli.c embed.h embed_distributed.h qwen_kernels.h qwen_tokenizer.h $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< -server_main.o: server_main.c pplx_embed.h pplx_server.h qwen_kernels.h - $(CC) $(CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< qwen_kernels.o: qwen_kernels.c qwen_kernels.h qwen_kernels_impl.h $(CC) $(CFLAGS) -c -o $@ $< @@ -233,6 +230,6 @@ deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h # ============================================================================= clean: - rm -f $(OBJS) pplx_embed_mlx.o pplx_embed_cuda.o main.o server_main.o \ + rm -f $(OBJS) embed_mlx.o embed_cuda.o embed_cli.o embed_server.o \ $(TARGET) $(SERVER_TARGET) $(LIB) libpplxembed.dylib libpplxembed.so \ tests/test_kernels_generic tests/test_kernels_blas @@ -1,5 +1,5 @@ -#include "pplx_embed.h" -#include "pplx_embed_internal.h" +#include "embed.h" +#include "embed_internal.h" #include "qwen_kernels.h" #include "qwen_safetensors.h" @@ -1,5 +1,5 @@ /* - * pplx_embed.h - Pure C inference for Perplexity AI's pplx-embed models. + * embed.h - Pure C inference for Perplexity AI's pplx-embed models. * * Supports all pplx-embed-v1 and pplx-embed-context-v1 variants (0.6B, 4B). */ diff --git a/embed_cli.c b/embed_cli.c index 140494b..b9dc311 100644 --- a/embed_cli.c +++ b/embed_cli.c @@ -1,15 +1,15 @@ -/* main.c - pplx_embed command-line tool */ +/* embed_cli.c - pplx-embed command-line tool */ -#include "pplx_embed.h" -#include "pplx_distributed.h" +#include "embed.h" +#include "embed_distributed.h" #include "qwen_kernels.h" #include "qwen_tokenizer.h" #ifdef USE_MLX -#include "pplx_embed_mlx.h" +#include "embed_mlx.h" #endif #ifdef USE_CUDA -#include "pplx_embed_cuda.h" +#include "embed_cuda.h" #endif #include <stdio.h> diff --git a/embed_cuda.cu b/embed_cuda.cu index a00fc84..c1406b4 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -1,6 +1,6 @@ extern "C" { -#include "pplx_embed_cuda.h" -#include "pplx_embed_internal.h" +#include "embed_cuda.h" +#include "embed_internal.h" } #include <cuda_runtime.h> diff --git a/embed_cuda.h b/embed_cuda.h index badff9c..8ebe6e2 100644 --- a/embed_cuda.h +++ b/embed_cuda.h @@ -1,11 +1,11 @@ /* - * pplx_embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference. + * embed_cuda.h - CUDA/cuBLAS backend for pplx-embed inference. */ #ifndef PPLX_EMBED_CUDA_H #define PPLX_EMBED_CUDA_H -#include "pplx_embed.h" +#include "embed.h" typedef struct pplx_cuda_ctx pplx_cuda_ctx_t; diff --git a/embed_distributed.c b/embed_distributed.c index d4254f7..92d22c1 100644 --- a/embed_distributed.c +++ b/embed_distributed.c @@ -1,5 +1,5 @@ /* - * pplx_distributed.c - Stateless two-node layer-sharded inference. + * embed_distributed.c - Stateless two-node layer-sharded inference. * * This first transport is deliberately synchronous: one coordinator sends a * packed hidden-state batch to one final worker and receives pooled @@ -7,10 +7,10 @@ * retries, and richer transport can extend it without changing model math. */ -#include "pplx_distributed.h" +#include "embed_distributed.h" #ifdef USE_MLX -#include "pplx_embed_mlx.h" +#include "embed_mlx.h" #endif #include <arpa/inet.h> diff --git a/embed_distributed.h b/embed_distributed.h index cd97147..4cd32e4 100644 --- a/embed_distributed.h +++ b/embed_distributed.h @@ -1,11 +1,11 @@ /* - * pplx_distributed.h - Stateless two-node layer-sharded inference. + * embed_distributed.h - Stateless two-node layer-sharded inference. */ #ifndef PPLX_DISTRIBUTED_H #define PPLX_DISTRIBUTED_H -#include "pplx_embed.h" +#include "embed.h" typedef struct pplx_dist_client pplx_dist_client_t; diff --git a/embed_internal.h b/embed_internal.h index 1e7e4d0..7d6f398 100644 --- a/embed_internal.h +++ b/embed_internal.h @@ -1,7 +1,7 @@ #ifndef PPLX_EMBED_INTERNAL_H #define PPLX_EMBED_INTERNAL_H -#include "pplx_embed.h" +#include "embed.h" #include "qwen_safetensors.h" typedef struct { diff --git a/embed_mlx.c b/embed_mlx.c index 602571d..4ef283c 100644 --- a/embed_mlx.c +++ b/embed_mlx.c @@ -1,5 +1,5 @@ -#include "pplx_embed_mlx.h" -#include "pplx_embed.h" +#include "embed_mlx.h" +#include "embed.h" #include "qwen_safetensors.h" #include <mlx/c/mlx.h> @@ -333,7 +333,7 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s) * ======================================================================== */ /* We duplicate a minimal config parser here so MLX can load standalone. - * (Same logic as pplx_embed.c:parse_config, kept inline for simplicity.) */ + * (Same logic as embed.c:parse_config, kept inline for simplicity.) */ static const char *skip_ws_m(const char *p) { diff --git a/embed_mlx.h b/embed_mlx.h index f2ad136..25376e9 100644 --- a/embed_mlx.h +++ b/embed_mlx.h @@ -1,5 +1,5 @@ /* - * pplx_embed_mlx.h - MLX GPU backend for pplx-embed inference + * embed_mlx.h - MLX GPU backend for pplx-embed inference * * Uses Apple's mlx-c to run the transformer on Metal. */ @@ -9,7 +9,7 @@ #include <stddef.h> #include <stdint.h> -#include "pplx_embed.h" +#include "embed.h" /* Opaque MLX model context */ typedef struct pplx_mlx_ctx pplx_mlx_ctx_t; @@ -143,7 +143,7 @@ int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, const pplx_context_input_t *inputs, int batch, float *out_embeddings); -/* Get the config (so main.c can read hidden_size etc.) */ +/* Get the config (so embed_cli.c can read hidden_size etc.) */ const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx); #endif /* PPLX_EMBED_MLX_H */ diff --git a/embed_server.c b/embed_server.c index d831062..f74bdad 100644 --- a/embed_server.c +++ b/embed_server.c @@ -2,16 +2,17 @@ #define _GNU_SOURCE #endif -#include "pplx_server.h" -#include "pplx_embed.h" +#include "embed_server.h" +#include "embed.h" #include "qwen_safetensors.h" +#include "qwen_kernels.h" #include "qwen_tokenizer.h" #ifdef USE_MLX -#include "pplx_embed_mlx.h" +#include "embed_mlx.h" #endif #ifdef USE_CUDA -#include "pplx_embed_cuda.h" +#include "embed_cuda.h" #endif #include "deps/ae/ae.h" @@ -2429,3 +2430,337 @@ int pplx_run_server(const pplx_server_config_t *cfg) { pthread_mutex_destroy(&s.mu); return 0; } + +/* ==================================================================== + * pplx-embed-server entry point + * ==================================================================== */ +#ifndef EMBED_SERVER_TEST + +static void print_usage(const char *prog) +{ + fprintf(stderr, + "Usage: %s --model ID=DIR [options]\n" + "\n" + "Serves POST /v1/embeddings and /v1/contextualizedembeddings.\n" + "\n" + "Options:\n" + " --model ID=DIR Model to serve (repeatable)\n" + " --backend cpu|mlx|cuda Execution backend (default: cpu); --mlx and\n" + " --cuda are shorthands\n" + " --host HOST Bind host (default: 127.0.0.1)\n" + " --port N Bind port (default: 8000)\n" + " --api-key K Require Authorization: Bearer K\n" + " --cors Enable CORS headers\n" + " -b, --batch-size N Max texts or documents per inference batch\n" + " (default: 8)\n" + " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --batch-wait-us N Micro-batch wait in microseconds\n" + " (default: 1000; 0 drains only queued work)\n" + " --memory-utilization F Fraction of physical memory the MLX model-set\n" + " preflight may plan for (default: 0.90; values\n" + " above 1.0 overcommit)\n" + " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" + " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" + " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" + " (default: f32, exact)\n" + " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" + " f32); bf16 halves weight memory\n" + " -t, --threads N CPU threads (default: all cores)\n" + " -v, --verbose Verbose (-vv for debug)\n" + " -h, --help Show this help\n" + "\n" + "Examples:\n" + " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" + " %s --backend mlx --mlx-quant-bits 8 \\\n" + " --model pplx-embed-v1-4b=./model-4b-bf16\n", + prog, prog, prog); +} + +typedef struct { + pplx_server_model_spec_t *v; + int n; + int cap; +} model_specs_t; + +static int append_model_spec(model_specs_t *specs, const char *arg) +{ + const char *eq = strchr(arg, '='); + if (!eq || eq == arg || !eq[1]) { + fprintf(stderr, "--model expects MODEL_ID=DIR\n"); + return -1; + } + if (specs->n == specs->cap) { + int new_cap = specs->cap ? specs->cap * 2 : 4; + void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); + if (!p) return -1; + specs->v = p; + specs->cap = new_cap; + } + size_t id_len = (size_t)(eq - arg); + char *id = (char *)malloc(id_len + 1); + char *path = strdup(eq + 1); + if (id) { + memcpy(id, arg, id_len); + id[id_len] = '\0'; + } + specs->v[specs->n].id = id; + specs->v[specs->n].path = path; + if (!specs->v[specs->n].id || !specs->v[specs->n].path) + return -1; + specs->n++; + return 0; +} + +static void free_model_specs(model_specs_t *specs) +{ + if (!specs) return; + for (int i = 0; i < specs->n; i++) { + free((char *)specs->v[i].id); + free((char *)specs->v[i].path); + } + free(specs->v); +} + +int main(int argc, char *argv[]) +{ + int n_threads = 0; + int verbose = 0; + int use_mlx = 0; + int use_cuda = 0; + int batch_size = 0; + int max_batch_tokens = 0; + int batch_wait_us = -1; + int mlx_quantize_bits = 0; + int mlx_quantize_group_size = 64; + const char *cuda_fast_gemm = NULL; + const char *cuda_weights = NULL; + double memory_utilization = 0.0; + const char *host = "127.0.0.1"; + const char *api_key = NULL; + int port = 8000; + int cors = 0; + model_specs_t model_specs = {0}; + + int arg = 1; + while (arg < argc && argv[arg][0] == '-') { + const char *f = argv[arg]; + if (!strcmp(f, "--model")) { + if (append_model_spec(&model_specs, argv[++arg]) != 0) { + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--backend")) { + const char *backend = argv[++arg]; + if (!strcmp(backend, "mlx")) { +#ifdef USE_MLX + use_mlx = 1; + use_cuda = 0; +#else + fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } else if (!strcmp(backend, "cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } else if (!strcmp(backend, "cpu")) { + use_mlx = 0; + use_cuda = 0; + } else { + fprintf(stderr, "invalid --backend: %s\n", backend); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx")) { +#ifdef USE_MLX + use_mlx = 1; + use_cuda = 0; +#else + fprintf(stderr, "--mlx not available (build with: make mlx)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + else if (!strcmp(f, "--cuda")) { +#ifdef USE_CUDA + use_cuda = 1; + use_mlx = 0; +#else + fprintf(stderr, "--cuda not available (build with: make cuda)\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + else if (!strcmp(f, "--host")) { host = argv[++arg]; } + else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); } + else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; } + else if (!strcmp(f, "--cors")) { cors = 1; } + else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + batch_size = atoi(argv[++arg]); + } + else if (!strcmp(f, "--max-batch-tokens")) { + max_batch_tokens = atoi(argv[++arg]); + if (max_batch_tokens <= 0) { + fprintf(stderr, "--max-batch-tokens must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--batch-wait-us")) { + batch_wait_us = atoi(argv[++arg]); + if (batch_wait_us < 0) { + fprintf(stderr, "--batch-wait-us must be >= 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--memory-utilization")) { + memory_utilization = atof(argv[++arg]); + if (memory_utilization <= 0.0) { + fprintf(stderr, "--memory-utilization must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx-quant-bits")) { + mlx_quantize_bits = atoi(argv[++arg]); + if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { + fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx-quant-group-size")) { + mlx_quantize_group_size = atoi(argv[++arg]); + if (mlx_quantize_group_size <= 0) { + fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--cuda-gemm-mode")) { + cuda_fast_gemm = argv[++arg]; + } + else if (!strcmp(f, "--cuda-weight-dtype")) { + cuda_weights = argv[++arg]; + } + else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + n_threads = atoi(argv[++arg]); + } + else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } + else if (!strcmp(f, "-vv")) { verbose = 2; } + else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + print_usage(argv[0]); + free_model_specs(&model_specs); + return 0; + } + else { + fprintf(stderr, "unknown option: %s\n", f); + print_usage(argv[0]); + free_model_specs(&model_specs); + return 1; + } + arg++; + } + + pplx_verbose = verbose; + qwen_verbose = verbose; + + if (arg < argc) { + fprintf(stderr, "unexpected argument: %s\n", argv[arg]); + print_usage(argv[0]); + free_model_specs(&model_specs); + return 1; + } + if (model_specs.n == 0) { + fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n"); + print_usage(argv[0]); + free_model_specs(&model_specs); + return 1; + } + if (mlx_quantize_bits && !use_mlx) { + fprintf(stderr, "--mlx-quant-bits requires --backend mlx\n"); + free_model_specs(&model_specs); + return 1; + } + + if (cuda_fast_gemm) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-gemm-mode requires --backend cuda\n"); + free_model_specs(&model_specs); + return 1; + } + if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { + fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); + free_model_specs(&model_specs); + return 1; + } +#else + fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + + if (cuda_weights) { +#ifdef USE_CUDA + if (!use_cuda) { + fprintf(stderr, "--cuda-weight-dtype requires --backend cuda\n"); + free_model_specs(&model_specs); + return 1; + } + if (!strcmp(cuda_weights, "bf16")) { + pplx_cuda_set_weights_bf16(1); + } else if (!strcmp(cuda_weights, "f32")) { + pplx_cuda_set_weights_bf16(0); + } else { + fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); + free_model_specs(&model_specs); + return 1; + } +#else + fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); + free_model_specs(&model_specs); + return 1; +#endif + } + + if (!use_mlx && !use_cuda) { + if (n_threads <= 0) n_threads = qwen_get_num_cpus(); + qwen_set_threads(n_threads); + if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); + } else if (verbose >= 1) { + fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); + } + + pplx_server_config_t scfg = { + .models = model_specs.v, + .n_models = model_specs.n, + .host = host, + .port = port, + .batch_size = batch_size > 0 ? batch_size : 8, + .max_batch_tokens = max_batch_tokens, + .batch_wait_us = batch_wait_us, + .use_mlx = use_mlx, + .use_cuda = use_cuda, + .mlx_quantize_bits = mlx_quantize_bits, + .mlx_quantize_group_size = mlx_quantize_group_size, + .memory_utilization = memory_utilization, + .enable_cors = cors, + .api_key = api_key, + }; + int rc = pplx_run_server(&scfg); + free_model_specs(&model_specs); + return rc; +} + +#endif /* EMBED_SERVER_TEST */ |
