aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore5
-rw-r--r--embed.c (renamed from pplx_embed.c)0
-rw-r--r--embed.h (renamed from pplx_embed.h)0
-rw-r--r--embed_cli.c (renamed from main.c)0
-rw-r--r--embed_cuda.cu (renamed from pplx_embed_cuda.cu)0
-rw-r--r--embed_cuda.h (renamed from pplx_embed_cuda.h)0
-rw-r--r--embed_distributed.c (renamed from pplx_distributed.c)0
-rw-r--r--embed_distributed.h (renamed from pplx_distributed.h)0
-rw-r--r--embed_internal.h (renamed from pplx_embed_internal.h)0
-rw-r--r--embed_mlx.c (renamed from pplx_embed_mlx.c)0
-rw-r--r--embed_mlx.h (renamed from pplx_embed_mlx.h)0
-rw-r--r--embed_server.c (renamed from pplx_server.c)0
-rw-r--r--embed_server.h (renamed from pplx_server.h)0
-rw-r--r--server_main.c340
14 files changed, 4 insertions, 341 deletions
diff --git a/.gitignore b/.gitignore
index d4750c8..987e7be 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,8 @@
*.o
-pplx_embed
+pplx-embed
+pplx-embed-server
+libpplxembed.*
+tests/test_kernels_*
book/
.venv/
__pycache__/
diff --git a/pplx_embed.c b/embed.c
index 996be9c..996be9c 100644
--- a/pplx_embed.c
+++ b/embed.c
diff --git a/pplx_embed.h b/embed.h
index 5a2501b..5a2501b 100644
--- a/pplx_embed.h
+++ b/embed.h
diff --git a/main.c b/embed_cli.c
index 140494b..140494b 100644
--- a/main.c
+++ b/embed_cli.c
diff --git a/pplx_embed_cuda.cu b/embed_cuda.cu
index a00fc84..a00fc84 100644
--- a/pplx_embed_cuda.cu
+++ b/embed_cuda.cu
diff --git a/pplx_embed_cuda.h b/embed_cuda.h
index badff9c..badff9c 100644
--- a/pplx_embed_cuda.h
+++ b/embed_cuda.h
diff --git a/pplx_distributed.c b/embed_distributed.c
index d4254f7..d4254f7 100644
--- a/pplx_distributed.c
+++ b/embed_distributed.c
diff --git a/pplx_distributed.h b/embed_distributed.h
index cd97147..cd97147 100644
--- a/pplx_distributed.h
+++ b/embed_distributed.h
diff --git a/pplx_embed_internal.h b/embed_internal.h
index 1e7e4d0..1e7e4d0 100644
--- a/pplx_embed_internal.h
+++ b/embed_internal.h
diff --git a/pplx_embed_mlx.c b/embed_mlx.c
index 602571d..602571d 100644
--- a/pplx_embed_mlx.c
+++ b/embed_mlx.c
diff --git a/pplx_embed_mlx.h b/embed_mlx.h
index f2ad136..f2ad136 100644
--- a/pplx_embed_mlx.h
+++ b/embed_mlx.h
diff --git a/pplx_server.c b/embed_server.c
index d831062..d831062 100644
--- a/pplx_server.c
+++ b/embed_server.c
diff --git a/pplx_server.h b/embed_server.h
index c38f0ca..c38f0ca 100644
--- a/pplx_server.h
+++ b/embed_server.h
diff --git a/server_main.c b/server_main.c
deleted file mode 100644
index ace0100..0000000
--- a/server_main.c
+++ /dev/null
@@ -1,340 +0,0 @@
-/* server_main.c - pplx-embed-server: Perplexity-compatible embeddings API */
-
-#include "pplx_embed.h"
-#include "pplx_server.h"
-#include "qwen_kernels.h"
-
-#ifdef USE_CUDA
-#include "pplx_embed_cuda.h"
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-static void print_usage(const char *prog)
-{
- fprintf(stderr,
- "Usage: %s --model ID=DIR [options]\n"
- "\n"
- "Serves POST /v1/embeddings and /v1/contextualizedembeddings.\n"
- "\n"
- "Options:\n"
- " --model ID=DIR Model to serve (repeatable)\n"
- " --backend cpu|mlx|cuda Execution backend (default: cpu); --mlx and\n"
- " --cuda are shorthands\n"
- " --host HOST Bind host (default: 127.0.0.1)\n"
- " --port N Bind port (default: 8000)\n"
- " --api-key K Require Authorization: Bearer K\n"
- " --cors Enable CORS headers\n"
- " -b, --batch-size N Max texts or documents per inference batch\n"
- " (default: 8)\n"
- " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
- " --batch-wait-us N Micro-batch wait in microseconds\n"
- " (default: 1000; 0 drains only queued work)\n"
- " --memory-utilization F Fraction of physical memory the MLX model-set\n"
- " preflight may plan for (default: 0.90; values\n"
- " above 1.0 overcommit)\n"
- " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
- " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
- " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
- " (default: f32, exact)\n"
- " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
- " f32); bf16 halves weight memory\n"
- " -t, --threads N CPU threads (default: all cores)\n"
- " -v, --verbose Verbose (-vv for debug)\n"
- " -h, --help Show this help\n"
- "\n"
- "Examples:\n"
- " %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
- " %s --backend mlx --mlx-quant-bits 8 \\\n"
- " --model pplx-embed-v1-4b=./model-4b-bf16\n",
- prog, prog, prog);
-}
-
-typedef struct {
- pplx_server_model_spec_t *v;
- int n;
- int cap;
-} model_specs_t;
-
-static int append_model_spec(model_specs_t *specs, const char *arg)
-{
- const char *eq = strchr(arg, '=');
- if (!eq || eq == arg || !eq[1]) {
- fprintf(stderr, "--model expects MODEL_ID=DIR\n");
- return -1;
- }
- if (specs->n == specs->cap) {
- int new_cap = specs->cap ? specs->cap * 2 : 4;
- void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
- if (!p) return -1;
- specs->v = p;
- specs->cap = new_cap;
- }
- size_t id_len = (size_t)(eq - arg);
- char *id = (char *)malloc(id_len + 1);
- char *path = strdup(eq + 1);
- if (id) {
- memcpy(id, arg, id_len);
- id[id_len] = '\0';
- }
- specs->v[specs->n].id = id;
- specs->v[specs->n].path = path;
- if (!specs->v[specs->n].id || !specs->v[specs->n].path)
- return -1;
- specs->n++;
- return 0;
-}
-
-static void free_model_specs(model_specs_t *specs)
-{
- if (!specs) return;
- for (int i = 0; i < specs->n; i++) {
- free((char *)specs->v[i].id);
- free((char *)specs->v[i].path);
- }
- free(specs->v);
-}
-
-int main(int argc, char *argv[])
-{
- int n_threads = 0;
- int verbose = 0;
- int use_mlx = 0;
- int use_cuda = 0;
- int batch_size = 0;
- int max_batch_tokens = 0;
- int batch_wait_us = -1;
- int mlx_quantize_bits = 0;
- int mlx_quantize_group_size = 64;
- const char *cuda_fast_gemm = NULL;
- const char *cuda_weights = NULL;
- double memory_utilization = 0.0;
- const char *host = "127.0.0.1";
- const char *api_key = NULL;
- int port = 8000;
- int cors = 0;
- model_specs_t model_specs = {0};
-
- int arg = 1;
- while (arg < argc && argv[arg][0] == '-') {
- const char *f = argv[arg];
- if (!strcmp(f, "--model")) {
- if (append_model_spec(&model_specs, argv[++arg]) != 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--backend")) {
- const char *backend = argv[++arg];
- if (!strcmp(backend, "mlx")) {
-#ifdef USE_MLX
- use_mlx = 1;
- use_cuda = 0;
-#else
- fprintf(stderr, "mlx backend not available (build with: make mlx)\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
- } else if (!strcmp(backend, "cuda")) {
-#ifdef USE_CUDA
- use_cuda = 1;
- use_mlx = 0;
-#else
- fprintf(stderr, "cuda backend not available (build with: make cuda)\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
- } else if (!strcmp(backend, "cpu")) {
- use_mlx = 0;
- use_cuda = 0;
- } else {
- fprintf(stderr, "invalid --backend: %s\n", backend);
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--mlx")) {
-#ifdef USE_MLX
- use_mlx = 1;
- use_cuda = 0;
-#else
- fprintf(stderr, "--mlx not available (build with: make mlx)\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
- }
- else if (!strcmp(f, "--cuda")) {
-#ifdef USE_CUDA
- use_cuda = 1;
- use_mlx = 0;
-#else
- fprintf(stderr, "--cuda not available (build with: make cuda)\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
- }
- else if (!strcmp(f, "--host")) { host = argv[++arg]; }
- else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); }
- else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; }
- else if (!strcmp(f, "--cors")) { cors = 1; }
- else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
- batch_size = atoi(argv[++arg]);
- }
- else if (!strcmp(f, "--max-batch-tokens")) {
- max_batch_tokens = atoi(argv[++arg]);
- if (max_batch_tokens <= 0) {
- fprintf(stderr, "--max-batch-tokens must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--batch-wait-us")) {
- batch_wait_us = atoi(argv[++arg]);
- if (batch_wait_us < 0) {
- fprintf(stderr, "--batch-wait-us must be >= 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--memory-utilization")) {
- memory_utilization = atof(argv[++arg]);
- if (memory_utilization <= 0.0) {
- fprintf(stderr, "--memory-utilization must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--mlx-quant-bits")) {
- mlx_quantize_bits = atoi(argv[++arg]);
- if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--mlx-quant-group-size")) {
- mlx_quantize_group_size = atoi(argv[++arg]);
- if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--cuda-gemm-mode")) {
- cuda_fast_gemm = argv[++arg];
- }
- else if (!strcmp(f, "--cuda-weight-dtype")) {
- cuda_weights = argv[++arg];
- }
- else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
- n_threads = atoi(argv[++arg]);
- }
- else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
- else if (!strcmp(f, "-vv")) { verbose = 2; }
- else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
- print_usage(argv[0]);
- free_model_specs(&model_specs);
- return 0;
- }
- else {
- fprintf(stderr, "unknown option: %s\n", f);
- print_usage(argv[0]);
- free_model_specs(&model_specs);
- return 1;
- }
- arg++;
- }
-
- pplx_verbose = verbose;
- qwen_verbose = verbose;
-
- if (arg < argc) {
- fprintf(stderr, "unexpected argument: %s\n", argv[arg]);
- print_usage(argv[0]);
- free_model_specs(&model_specs);
- return 1;
- }
- if (model_specs.n == 0) {
- fprintf(stderr, "at least one --model MODEL_ID=DIR is required\n");
- print_usage(argv[0]);
- free_model_specs(&model_specs);
- return 1;
- }
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quant-bits requires --backend mlx\n");
- free_model_specs(&model_specs);
- return 1;
- }
-
- if (cuda_fast_gemm) {
-#ifdef USE_CUDA
- if (!use_cuda) {
- fprintf(stderr, "--cuda-gemm-mode requires --backend cuda\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
- fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
- free_model_specs(&model_specs);
- return 1;
- }
-#else
- fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
- }
-
- if (cuda_weights) {
-#ifdef USE_CUDA
- if (!use_cuda) {
- fprintf(stderr, "--cuda-weight-dtype requires --backend cuda\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (!strcmp(cuda_weights, "bf16")) {
- pplx_cuda_set_weights_bf16(1);
- } else if (!strcmp(cuda_weights, "f32")) {
- pplx_cuda_set_weights_bf16(0);
- } else {
- fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
- free_model_specs(&model_specs);
- return 1;
- }
-#else
- fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
- }
-
- if (!use_mlx && !use_cuda) {
- if (n_threads <= 0) n_threads = qwen_get_num_cpus();
- qwen_set_threads(n_threads);
- if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- } else if (verbose >= 1) {
- fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
- }
-
- pplx_server_config_t scfg = {
- .models = model_specs.v,
- .n_models = model_specs.n,
- .host = host,
- .port = port,
- .batch_size = batch_size > 0 ? batch_size : 8,
- .max_batch_tokens = max_batch_tokens,
- .batch_wait_us = batch_wait_us,
- .use_mlx = use_mlx,
- .use_cuda = use_cuda,
- .mlx_quantize_bits = mlx_quantize_bits,
- .mlx_quantize_group_size = mlx_quantize_group_size,
- .memory_utilization = memory_utilization,
- .enable_cors = cors,
- .api_key = api_key,
- };
- int rc = pplx_run_server(&scfg);
- free_model_specs(&model_specs);
- return rc;
-}