diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 07:30:34 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | c14151290df20e1a2ebc9fb06a77b075ca4ad959 (patch) | |
| tree | d4b53dd131f0e5f5f8f5dd60727cc13a76c42534 /main.c | |
| parent | dcc54e7a267f88ed456209c63f58c7130274e8da (diff) | |
| download | ffwd-c14151290df20e1a2ebc9fb06a77b075ca4ad959.tar.gz ffwd-c14151290df20e1a2ebc9fb06a77b075ca4ad959.zip | |
cli: rename flags, add --stream, split serving into pplx-embed-server
Diffstat (limited to 'main.c')
| -rw-r--r-- | main.c | 245 |
1 files changed, 42 insertions, 203 deletions
@@ -2,7 +2,6 @@ #include "pplx_embed.h" #include "pplx_distributed.h" -#include "pplx_server.h" #include "qwen_asr_kernels.h" #include "qwen_asr_tokenizer.h" @@ -35,59 +34,50 @@ static void print_usage(const char *prog) " -d <dir> Model directory (required)\n" " --mlx Use Apple MLX GPU backend\n" " --cuda Use CUDA/cuBLAS GPU backend\n" - " --stdin Read lines from stdin, write JSON embeddings to stdout\n" - " --serve Serve the Perplexity-compatible HTTP API\n" + " --stream Read lines from stdin, write one JSON embedding per line\n" " --dist-worker\n" " Serve one final distributed layer shard\n" " --dist-remote HOST:PORT\n" - " Use one remote final shard for CLI or --stdin inference\n" + " Use one remote final shard for CLI or --stream inference\n" " --layers A:B Distributed half-open layer range, for example 0:14\n" " --dist-activation-bits N\n" " Distributed activation transport width: 32 or 16 (default: 32)\n" - " --model ID=DIR\n" - " Load model ID from directory for --serve (repeatable)\n" - " --backend cpu|mlx|cuda\n" - " Backend for --serve (default: cpu)\n" - " --mlx-quantize-bits N\n" + " --mlx-quant-bits N\n" " Quantize MLX linear weights to 8 bits at load time\n" - " --mlx-quantize-group-size N\n" + " --mlx-quant-group-size N\n" " MLX quantization group size (default: 64)\n" - " --cuda-fast-gemm MODE\n" + " --cuda-gemm-mode MODE\n" " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" - " --cuda-weights DTYPE\n" + " --cuda-weight-dtype DTYPE\n" " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" " weight memory and uses BF16 tensor cores\n" - " --allow-memory-overcommit\n" - " Allow an MLX server model set above the host-memory safety budget\n" - " --host HOST Server bind host (default: 127.0.0.1)\n" - " --port N Server bind port (default: 8000)\n" - " --cors Enable CORS headers in --serve mode\n" - " --api-key K Require Authorization: Bearer K in --serve mode\n" - " -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n" - " --max-batch-tokens N\n" - " Server max tokens per standard inference batch (default: 16384)\n" - " --batch-wait-us N\n" - " Server micro-batch wait in microseconds (default: 1000; 0 disables)\n" - " -t <n> CPU threads (default: all cores)\n" - " -e Print raw embeddings (with multiple texts)\n" - " -v Verbose (-vv for debug)\n" + " --host HOST Bind host for --dist-worker (default: 127.0.0.1)\n" + " --port N Bind port for --dist-worker (default: 8000)\n" + " -b, --batch-size N\n" + " Max texts per engine batch (default: all; --stream: 1)\n" + " -t, --threads N\n" + " CPU threads (default: all cores)\n" + " -e, --embeddings\n" + " Print raw embeddings (with multiple texts)\n" + " -v, --verbose\n" + " Verbose (-vv for debug)\n" " -h Show this help\n" "\n" "Modes:\n" " 1 text arg Print embedding as space-separated floats\n" " 2+ text args Print cosine similarity matrix\n" " no args Batch: read all stdin lines, then similarity matrix\n" - " --stdin Streaming: read stdin lines, write JSON per line\n" - " --serve HTTP server for /v1/embeddings and /v1/contextualizedembeddings\n" + " --stream Streaming: read stdin lines, write JSON per line\n" " --dist-worker Final layer-shard worker\n" "\n" + "The HTTP API is served by the separate pplx-embed-server binary.\n" + "\n" "Examples:\n" " %s -d ./model \"query: what is AI?\"\n" - " %s -d ./model --mlx --stdin < texts.txt\n" - " %s --serve --model pplx-embed-v1-0.6b=./model --port 8000\n" + " %s -d ./model --mlx --stream < texts.txt\n" " %s -d ./model --dist-worker --layers 14:28 --port 9000\n" " %s -d ./model --dist-remote 127.0.0.1:9000 --layers 0:14 \"text\"\n", - prog, prog, prog, prog, prog, prog); + prog, prog, prog, prog, prog); } /* ======================================================================== @@ -114,12 +104,6 @@ typedef struct { int n_tokens; } token_buf_t; -typedef struct { - pplx_server_model_spec_t *v; - int n; - int cap; -} model_specs_t; - static int engine_embed_batch(engine_t *e, const pplx_input_t *inputs, int batch, float *out_embeddings) { @@ -192,45 +176,6 @@ static void free_tokens(token_buf_t *tokens, int n) free(tokens[i].ids); } -static int append_model_spec(model_specs_t *specs, const char *arg) -{ - const char *eq = strchr(arg, '='); - if (!eq || eq == arg || !eq[1]) { - fprintf(stderr, "--model expects MODEL_ID=DIR\n"); - return -1; - } - if (specs->n == specs->cap) { - int new_cap = specs->cap ? specs->cap * 2 : 4; - void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); - if (!p) return -1; - specs->v = p; - specs->cap = new_cap; - } - size_t id_len = (size_t)(eq - arg); - char *id = (char *)malloc(id_len + 1); - char *path = strdup(eq + 1); - if (id) { - memcpy(id, arg, id_len); - id[id_len] = '\0'; - } - specs->v[specs->n].id = id; - specs->v[specs->n].path = path; - if (!specs->v[specs->n].id || !specs->v[specs->n].path) - return -1; - specs->n++; - return 0; -} - -static void free_model_specs(model_specs_t *specs) -{ - if (!specs) return; - for (int i = 0; i < specs->n; i++) { - free((char *)specs->v[i].id); - free((char *)specs->v[i].path); - } - free(specs->v); -} - /* ======================================================================== * Output helpers * ======================================================================== */ @@ -531,56 +476,29 @@ int main(int argc, char *argv[]) int use_mlx = 0; int use_cuda = 0; int stdin_mode = 0; - int serve_mode = 0; int dist_worker_mode = 0; - int allow_memory_overcommit = 0; const char *dist_remote = NULL; int layer_start = -1; int layer_end = -1; int batch_size = 0; - int max_batch_tokens = 0; - int batch_wait_us = -1; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; const char *cuda_fast_gemm = NULL; const char *cuda_weights = NULL; int dist_activation_bits = 32; const char *host = "127.0.0.1"; - const char *api_key = NULL; int port = 8000; - int cors = 0; - model_specs_t model_specs = {0}; int arg_start = 1; while (arg_start < argc && argv[arg_start][0] == '-') { const char *f = argv[arg_start]; if (!strcmp(f, "-d")) { model_dir = argv[++arg_start]; } - else if (!strcmp(f, "-t")) { n_threads = atoi(argv[++arg_start]); } + else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + n_threads = atoi(argv[++arg_start]); + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg_start]); } - else if (!strcmp(f, "--batch-wait-us")) { - batch_wait_us = atoi(argv[++arg_start]); - if (batch_wait_us < 0) { - fprintf(stderr, "--batch-wait-us must be >= 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--max-batch-tokens")) { - max_batch_tokens = atoi(argv[++arg_start]); - if (max_batch_tokens <= 0) { - fprintf(stderr, "--max-batch-tokens must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--model")) { - if (append_model_spec(&model_specs, argv[++arg_start]) != 0) { - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--backend")) { const char *backend = argv[++arg_start]; if (!strcmp(backend, "mlx")) { @@ -589,7 +507,6 @@ int main(int argc, char *argv[]) use_cuda = 0; #else fprintf(stderr, "mlx backend not available (build with: make mlx)\n"); - free_model_specs(&model_specs); return 1; #endif } else if (!strcmp(backend, "cuda")) { @@ -598,7 +515,6 @@ int main(int argc, char *argv[]) use_mlx = 0; #else fprintf(stderr, "cuda backend not available (build with: make cuda)\n"); - free_model_specs(&model_specs); return 1; #endif } else if (!strcmp(backend, "cpu")) { @@ -606,41 +522,34 @@ int main(int argc, char *argv[]) use_cuda = 0; } else { fprintf(stderr, "invalid --backend: %s\n", backend); - free_model_specs(&model_specs); return 1; } } - else if (!strcmp(f, "--mlx-quantize-bits")) { + else if (!strcmp(f, "--mlx-quant-bits")) { mlx_quantize_bits = atoi(argv[++arg_start]); if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quantize-bits must be 0 or 8\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); return 1; } } - else if (!strcmp(f, "--mlx-quantize-group-size")) { + else if (!strcmp(f, "--mlx-quant-group-size")) { mlx_quantize_group_size = atoi(argv[++arg_start]); if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quantize-group-size must be > 0\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); return 1; } } - else if (!strcmp(f, "--cuda-fast-gemm")) { + else if (!strcmp(f, "--cuda-gemm-mode")) { cuda_fast_gemm = argv[++arg_start]; } - else if (!strcmp(f, "--cuda-weights")) { + else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg_start]; } else if (!strcmp(f, "--host")) { host = argv[++arg_start]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg_start]); } - else if (!strcmp(f, "--allow-memory-overcommit")) { - allow_memory_overcommit = 1; - } else if (!strcmp(f, "--layers")) { if (parse_layers(argv[++arg_start], &layer_start, &layer_end) != 0) { fprintf(stderr, "--layers expects START:END with START < END\n"); - free_model_specs(&model_specs); return 1; } } @@ -648,17 +557,15 @@ int main(int argc, char *argv[]) dist_activation_bits = atoi(argv[++arg_start]); if (dist_activation_bits != 32 && dist_activation_bits != 16) { fprintf(stderr, "--dist-activation-bits must be 32 or 16\n"); - free_model_specs(&model_specs); return 1; } } - else if (!strcmp(f, "--cors")) { cors = 1; } - else if (!strcmp(f, "--api-key")){ api_key = argv[++arg_start]; } - else if (!strcmp(f, "-e")) { print_embs = 1; } - else if (!strcmp(f, "-v")) { verbose++; } + else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { + print_embs = 1; + } + else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } - else if (!strcmp(f, "--stdin")) { stdin_mode = 1; } - else if (!strcmp(f, "--serve")) { serve_mode = 1; } + else if (!strcmp(f, "--stream")) { stdin_mode = 1; } else if (!strcmp(f, "--dist-worker")) { dist_worker_mode = 1; } else if (!strcmp(f, "--dist-remote")) { dist_remote = argv[++arg_start]; } else if (!strcmp(f, "--mlx")) { @@ -667,7 +574,6 @@ int main(int argc, char *argv[]) use_cuda = 0; #else fprintf(stderr, "--mlx not available (build with: make mlx)\n"); - free_model_specs(&model_specs); return 1; #endif } @@ -677,13 +583,11 @@ int main(int argc, char *argv[]) use_mlx = 0; #else fprintf(stderr, "--cuda not available (build with: make cuda)\n"); - free_model_specs(&model_specs); return 1; #endif } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(argv[0]); - free_model_specs(&model_specs); return 0; } else break; @@ -696,18 +600,15 @@ int main(int argc, char *argv[]) if (cuda_fast_gemm) { #ifdef USE_CUDA if (!use_cuda) { - fprintf(stderr, "--cuda-fast-gemm requires --cuda or --backend cuda\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-gemm-mode requires --cuda or --backend cuda\n"); return 1; } if (pplx_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { - fprintf(stderr, "--cuda-fast-gemm must be f32, tf32, bf16, or 16f\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); return 1; } #else - fprintf(stderr, "--cuda-fast-gemm requires a CUDA build\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); return 1; #endif } @@ -715,8 +616,7 @@ int main(int argc, char *argv[]) if (cuda_weights) { #ifdef USE_CUDA if (!use_cuda) { - fprintf(stderr, "--cuda-weights requires --cuda or --backend cuda\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-weight-dtype requires --cuda or --backend cuda\n"); return 1; } if (!strcmp(cuda_weights, "bf16")) { @@ -724,77 +624,18 @@ int main(int argc, char *argv[]) } else if (!strcmp(cuda_weights, "f32")) { pplx_cuda_set_weights_bf16(0); } else { - fprintf(stderr, "--cuda-weights must be f32 or bf16\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); return 1; } #else - fprintf(stderr, "--cuda-weights requires a CUDA build\n"); - free_model_specs(&model_specs); + fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); return 1; #endif } - if (serve_mode) { - if (stdin_mode || dist_worker_mode || dist_remote) { - fprintf(stderr, "--serve cannot be combined with --stdin or distributed CLI modes\n"); - free_model_specs(&model_specs); - return 1; - } - if (model_specs.n == 0) { - fprintf(stderr, "--serve requires at least one --model MODEL_ID=DIR\n"); - print_usage(argv[0]); - free_model_specs(&model_specs); - return 1; - } - if (dist_activation_bits != 32) { - fprintf(stderr, "--dist-activation-bits is only valid with --dist-remote\n"); - free_model_specs(&model_specs); - return 1; - } - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quantize-bits requires --backend mlx\n"); - free_model_specs(&model_specs); - return 1; - } - if (!use_mlx && !use_cuda) { - if (n_threads <= 0) n_threads = qwen_get_num_cpus(); - qwen_set_threads(n_threads); - if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } else if (verbose >= 1) { - fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); - } - pplx_server_config_t scfg = { - .models = model_specs.v, - .n_models = model_specs.n, - .host = host, - .port = port, - .batch_size = batch_size > 0 ? batch_size : 8, - .max_batch_tokens = max_batch_tokens, - .batch_wait_us = batch_wait_us, - .use_mlx = use_mlx, - .use_cuda = use_cuda, - .mlx_quantize_bits = mlx_quantize_bits, - .mlx_quantize_group_size = mlx_quantize_group_size, - .allow_memory_overcommit = allow_memory_overcommit, - .enable_cors = cors, - .api_key = api_key, - }; - int rc = pplx_run_server(&scfg); - free_model_specs(&model_specs); - return rc; - } - - if (model_specs.n != 0) { - fprintf(stderr, "--model is only valid with --serve\n"); - free_model_specs(&model_specs); - return 1; - } - if (!model_dir || !model_dir[0]) { fprintf(stderr, "model directory required (-d <dir>)\n"); print_usage(argv[0]); - free_model_specs(&model_specs); return 1; } @@ -831,7 +672,7 @@ int main(int argc, char *argv[]) } if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quantize-bits requires --mlx or --backend mlx\n"); + fprintf(stderr, "--mlx-quant-bits requires --mlx or --backend mlx\n"); return 1; } @@ -958,7 +799,6 @@ int main(int argc, char *argv[]) if (e.cuda_ctx) pplx_cuda_free(e.cuda_ctx); #endif qwen_tokenizer_free(tok); - free_model_specs(&model_specs); return 1; } @@ -981,6 +821,5 @@ int main(int argc, char *argv[]) #endif qwen_tokenizer_workspace_free(e.tok_ws); qwen_tokenizer_free(tok); - free_model_specs(&model_specs); return rc; } |
