aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-14 10:22:47 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 10:22:47 +0200
commita33e5395ae1bc968c8fa007963b36327de582a99 (patch)
tree4c2def99e2fbb484ea77e6f56f67b9652ece230f
parent042cb0ece4370adee9d967acf4ab487054e3325e (diff)
downloadffwd-a33e5395ae1bc968c8fa007963b36327de582a99.tar.gz
ffwd-a33e5395ae1bc968c8fa007963b36327de582a99.zip
cli: scope GPU options to their respective build targets
-rw-r--r--embed_cli.c90
-rw-r--r--embed_server.c145
-rw-r--r--tests/test_cli.c17
3 files changed, 130 insertions, 122 deletions
diff --git a/embed_cli.c b/embed_cli.c
index 1d62fc2..ebdb223 100644
--- a/embed_cli.c
+++ b/embed_cli.c
@@ -31,18 +31,26 @@ static void print_usage(const char *prog)
"\n"
"Options:\n"
" -d <dir> Model directory (required)\n"
+#ifdef USE_MLX
" --mlx Use Apple MLX GPU backend\n"
+#endif
+#ifdef USE_CUDA
" --cuda Use CUDA/cuBLAS GPU backend\n"
+#endif
" --stream Read lines from stdin, write one JSON embedding per line\n"
+#ifdef USE_MLX
" --mlx-quant-bits N\n"
" Quantize MLX linear weights to 8 bits at load time\n"
" --mlx-quant-group-size N\n"
" MLX quantization group size (default: 64)\n"
+#endif
+#ifdef USE_CUDA
" --cuda-gemm-mode MODE\n"
" CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
" --cuda-weight-dtype DTYPE\n"
" CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
" weight memory and uses BF16 tensor cores\n"
+#endif
" -b, --batch-size N\n"
" Max texts per engine batch (default: all; --stream: 1)\n"
" -t, --threads N\n"
@@ -61,8 +69,13 @@ static void print_usage(const char *prog)
"\n"
"Examples:\n"
" %s -d ./model \"query: what is AI?\"\n"
+#ifdef USE_MLX
" %s -d ./model --mlx --stream < texts.txt\n",
prog, prog, prog);
+#else
+ ,
+ prog, prog);
+#endif
}
/* ========================================================================
@@ -430,14 +443,18 @@ int main(int argc, char *argv[])
int n_threads = 0;
int print_embs = 0;
int verbose = 0;
- int use_mlx = 0;
- int use_cuda = 0;
int stdin_mode = 0;
int batch_size = 0;
+#ifdef USE_MLX
+ int use_mlx = 0;
int mlx_quantize_bits = 0;
int mlx_quantize_group_size = 64;
+#endif
+#ifdef USE_CUDA
+ int use_cuda = 0;
const char *cuda_fast_gemm = NULL;
const char *cuda_weights = NULL;
+#endif
int arg_start = 1;
while (arg_start < argc && argv[arg_start][0] == '-') {
@@ -449,6 +466,7 @@ int main(int argc, char *argv[])
else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg_start]);
}
+#ifdef USE_MLX
else if (!strcmp(f, "--mlx-quant-bits")) {
mlx_quantize_bits = atoi(argv[++arg_start]);
if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
@@ -463,36 +481,27 @@ int main(int argc, char *argv[])
return 1;
}
}
+#endif
+#ifdef USE_CUDA
else if (!strcmp(f, "--cuda-gemm-mode")) {
cuda_fast_gemm = argv[++arg_start];
}
else if (!strcmp(f, "--cuda-weight-dtype")) {
cuda_weights = argv[++arg_start];
}
+#endif
else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
print_embs = 1;
}
else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { verbose++; }
else if (!strcmp(f, "-vv")) { verbose = 2; }
else if (!strcmp(f, "--stream")) { stdin_mode = 1; }
- else if (!strcmp(f, "--mlx")) {
#ifdef USE_MLX
- use_mlx = 1;
- use_cuda = 0;
-#else
- fprintf(stderr, "--mlx not available (build with: make mlx)\n");
- return 1;
+ else if (!strcmp(f, "--mlx")) { use_mlx = 1; }
#endif
- }
- else if (!strcmp(f, "--cuda")) {
#ifdef USE_CUDA
- use_cuda = 1;
- use_mlx = 0;
-#else
- fprintf(stderr, "--cuda not available (build with: make cuda)\n");
- return 1;
+ else if (!strcmp(f, "--cuda")) { use_cuda = 1; }
#endif
- }
else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
print_usage(argv[0]);
return 0;
@@ -504,8 +513,8 @@ int main(int argc, char *argv[])
embed_verbose = verbose;
qwen_verbose = verbose;
- if (cuda_fast_gemm) {
#ifdef USE_CUDA
+ if (cuda_fast_gemm) {
if (!use_cuda) {
fprintf(stderr, "--cuda-gemm-mode requires --cuda\n");
return 1;
@@ -514,14 +523,8 @@ int main(int argc, char *argv[])
fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
return 1;
}
-#else
- fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
- return 1;
-#endif
}
-
if (cuda_weights) {
-#ifdef USE_CUDA
if (!use_cuda) {
fprintf(stderr, "--cuda-weight-dtype requires --cuda\n");
return 1;
@@ -534,11 +537,14 @@ int main(int argc, char *argv[])
fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
return 1;
}
-#else
- fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
- return 1;
+ }
#endif
+#ifdef USE_MLX
+ if (mlx_quantize_bits && !use_mlx) {
+ fprintf(stderr, "--mlx-quant-bits requires --mlx\n");
+ return 1;
}
+#endif
if (!model_dir || !model_dir[0]) {
fprintf(stderr, "model directory required (-d <dir>)\n");
@@ -546,19 +552,20 @@ int main(int argc, char *argv[])
return 1;
}
- /* Threads (CPU backend) */
- if (!use_mlx && !use_cuda) {
+#ifdef USE_MLX
+ if (use_mlx) {
+ if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ } else
+#endif
+#ifdef USE_CUDA
+ if (use_cuda) {
+ if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n");
+ } else
+#endif
+ {
if (n_threads <= 0) n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- } else {
- if (verbose >= 1)
- fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
- }
-
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quant-bits requires --mlx\n");
- return 1;
}
/* Tokenizer */
@@ -617,10 +624,15 @@ int main(int argc, char *argv[])
e.dim = embed_model_config(e.model)->hidden_size;
}
if (verbose >= 1)
- fprintf(stderr, "Model: %d-dim, %.0f ms%s%s\n",
+ fprintf(stderr, "Model: %d-dim, %.0f ms%s\n",
e.dim, now_ms() - t0,
- use_mlx ? " (MLX)" : "",
- use_cuda ? " (CUDA)" : "");
+#ifdef USE_MLX
+ use_mlx ? " (MLX)" :
+#endif
+#ifdef USE_CUDA
+ use_cuda ? " (CUDA)" :
+#endif
+ "");
e.tok_ws = qwen_tokenizer_workspace_new();
if (!e.tok_ws) {
diff --git a/embed_server.c b/embed_server.c
index b34cecd..d494dc7 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -3046,8 +3046,12 @@ static void print_usage(const char *prog)
"\n"
"Options:\n"
" --model ID=DIR Model to serve (repeatable)\n"
+#ifdef USE_MLX
" --mlx Use Apple MLX GPU backend\n"
+#endif
+#ifdef USE_CUDA
" --cuda Use NVIDIA CUDA GPU backend\n"
+#endif
" --host HOST Bind host (default: 127.0.0.1)\n"
" --port N Bind port (default: 8000)\n"
" --api-key K Require Authorization: Bearer K\n"
@@ -3055,25 +3059,38 @@ static void print_usage(const char *prog)
" (default: 32)\n"
" --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
" --batch-wait-us N First-arrival micro-batch deadline in us\n"
- " (default: CUDA 1000; CPU/MLX 0)\n"
+#ifdef USE_CUDA
+ " (default: CUDA 1000; CPU 0)\n"
+#else
+ " (default: 0)\n"
+#endif
+#ifdef USE_MLX
" --memory-utilization F Fraction of physical memory the MLX model-set\n"
" preflight may plan for (default: 0.90; values\n"
" above 1.0 overcommit)\n"
" --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
" --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
+#endif
+#ifdef USE_CUDA
" --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
" (default: f32, exact)\n"
" --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
" f32); bf16 halves weight memory\n"
+#endif
" -t, --threads N CPU threads (default: all cores)\n"
" -v, --verbose Verbose (-vv for debug)\n"
" -h, --help Show this help\n"
"\n"
"Examples:\n"
" %s --model pplx-embed-v1-0.6b=./model --port 8000\n"
+#ifdef USE_MLX
" %s --mlx --mlx-quant-bits 8 \\\n"
" --model pplx-embed-v1-4b=./model-4b-bf16\n",
prog, prog, prog);
+#else
+ ,
+ prog, prog);
+#endif
}
typedef struct {
@@ -3125,16 +3142,20 @@ int main(int argc, char *argv[])
{
int n_threads = 0;
int verbose = 0;
- int use_mlx = 0;
- int use_cuda = 0;
int batch_size = 0;
int max_batch_tokens = 0;
int batch_wait_us = -1;
+#ifdef USE_MLX
+ int use_mlx = 0;
int mlx_quantize_bits = 0;
int mlx_quantize_group_size = 64;
+ double memory_utilization = 0.0;
+#endif
+#ifdef USE_CUDA
+ int use_cuda = 0;
const char *cuda_fast_gemm = NULL;
const char *cuda_weights = NULL;
- double memory_utilization = 0.0;
+#endif
const char *host = "127.0.0.1";
const char *api_key = NULL;
int port = 8000;
@@ -3149,26 +3170,42 @@ int main(int argc, char *argv[])
return 1;
}
}
- else if (!strcmp(f, "--mlx")) {
#ifdef USE_MLX
- use_mlx = 1;
- use_cuda = 0;
-#else
- fprintf(stderr, "--mlx not available (build with: make mlx)\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
+ else if (!strcmp(f, "--mlx")) { use_mlx = 1; }
+ else if (!strcmp(f, "--mlx-quant-bits")) {
+ mlx_quantize_bits = atoi(argv[++arg]);
+ if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
+ fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--mlx-quant-group-size")) {
+ mlx_quantize_group_size = atoi(argv[++arg]);
+ if (mlx_quantize_group_size <= 0) {
+ fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
+ else if (!strcmp(f, "--memory-utilization")) {
+ memory_utilization = atof(argv[++arg]);
+ if (memory_utilization <= 0.0) {
+ fprintf(stderr, "--memory-utilization must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
}
- else if (!strcmp(f, "--cuda")) {
-#ifdef USE_CUDA
- use_cuda = 1;
- use_mlx = 0;
-#else
- fprintf(stderr, "--cuda not available (build with: make cuda)\n");
- free_model_specs(&model_specs);
- return 1;
#endif
+#ifdef USE_CUDA
+ else if (!strcmp(f, "--cuda")) { use_cuda = 1; }
+ else if (!strcmp(f, "--cuda-gemm-mode")) {
+ cuda_fast_gemm = argv[++arg];
+ }
+ else if (!strcmp(f, "--cuda-weight-dtype")) {
+ cuda_weights = argv[++arg];
}
+#endif
else if (!strcmp(f, "--host")) { host = argv[++arg]; }
else if (!strcmp(f, "--port")) { port = atoi(argv[++arg]); }
else if (!strcmp(f, "--api-key")) { api_key = argv[++arg]; }
@@ -3191,36 +3228,6 @@ int main(int argc, char *argv[])
return 1;
}
}
- else if (!strcmp(f, "--memory-utilization")) {
- memory_utilization = atof(argv[++arg]);
- if (memory_utilization <= 0.0) {
- fprintf(stderr, "--memory-utilization must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--mlx-quant-bits")) {
- mlx_quantize_bits = atoi(argv[++arg]);
- if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--mlx-quant-group-size")) {
- mlx_quantize_group_size = atoi(argv[++arg]);
- if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- else if (!strcmp(f, "--cuda-gemm-mode")) {
- cuda_fast_gemm = argv[++arg];
- }
- else if (!strcmp(f, "--cuda-weight-dtype")) {
- cuda_weights = argv[++arg];
- }
else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
n_threads = atoi(argv[++arg]);
}
@@ -3255,14 +3262,15 @@ int main(int argc, char *argv[])
free_model_specs(&model_specs);
return 1;
}
+#ifdef USE_MLX
if (mlx_quantize_bits && !use_mlx) {
fprintf(stderr, "--mlx-quant-bits requires --mlx\n");
free_model_specs(&model_specs);
return 1;
}
-
- if (cuda_fast_gemm) {
+#endif
#ifdef USE_CUDA
+ if (cuda_fast_gemm) {
if (!use_cuda) {
fprintf(stderr, "--cuda-gemm-mode requires --cuda\n");
free_model_specs(&model_specs);
@@ -3273,15 +3281,8 @@ int main(int argc, char *argv[])
free_model_specs(&model_specs);
return 1;
}
-#else
- fprintf(stderr, "--cuda-gemm-mode requires a CUDA build\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
}
-
if (cuda_weights) {
-#ifdef USE_CUDA
if (!use_cuda) {
fprintf(stderr, "--cuda-weight-dtype requires --cuda\n");
free_model_specs(&model_specs);
@@ -3296,19 +3297,23 @@ int main(int argc, char *argv[])
free_model_specs(&model_specs);
return 1;
}
-#else
- fprintf(stderr, "--cuda-weight-dtype requires a CUDA build\n");
- free_model_specs(&model_specs);
- return 1;
-#endif
}
+#endif
- if (!use_mlx && !use_cuda) {
+#ifdef USE_MLX
+ if (use_mlx) {
+ if (verbose >= 1) fprintf(stderr, "Using MLX GPU backend\n");
+ } else
+#endif
+#ifdef USE_CUDA
+ if (use_cuda) {
+ if (verbose >= 1) fprintf(stderr, "Using CUDA GPU backend\n");
+ } else
+#endif
+ {
if (n_threads <= 0) n_threads = qwen_get_num_cpus();
qwen_set_threads(n_threads);
if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- } else if (verbose >= 1) {
- fprintf(stderr, "Using %s GPU backend\n", use_mlx ? "MLX" : "CUDA");
}
embed_server_config_t scfg = {
@@ -3319,11 +3324,15 @@ int main(int argc, char *argv[])
.batch_size = batch_size > 0 ? batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE,
.max_batch_tokens = max_batch_tokens,
.batch_wait_us = batch_wait_us,
+#ifdef USE_MLX
.use_mlx = use_mlx,
- .use_cuda = use_cuda,
.mlx_quantize_bits = mlx_quantize_bits,
.mlx_quantize_group_size = mlx_quantize_group_size,
.memory_utilization = memory_utilization,
+#endif
+#ifdef USE_CUDA
+ .use_cuda = use_cuda,
+#endif
.api_key = api_key,
};
int rc = embed_run_server(&scfg);
diff --git a/tests/test_cli.c b/tests/test_cli.c
index c5151e9..76a3f14 100644
--- a/tests/test_cli.c
+++ b/tests/test_cli.c
@@ -157,26 +157,13 @@ int main(int argc, char **argv)
"--stream -b 2 exits 0");
expect(count_lines(g_out) == 3, "--stream -b 2 covers the tail batch");
- /* Flag validation: every bad invocation must exit 1.
- * This is a CPU-only build, so the GPU flags are rejected too. */
+ /* Flag validation: missing model dir must exit 1. */
static const struct { const char *args; const char *what; } bad[] = {
- {"--backend foo", "invalid --backend"},
- {"--backend mlx", "--backend mlx without MLX build"},
- {"--backend cuda", "--backend cuda without CUDA build"},
- {"--mlx", "--mlx without MLX build"},
- {"--cuda", "--cuda without CUDA build"},
- {"--mlx-quant-bits 4", "--mlx-quant-bits 4"},
- {"--mlx-quant-group-size 0", "--mlx-quant-group-size 0"},
- {"--cuda-gemm-mode f32 -d X t", "--cuda-gemm-mode without CUDA"},
- {"--cuda-weight-dtype bf16 -d X t", "--cuda-weight-dtype without CUDA"},
+ {"-d '' t", "empty model dir"},
};
for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++)
expect(run_cli(bad[i].args, NULL) == 1, bad[i].what);
- /* Validation that runs after the model dir is accepted. */
- snprintf(args, sizeof(args), "-d '%s' --mlx-quant-bits 8 t", g_dir);
- expect(run_cli(args, NULL) == 1, "--mlx-quant-bits without --mlx");
-
if (g_failures) {
fprintf(stderr, "%d CLI check(s) failed\n", g_failures);
return 1;