aboutsummaryrefslogtreecommitdiff
path: root/embed_server.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-13 01:34:18 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:10 +0200
commit13c56b2218724ebcd2ef97bc386bc60930cf1ff0 (patch)
treed090bbe94bd5fb7156fff65efcd139f3bacfba96 /embed_server.c
parentce352727a6f56ae19038a95b4f8f533c487fac4b (diff)
downloadffwd-13c56b2218724ebcd2ef97bc386bc60930cf1ff0.tar.gz
ffwd-13c56b2218724ebcd2ef97bc386bc60930cf1ff0.zip
server: tune admission deadline by backend
Diffstat (limited to 'embed_server.c')
-rw-r--r--embed_server.c7
1 files changed, 5 insertions, 2 deletions
diff --git a/embed_server.c b/embed_server.c
index 793d764..d4f19f1 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -66,6 +66,7 @@
#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
#define EMBED_SERVER_BATCH_WAIT_US 0
+#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
#define EMBED_MLX_RESIDENT_MULTIPLIER 2
@@ -2358,7 +2359,9 @@ int embed_run_server(const embed_server_config_t *cfg) {
s.max_batch_tokens = cfg->max_batch_tokens > 0
? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS;
s.batch_wait_us = cfg->batch_wait_us >= 0
- ? cfg->batch_wait_us : EMBED_SERVER_BATCH_WAIT_US;
+ ? cfg->batch_wait_us
+ : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US
+ : EMBED_SERVER_BATCH_WAIT_US);
s.enable_cors = cfg->enable_cors;
s.use_mlx = cfg->use_mlx;
s.use_cuda = cfg->use_cuda;
@@ -2506,7 +2509,7 @@ static void print_usage(const char *prog)
" (default: 32)\n"
" --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
" --batch-wait-us N First-arrival micro-batch deadline in us\n"
- " (default: 0; drain queued work immediately)\n"
+ " (default: CUDA 1000; CPU/MLX 0)\n"
" --memory-utilization F Fraction of physical memory the MLX model-set\n"
" preflight may plan for (default: 0.90; values\n"
" above 1.0 overcommit)\n"