From 13c56b2218724ebcd2ef97bc386bc60930cf1ff0 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 13 Jun 2026 01:34:18 +0200 Subject: server: tune admission deadline by backend --- embed_server.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'embed_server.c') diff --git a/embed_server.c b/embed_server.c index 793d764..d4f19f1 100644 --- a/embed_server.c +++ b/embed_server.c @@ -66,6 +66,7 @@ #define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 #define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 #define EMBED_SERVER_BATCH_WAIT_US 0 +#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 #define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 #define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 #define EMBED_MLX_RESIDENT_MULTIPLIER 2 @@ -2358,7 +2359,9 @@ int embed_run_server(const embed_server_config_t *cfg) { s.max_batch_tokens = cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; s.batch_wait_us = cfg->batch_wait_us >= 0 - ? cfg->batch_wait_us : EMBED_SERVER_BATCH_WAIT_US; + ? cfg->batch_wait_us + : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US + : EMBED_SERVER_BATCH_WAIT_US); s.enable_cors = cfg->enable_cors; s.use_mlx = cfg->use_mlx; s.use_cuda = cfg->use_cuda; @@ -2506,7 +2509,7 @@ static void print_usage(const char *prog) " (default: 32)\n" " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" " --batch-wait-us N First-arrival micro-batch deadline in us\n" - " (default: 0; drain queued work immediately)\n" + " (default: CUDA 1000; CPU/MLX 0)\n" " --memory-utilization F Fraction of physical memory the MLX model-set\n" " preflight may plan for (default: 0.90; values\n" " above 1.0 overcommit)\n" -- cgit v1.3.1