diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 21:21:52 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 21:21:52 +0200 |
| commit | c2b369f634d1e3087465a65b1a956aea872a156c (patch) | |
| tree | 78d8bb1d9989a3435e0318eeb6307865a6098b32 /tools/server/main.c | |
| parent | 4d79e6337c05d0a768de80eeb164f1de3b1624d3 (diff) | |
| download | ffwd-c2b369f634d1e3087465a65b1a956aea872a156c.tar.gz ffwd-c2b369f634d1e3087465a65b1a956aea872a156c.zip | |
server: make the per-request token cap configurable
Diffstat (limited to 'tools/server/main.c')
| -rw-r--r-- | tools/server/main.c | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/tools/server/main.c b/tools/server/main.c index baa4e39..4d79872 100644 --- a/tools/server/main.c +++ b/tools/server/main.c @@ -38,6 +38,7 @@ static void print_usage(const char *prog) { " -b, --batch-size N Max texts or documents per inference batch\n" " (default: 32)\n" " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --max-request-tokens N Max total tokens per API request (default: 120000)\n" " --batch-wait-us N First-arrival micro-batch deadline in us\n" " (default: %d)\n", prog, ffwd_default_batch_wait_us()); @@ -210,6 +211,7 @@ int main(int argc, char *argv[]) { int verbose = 0; int batch_size = 0; int max_batch_tokens = 0; + int max_request_tokens = 0; int batch_wait_us = -1; /* Backend tuning flags parse into opts regardless of backend; the linked * backend uses what it supports. */ @@ -279,6 +281,13 @@ int main(int argc, char *argv[]) { free_model_specs(&model_specs); return 1; } + } else if (!strcmp(f, "--max-request-tokens")) { + max_request_tokens = atoi(argv[++arg]); + if (max_request_tokens <= 0) { + fprintf(stderr, "--max-request-tokens must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } } else if (!strcmp(f, "--batch-wait-us")) { batch_wait_us = atoi(argv[++arg]); if (batch_wait_us < 0) { @@ -333,6 +342,7 @@ int main(int argc, char *argv[]) { .port = port, .batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE, .max_batch_tokens = max_batch_tokens, + .max_request_tokens = max_request_tokens, .batch_wait_us = batch_wait_us, .gpu_quantize_bits = opts.gpu_quant_bits, .gpu_quantize_group_size = opts.gpu_quant_group_size, |
