aboutsummaryrefslogtreecommitdiff
path: root/tools/server/main.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-02 21:21:52 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-02 21:21:52 +0200
commitc2b369f634d1e3087465a65b1a956aea872a156c (patch)
tree78d8bb1d9989a3435e0318eeb6307865a6098b32 /tools/server/main.c
parent4d79e6337c05d0a768de80eeb164f1de3b1624d3 (diff)
downloadffwd-c2b369f634d1e3087465a65b1a956aea872a156c.tar.gz
ffwd-c2b369f634d1e3087465a65b1a956aea872a156c.zip
server: make the per-request token cap configurable
Diffstat (limited to 'tools/server/main.c')
-rw-r--r--tools/server/main.c10
1 files changed, 10 insertions, 0 deletions
diff --git a/tools/server/main.c b/tools/server/main.c
index baa4e39..4d79872 100644
--- a/tools/server/main.c
+++ b/tools/server/main.c
@@ -38,6 +38,7 @@ static void print_usage(const char *prog) {
" -b, --batch-size N Max texts or documents per inference batch\n"
" (default: 32)\n"
" --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --max-request-tokens N Max total tokens per API request (default: 120000)\n"
" --batch-wait-us N First-arrival micro-batch deadline in us\n"
" (default: %d)\n",
prog, ffwd_default_batch_wait_us());
@@ -210,6 +211,7 @@ int main(int argc, char *argv[]) {
int verbose = 0;
int batch_size = 0;
int max_batch_tokens = 0;
+ int max_request_tokens = 0;
int batch_wait_us = -1;
/* Backend tuning flags parse into opts regardless of backend; the linked
* backend uses what it supports. */
@@ -279,6 +281,13 @@ int main(int argc, char *argv[]) {
free_model_specs(&model_specs);
return 1;
}
+ } else if (!strcmp(f, "--max-request-tokens")) {
+ max_request_tokens = atoi(argv[++arg]);
+ if (max_request_tokens <= 0) {
+ fprintf(stderr, "--max-request-tokens must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
} else if (!strcmp(f, "--batch-wait-us")) {
batch_wait_us = atoi(argv[++arg]);
if (batch_wait_us < 0) {
@@ -333,6 +342,7 @@ int main(int argc, char *argv[]) {
.port = port,
.batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE,
.max_batch_tokens = max_batch_tokens,
+ .max_request_tokens = max_request_tokens,
.batch_wait_us = batch_wait_us,
.gpu_quantize_bits = opts.gpu_quant_bits,
.gpu_quantize_group_size = opts.gpu_quant_group_size,