aboutsummaryrefslogtreecommitdiff
path: root/main.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-31 00:01:27 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit2ceffe319f42bbf966f085a581c71eb9e369cf19 (patch)
treeb652bc7a302c2be91369716f51e841abc2728dc7 /main.c
parent6c60a776d5f2f2f494d20072a595fa0e10dd51d1 (diff)
downloadffwd-2ceffe319f42bbf966f085a581c71eb9e369cf19.tar.gz
ffwd-2ceffe319f42bbf966f085a581c71eb9e369cf19.zip
server: batch standard requests by token budget
Diffstat (limited to 'main.c')
-rw-r--r--main.c12
1 files changed, 12 insertions, 0 deletions
diff --git a/main.c b/main.c
index 55756e2..8bcf405 100644
--- a/main.c
+++ b/main.c
@@ -47,6 +47,8 @@ static void print_usage(const char *prog)
" --cors Enable CORS headers in --serve mode\n"
" --api-key K Require Authorization: Bearer K in --serve mode\n"
" -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n"
+ " --max-batch-tokens N\n"
+ " Server max tokens per standard inference batch (default: 16384)\n"
" --batch-wait-us N\n"
" Server micro-batch wait in microseconds (default: 1000; 0 disables)\n"
" -t <n> CPU threads (default: all cores)\n"
@@ -509,6 +511,7 @@ int main(int argc, char *argv[])
int layer_start = -1;
int layer_end = -1;
int batch_size = 0;
+ int max_batch_tokens = 0;
int batch_wait_us = -1;
const char *backend = "cpu";
const char *host = "127.0.0.1";
@@ -533,6 +536,14 @@ int main(int argc, char *argv[])
return 1;
}
}
+ else if (!strcmp(f, "--max-batch-tokens")) {
+ max_batch_tokens = atoi(argv[++arg_start]);
+ if (max_batch_tokens <= 0) {
+ fprintf(stderr, "--max-batch-tokens must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
else if (!strcmp(f, "--model")) {
if (append_model_spec(&model_specs, argv[++arg_start]) != 0) return 1;
}
@@ -619,6 +630,7 @@ int main(int argc, char *argv[])
.host = host,
.port = port,
.batch_size = batch_size > 0 ? batch_size : 8,
+ .max_batch_tokens = max_batch_tokens,
.batch_wait_us = batch_wait_us,
.use_mlx = use_mlx,
.enable_cors = cors,