From ed23a1571fe6911eba9d4727bef30c82df7be06d Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 4 Jul 2026 02:37:58 +0200 Subject: server: per-model input token cap, auto-truncate, and 429 admission --- tools/server/server.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tools/server/server.h') diff --git a/tools/server/server.h b/tools/server/server.h index 97fb57f..151a793 100644 --- a/tools/server/server.h +++ b/tools/server/server.h @@ -39,7 +39,9 @@ typedef struct { int port; int batch_size; int max_batch_tokens; - int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */ + int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */ + int max_concurrent_requests; /* in-flight request cap before 429; 0 = default 512 */ + int auto_truncate; /* truncate over-long inputs instead of rejecting them */ int batch_wait_us; /* GPU build tuning; ignored by backends that do not use it. */ int gpu_quantize_bits; -- cgit v1.3.1