aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-02 21:21:52 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-02 21:21:52 +0200
commitc2b369f634d1e3087465a65b1a956aea872a156c (patch)
tree78d8bb1d9989a3435e0318eeb6307865a6098b32 /tools
parent4d79e6337c05d0a768de80eeb164f1de3b1624d3 (diff)
downloadffwd-c2b369f634d1e3087465a65b1a956aea872a156c.tar.gz
ffwd-c2b369f634d1e3087465a65b1a956aea872a156c.zip
server: make the per-request token cap configurable
Diffstat (limited to 'tools')
-rw-r--r--tools/server/handlers.c12
-rw-r--r--tools/server/main.c10
-rw-r--r--tools/server/server.c2
-rw-r--r--tools/server/server.h1
-rw-r--r--tools/server/server_internal.h1
5 files changed, 20 insertions, 6 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c
index 204f29e..95eaad1 100644
--- a/tools/server/handlers.c
+++ b/tools/server/handlers.c
@@ -385,8 +385,8 @@ void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, emb
}
}
- if (out->total_tokens > FFWD_API_MAX_TOTAL_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ if (out->total_tokens > s->max_request_tokens)
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds the request token limit",
"value_error.context_length");
}
@@ -533,8 +533,8 @@ void prepare_contextual_request(job *j,
free_token_bufs(chunk_tokens, n_chunks);
free(chunk_tokens);
}
- if (request_tokens > FFWD_API_MAX_TOTAL_TOKENS) {
- ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ if (request_tokens > s->max_request_tokens) {
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds the request token limit",
"value_error.context_length");
} else {
out->total_tokens = (int)request_tokens;
@@ -693,8 +693,8 @@ void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank
out->document_tokens = INT_MAX;
}
}
- if (out->query_tokens > FFWD_API_MAX_TOTAL_TOKENS - out->document_tokens) {
- ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit",
+ if (out->query_tokens > s->max_request_tokens - out->document_tokens) {
+ ve_add(detail, "[\"body\",\"documents\"]", "request exceeds the request token limit",
"value_error.context_length");
}
}
diff --git a/tools/server/main.c b/tools/server/main.c
index baa4e39..4d79872 100644
--- a/tools/server/main.c
+++ b/tools/server/main.c
@@ -38,6 +38,7 @@ static void print_usage(const char *prog) {
" -b, --batch-size N Max texts or documents per inference batch\n"
" (default: 32)\n"
" --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --max-request-tokens N Max total tokens per API request (default: 120000)\n"
" --batch-wait-us N First-arrival micro-batch deadline in us\n"
" (default: %d)\n",
prog, ffwd_default_batch_wait_us());
@@ -210,6 +211,7 @@ int main(int argc, char *argv[]) {
int verbose = 0;
int batch_size = 0;
int max_batch_tokens = 0;
+ int max_request_tokens = 0;
int batch_wait_us = -1;
/* Backend tuning flags parse into opts regardless of backend; the linked
* backend uses what it supports. */
@@ -279,6 +281,13 @@ int main(int argc, char *argv[]) {
free_model_specs(&model_specs);
return 1;
}
+ } else if (!strcmp(f, "--max-request-tokens")) {
+ max_request_tokens = atoi(argv[++arg]);
+ if (max_request_tokens <= 0) {
+ fprintf(stderr, "--max-request-tokens must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
} else if (!strcmp(f, "--batch-wait-us")) {
batch_wait_us = atoi(argv[++arg]);
if (batch_wait_us < 0) {
@@ -333,6 +342,7 @@ int main(int argc, char *argv[]) {
.port = port,
.batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE,
.max_batch_tokens = max_batch_tokens,
+ .max_request_tokens = max_request_tokens,
.batch_wait_us = batch_wait_us,
.gpu_quantize_bits = opts.gpu_quant_bits,
.gpu_quantize_group_size = opts.gpu_quant_group_size,
diff --git a/tools/server/server.c b/tools/server/server.c
index 2bf7153..31eed36 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -341,6 +341,8 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE;
s.max_batch_tokens =
cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
+ s.max_request_tokens =
+ cfg->max_request_tokens > 0 ? cfg->max_request_tokens : FFWD_API_MAX_TOTAL_TOKENS;
s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
s.backend_opts = backend_opts;
if (cfg->api_key && cfg->api_key[0])
diff --git a/tools/server/server.h b/tools/server/server.h
index 2add3f0..97fb57f 100644
--- a/tools/server/server.h
+++ b/tools/server/server.h
@@ -39,6 +39,7 @@ typedef struct {
int port;
int batch_size;
int max_batch_tokens;
+ int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */
int batch_wait_us;
/* GPU build tuning; ignored by backends that do not use it. */
int gpu_quantize_bits;
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index 2369031..d2829aa 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -92,6 +92,7 @@ typedef struct {
int port;
int batch_size;
int max_batch_tokens;
+ int max_request_tokens; // per-request total-token cap (FFWD_API_MAX_TOTAL_TOKENS default)
int batch_wait_us;
char *api_key;
loaded_model *models;