diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 21:21:52 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-07-02 21:21:52 +0200 |
| commit | c2b369f634d1e3087465a65b1a956aea872a156c (patch) | |
| tree | 78d8bb1d9989a3435e0318eeb6307865a6098b32 | |
| parent | 4d79e6337c05d0a768de80eeb164f1de3b1624d3 (diff) | |
| download | ffwd-c2b369f634d1e3087465a65b1a956aea872a156c.tar.gz ffwd-c2b369f634d1e3087465a65b1a956aea872a156c.zip | |
server: make the per-request token cap configurable
| -rw-r--r-- | tools/server/handlers.c | 12 | ||||
| -rw-r--r-- | tools/server/main.c | 10 | ||||
| -rw-r--r-- | tools/server/server.c | 2 | ||||
| -rw-r--r-- | tools/server/server.h | 1 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 1 |
5 files changed, 20 insertions, 6 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c index 204f29e..95eaad1 100644 --- a/tools/server/handlers.c +++ b/tools/server/handlers.c @@ -385,8 +385,8 @@ void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, emb } } - if (out->total_tokens > FFWD_API_MAX_TOTAL_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + if (out->total_tokens > s->max_request_tokens) + ve_add(detail, "[\"body\",\"input\"]", "request exceeds the request token limit", "value_error.context_length"); } @@ -533,8 +533,8 @@ void prepare_contextual_request(job *j, free_token_bufs(chunk_tokens, n_chunks); free(chunk_tokens); } - if (request_tokens > FFWD_API_MAX_TOTAL_TOKENS) { - ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + if (request_tokens > s->max_request_tokens) { + ve_add(detail, "[\"body\",\"input\"]", "request exceeds the request token limit", "value_error.context_length"); } else { out->total_tokens = (int)request_tokens; @@ -693,8 +693,8 @@ void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank out->document_tokens = INT_MAX; } } - if (out->query_tokens > FFWD_API_MAX_TOTAL_TOKENS - out->document_tokens) { - ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", + if (out->query_tokens > s->max_request_tokens - out->document_tokens) { + ve_add(detail, "[\"body\",\"documents\"]", "request exceeds the request token limit", "value_error.context_length"); } } diff --git a/tools/server/main.c b/tools/server/main.c index baa4e39..4d79872 100644 --- a/tools/server/main.c +++ b/tools/server/main.c @@ -38,6 +38,7 @@ static void print_usage(const char *prog) { " -b, --batch-size N Max texts or documents per inference batch\n" " (default: 32)\n" " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --max-request-tokens N Max total tokens per API request (default: 120000)\n" " --batch-wait-us N First-arrival micro-batch deadline in us\n" " (default: %d)\n", prog, ffwd_default_batch_wait_us()); @@ -210,6 +211,7 @@ int main(int argc, char *argv[]) { int verbose = 0; int batch_size = 0; int max_batch_tokens = 0; + int max_request_tokens = 0; int batch_wait_us = -1; /* Backend tuning flags parse into opts regardless of backend; the linked * backend uses what it supports. */ @@ -279,6 +281,13 @@ int main(int argc, char *argv[]) { free_model_specs(&model_specs); return 1; } + } else if (!strcmp(f, "--max-request-tokens")) { + max_request_tokens = atoi(argv[++arg]); + if (max_request_tokens <= 0) { + fprintf(stderr, "--max-request-tokens must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } } else if (!strcmp(f, "--batch-wait-us")) { batch_wait_us = atoi(argv[++arg]); if (batch_wait_us < 0) { @@ -333,6 +342,7 @@ int main(int argc, char *argv[]) { .port = port, .batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE, .max_batch_tokens = max_batch_tokens, + .max_request_tokens = max_request_tokens, .batch_wait_us = batch_wait_us, .gpu_quantize_bits = opts.gpu_quant_bits, .gpu_quantize_group_size = opts.gpu_quant_group_size, diff --git a/tools/server/server.c b/tools/server/server.c index 2bf7153..31eed36 100644 --- a/tools/server/server.c +++ b/tools/server/server.c @@ -341,6 +341,8 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE; s.max_batch_tokens = cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS; + s.max_request_tokens = + cfg->max_request_tokens > 0 ? cfg->max_request_tokens : FFWD_API_MAX_TOTAL_TOKENS; s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us(); s.backend_opts = backend_opts; if (cfg->api_key && cfg->api_key[0]) diff --git a/tools/server/server.h b/tools/server/server.h index 2add3f0..97fb57f 100644 --- a/tools/server/server.h +++ b/tools/server/server.h @@ -39,6 +39,7 @@ typedef struct { int port; int batch_size; int max_batch_tokens; + int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */ int batch_wait_us; /* GPU build tuning; ignored by backends that do not use it. */ int gpu_quantize_bits; diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index 2369031..d2829aa 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -92,6 +92,7 @@ typedef struct { int port; int batch_size; int max_batch_tokens; + int max_request_tokens; // per-request total-token cap (FFWD_API_MAX_TOTAL_TOKENS default) int batch_wait_us; char *api_key; loaded_model *models; |
