aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-07-04 02:37:58 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-07-04 02:37:58 +0200
commited23a1571fe6911eba9d4727bef30c82df7be06d (patch)
treedd9efa8053620f3b9ee662375837671c9880adc8 /tools
parentc2b369f634d1e3087465a65b1a956aea872a156c (diff)
downloadffwd-ed23a1571fe6911eba9d4727bef30c82df7be06d.tar.gz
ffwd-ed23a1571fe6911eba9d4727bef30c82df7be06d.zip
server: per-model input token cap, auto-truncate, and 429 admission
Diffstat (limited to 'tools')
-rw-r--r--tools/server/README.md17
-rw-r--r--tools/server/handlers.c61
-rw-r--r--tools/server/main.c20
-rw-r--r--tools/server/models.c34
-rw-r--r--tools/server/schedule.c2
-rw-r--r--tools/server/server.c30
-rw-r--r--tools/server/server.h4
-rw-r--r--tools/server/server_internal.h17
-rw-r--r--tools/server/test_models.c42
9 files changed, 203 insertions, 24 deletions
diff --git a/tools/server/README.md b/tools/server/README.md
index 594cabc..3098ed9 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -58,7 +58,9 @@ Run `./ffwd-server --help` for the exact, current list. Some important flags:
| `--port PORT` | - | Listen port. |
| `-t, --threads N` | backend | Worker thread count for CPU inference. |
| `-b, --batch-size N` | `32` | Max texts or contextual documents per inference micro-batch. |
-| `--max-batch-tokens N` | `16384` | Token budget per batch (packed for CPU, padded dense rows for MLX). |
+| `--max-batch-tokens N` | `16384` | Token budget per batch (packed for CPU, padded dense rows for MLX). Also caps a single input. |
+| `--max-concurrent-requests N` | `512` | In-flight request cap; excess requests are answered `429` immediately. |
+| `--auto-truncate` | off | Truncate inputs that exceed the per-input token cap instead of rejecting them. |
| `--batch-wait-us N` | backend | Micro-batch deadline from the first request's arrival (CPU/MLX `0`, CUDA `1000`). |
| `--memory-utilization F` | `0.90` | Fraction of physical memory the MLX load preflight may plan for. |
| `--gpu-quant-bits N` | off | MLX: quantize transformer linear weights at load. Opt-in, approximate. |
@@ -236,7 +238,16 @@ curl http://127.0.0.1:8000/v1/rerank \
### Limits and errors
-- A request allows up to 512 inputs (1000 documents for `/v1/rerank`), 32768
- tokens per item, 120000 tokens total, and a 64 MiB body.
+- A request allows up to 512 inputs (1000 documents for `/v1/rerank`), 120000
+ tokens total, and a 64 MiB body.
+- Each input (or packed contextual document, or rerank text) is capped at the
+ smallest of: the model's positional capacity (for example 512 for BERT
+ encoders), `--max-batch-tokens`, and 32768. Inputs past the cap return `422`
+ (`value_error.context_length`, the message states the cap), or are truncated
+ to it when the server runs with `--auto-truncate`. Truncation preserves the
+ model's trailing special token. The server logs each model's effective cap
+ at startup.
- Invalid input returns `422` with a `detail` list naming the offending field.
Unknown labels, and labels loaded for another endpoint, also return `422`.
+- Beyond `--max-concurrent-requests` in-flight requests, new requests return
+ `429` immediately instead of queueing; retry with backoff.
diff --git a/tools/server/handlers.c b/tools/server/handlers.c
index 95eaad1..091ba01 100644
--- a/tools/server/handlers.c
+++ b/tools/server/handlers.c
@@ -288,6 +288,26 @@ static int validate_common(yyjson_val *root,
return 0;
}
+/* Apply the per-item token cap after tokenization: truncate when the server
+ * policy allows it, otherwise record a 422 validation error. Returns 0 when
+ * the input is admitted. Enforced at admission because an over-capacity
+ * sequence fails inside the forward pass, which would fail every request
+ * merged into the same batch. */
+static int admit_input_tokens(
+ http_server *s, const loaded_model *m, token_buf *t, yyjson_mut_doc *detail, const char *loc) {
+ int cap = model_item_token_cap(m->info, s->max_batch_tokens);
+ if (t->n_tokens <= cap)
+ return 0;
+ if (s->auto_truncate) {
+ truncate_token_buf(t, cap, m->info->truncate_keep_tail);
+ return 0;
+ }
+ char msg[64];
+ snprintf(msg, sizeof(msg), "input exceeds %d token limit", cap);
+ ve_add(detail, loc, msg, "value_error.context_length");
+ return -1;
+}
+
void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, embedding_request *out) {
memset(out, 0, sizeof(*out));
out->j = j;
@@ -353,13 +373,11 @@ void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, emb
if (tokenize_input(m, j, single, query_instruct, &out->tokens[0]) != 0) {
ve_add(detail, "[\"body\",\"input\"]", "tokenization failed",
"value_error.tokenization");
- } else {
+ } else if (admit_input_tokens(s, m, &out->tokens[0], detail, "[\"body\",\"input\"]") ==
+ 0) {
out->inputs[0].ids = out->tokens[0].ids;
out->inputs[0].n_tokens = out->tokens[0].n_tokens;
out->total_tokens = out->tokens[0].n_tokens;
- if (out->tokens[0].n_tokens > FFWD_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit",
- "value_error.context_length");
}
} else {
size_t idx, max;
@@ -374,13 +392,10 @@ void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, emb
}
if (tokenize_input(m, j, text, query_instruct, &out->tokens[idx]) != 0) {
ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
- } else {
+ } else if (admit_input_tokens(s, m, &out->tokens[idx], detail, loc) == 0) {
out->inputs[idx].ids = out->tokens[idx].ids;
out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
out->total_tokens += out->tokens[idx].n_tokens;
- if (out->tokens[idx].n_tokens > FFWD_API_MAX_ITEM_TOKENS)
- ve_add(detail, loc, "input exceeds 32768 token limit",
- "value_error.context_length");
}
}
}
@@ -474,6 +489,11 @@ void prepare_contextual_request(job *j,
}
if (ve_count(detail) == 0 && m) {
+ /* A packed document (chunks joined by separators) must fit the model
+ * and the batch budget like any other sequence. Truncation is not
+ * offered here: cutting the packed stream would drop chunks whose
+ * embeddings the response is required to contain. */
+ int item_cap = model_item_token_cap(m->info, s->max_batch_tokens);
out->n_docs = n_docs;
out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs));
int64_t request_tokens = 0;
@@ -501,11 +521,12 @@ void prepare_contextual_request(job *j,
}
}
- if (doc_tokens > FFWD_API_MAX_ITEM_TOKENS) {
+ if (doc_tokens > item_cap) {
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", (int)di);
- ve_add(detail, loc, "document exceeds 32768 token limit",
- "value_error.context_length");
+ char msg[64];
+ snprintf(msg, sizeof(msg), "document exceeds %d token limit", item_cap);
+ ve_add(detail, loc, msg, "value_error.context_length");
doc_valid = 0;
}
request_tokens += doc_tokens;
@@ -663,15 +684,20 @@ void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank
}
if (ve_count(detail) == 0 && out->model) {
+ /* Late-interaction sequences carry keep masks alongside the ids, so
+ * truncation is not offered; over-capacity inputs are rejected. */
+ int item_cap = model_item_token_cap(out->model->info, s->max_batch_tokens);
+ char cap_msg[64];
out->n_documents = n_documents;
out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) {
ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization");
} else {
out->query_tokens = out->query.n_tokens;
- if (out->query_tokens > FFWD_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit",
- "value_error.context_length");
+ if (out->query_tokens > item_cap) {
+ snprintf(cap_msg, sizeof(cap_msg), "query exceeds %d token limit", item_cap);
+ ve_add(detail, "[\"body\",\"query\"]", cap_msg, "value_error.context_length");
+ }
}
size_t i, dmax;
@@ -684,9 +710,10 @@ void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank
ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
} else {
int n = out->documents[i].n_tokens;
- if (n > FFWD_API_MAX_ITEM_TOKENS)
- ve_add(detail, loc, "document exceeds 32768 token limit",
- "value_error.context_length");
+ if (n > item_cap) {
+ snprintf(cap_msg, sizeof(cap_msg), "document exceeds %d token limit", item_cap);
+ ve_add(detail, loc, cap_msg, "value_error.context_length");
+ }
if (out->document_tokens <= INT_MAX - n)
out->document_tokens += n;
else
diff --git a/tools/server/main.c b/tools/server/main.c
index 4d79872..c0d01bc 100644
--- a/tools/server/main.c
+++ b/tools/server/main.c
@@ -39,9 +39,14 @@ static void print_usage(const char *prog) {
" (default: 32)\n"
" --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
" --max-request-tokens N Max total tokens per API request (default: 120000)\n"
+ " --max-concurrent-requests N\n"
+ " In-flight request cap; excess requests get 429\n"
+ " (default: %d)\n"
+ " --auto-truncate Truncate inputs that exceed the per-input token\n"
+ " cap instead of rejecting them with 422\n"
" --batch-wait-us N First-arrival micro-batch deadline in us\n"
" (default: %d)\n",
- prog, ffwd_default_batch_wait_us());
+ prog, FFWD_SERVER_DEFAULT_MAX_CONCURRENT, ffwd_default_batch_wait_us());
ffwd_server_help(stderr);
fprintf(stderr,
" -v, --verbose Verbose (-vv for debug)\n"
@@ -212,6 +217,8 @@ int main(int argc, char *argv[]) {
int batch_size = 0;
int max_batch_tokens = 0;
int max_request_tokens = 0;
+ int max_concurrent_requests = 0;
+ int auto_truncate = 0;
int batch_wait_us = -1;
/* Backend tuning flags parse into opts regardless of backend; the linked
* backend uses what it supports. */
@@ -288,6 +295,15 @@ int main(int argc, char *argv[]) {
free_model_specs(&model_specs);
return 1;
}
+ } else if (!strcmp(f, "--max-concurrent-requests")) {
+ max_concurrent_requests = atoi(argv[++arg]);
+ if (max_concurrent_requests <= 0) {
+ fprintf(stderr, "--max-concurrent-requests must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--auto-truncate")) {
+ auto_truncate = 1;
} else if (!strcmp(f, "--batch-wait-us")) {
batch_wait_us = atoi(argv[++arg]);
if (batch_wait_us < 0) {
@@ -343,6 +359,8 @@ int main(int argc, char *argv[]) {
.batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE,
.max_batch_tokens = max_batch_tokens,
.max_request_tokens = max_request_tokens,
+ .max_concurrent_requests = max_concurrent_requests,
+ .auto_truncate = auto_truncate,
.batch_wait_us = batch_wait_us,
.gpu_quantize_bits = opts.gpu_quant_bits,
.gpu_quantize_group_size = opts.gpu_quant_group_size,
diff --git a/tools/server/models.c b/tools/server/models.c
index 2cb35a6..7420542 100644
--- a/tools/server/models.c
+++ b/tools/server/models.c
@@ -197,6 +197,31 @@ void rerank_request_free(rerank_request *r) {
memset(r, 0, sizeof(*r));
}
+/* Largest token count one input may carry for a model: the smallest of the
+ * model's positional capacity, the API item cap, and the per-batch token
+ * budget. Including the batch budget keeps --max-batch-tokens an honest bound
+ * on a single inference launch; without it a lone oversized input would run as
+ * a batch bigger than the configured budget. */
+int model_item_token_cap(const model_info *info, int max_batch_tokens) {
+ int cap = FFWD_API_MAX_ITEM_TOKENS;
+ if (info && info->max_seq_tokens > 0 && info->max_seq_tokens < cap)
+ cap = info->max_seq_tokens;
+ if (max_batch_tokens > 0 && max_batch_tokens < cap)
+ cap = max_batch_tokens;
+ return cap;
+}
+
+/* Cut a tokenized input to cap tokens. When the layout ends with a required
+ * special token ([SEP], </s>, or the Qwen terminal), that token moves to the
+ * new end so the truncated sequence keeps a valid layout. */
+void truncate_token_buf(token_buf *t, int cap, int keep_tail) {
+ if (!t || !t->ids || cap <= 0 || t->n_tokens <= cap)
+ return;
+ if (keep_tail)
+ t->ids[cap - 1] = t->ids[t->n_tokens - 1];
+ t->n_tokens = cap;
+}
+
/* Thin wrappers that add server per-stage timing to the tokenization job.
* Request handlers call them unchanged. */
int tokenize_input(
@@ -277,6 +302,15 @@ int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int tok
m->info->dim = config->hidden_size;
m->info->min_dim = min_dim;
m->info->token_dim = token_dim;
+ /* Positional capacity: BERT-family positions live in a learned table of
+ * max_position_embeddings rows starting at position_id_offset; RoPE models
+ * state a trained context length. 0 (absent) means no model-side cap. */
+ m->info->max_seq_tokens = 0;
+ if (config->max_position_embeddings > 0) {
+ int cap = config->max_position_embeddings - config->position_id_offset;
+ m->info->max_seq_tokens = cap > 0 ? cap : 0;
+ }
+ m->info->truncate_keep_tail = config->family == FFWD_FAMILY_BERT || config->append_terminal_token;
m->info->attention_mode = config->attention_mode;
m->info->pooling_mode = config->pooling_mode;
m->info->normalize_embeddings = config->normalize_embeddings;
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index 8254739..c71cad3 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -250,6 +250,8 @@ void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
job *j;
while ((j = pop_done(s)) != NULL) {
+ /* Every admitted job ends here; release its admission slot. */
+ s->inflight_jobs--;
client *c = j->c;
int refs_to_drop = 1; /* job reference */
if (!c->cancelled) {
diff --git a/tools/server/server.c b/tools/server/server.c
index 31eed36..9686019 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -94,6 +94,21 @@ void dispatch_request(client *c) {
return;
}
+ /* Admission control: beyond the in-flight cap, shed load with an immediate
+ * 429 instead of queueing without bound. The counter lives on the event
+ * loop thread (incremented here, decremented in completion_cb), so no lock
+ * is needed. */
+ if (c->srv->inflight_jobs >= c->srv->max_concurrent_requests) {
+ size_t len;
+ char *body = json_error_body("server is overloaded", "overloaded_error", &len);
+ append_http_response(c, 429, "application/json", body, len);
+ free(body);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+ return;
+ }
+ c->srv->inflight_jobs++;
+
job *j = xcalloc(1, sizeof(*j));
j->srv = c->srv;
j->c = c;
@@ -343,6 +358,9 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
s.max_request_tokens =
cfg->max_request_tokens > 0 ? cfg->max_request_tokens : FFWD_API_MAX_TOTAL_TOKENS;
+ s.max_concurrent_requests = cfg->max_concurrent_requests > 0 ? cfg->max_concurrent_requests
+ : FFWD_SERVER_DEFAULT_MAX_CONCURRENT;
+ s.auto_truncate = cfg->auto_truncate ? 1 : 0;
s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
s.backend_opts = backend_opts;
if (cfg->api_key && cfg->api_key[0])
@@ -460,6 +478,18 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
return 1;
}
+ /* Capacity line per model, after worker init so a GPU build has finalized
+ * the config: the effective per-input token cap and the batch shape make
+ * the serving envelope visible at startup. */
+ for (int i = 0; i < s.n_models; i++) {
+ loaded_model *m = &s.models[i];
+ server_log("ffwd-server: model %s: max %d tokens per input%s, batch budget %d tokens x %d "
+ "rows, max %d concurrent requests",
+ m->info->id, model_item_token_cap(m->info, s.max_batch_tokens),
+ s.auto_truncate ? " (auto-truncate)" : "", s.max_batch_tokens, s.batch_size,
+ s.max_concurrent_requests);
+ }
+
/* Started after worker init so the tokenizer sees a finalized model config
* (a GPU build sets dims/pooling on the worker, and request validation reads
* them). Joined before the worker at shutdown so it drains raw jobs into the
diff --git a/tools/server/server.h b/tools/server/server.h
index 97fb57f..151a793 100644
--- a/tools/server/server.h
+++ b/tools/server/server.h
@@ -39,7 +39,9 @@ typedef struct {
int port;
int batch_size;
int max_batch_tokens;
- int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */
+ int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */
+ int max_concurrent_requests; /* in-flight request cap before 429; 0 = default 512 */
+ int auto_truncate; /* truncate over-long inputs instead of rejecting them */
int batch_wait_us;
/* GPU build tuning; ignored by backends that do not use it. */
int gpu_quantize_bits;
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index d2829aa..51ef7dc 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -42,6 +42,9 @@
* was no faster and inflated long-document tail latency. */
#define FFWD_SERVER_DEFAULT_BATCH_SIZE 32
#define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
+/* In-flight request cap; dispatch answers 429 beyond it so overload sheds load
+ * instead of growing the job queues without bound. */
+#define FFWD_SERVER_DEFAULT_MAX_CONCURRENT 512
/* Microseconds the worker waits for more requests before dispatching a batch.
* CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
* busy; MLX and CPU dispatch immediately, where waiting only adds latency.
@@ -65,6 +68,13 @@ typedef struct {
int dim;
int min_dim;
int token_dim;
+ /* Positional capacity in tokens (max_position_embeddings minus the
+ * position-id offset); 0 when the config does not state one. Inputs past
+ * this fail in the forward pass, so admission enforces it up front. */
+ int max_seq_tokens;
+ /* The token layout ends with a required special token ([SEP], </s>, or the
+ * Qwen terminal) that truncation must keep as the final token. */
+ int truncate_keep_tail;
ffwd_attention_mode_t attention_mode;
ffwd_pooling_mode_t pooling_mode;
int normalize_embeddings;
@@ -92,7 +102,10 @@ typedef struct {
int port;
int batch_size;
int max_batch_tokens;
- int max_request_tokens; // per-request total-token cap (FFWD_API_MAX_TOTAL_TOKENS default)
+ int max_request_tokens; // per-request total-token cap (FFWD_API_MAX_TOTAL_TOKENS default)
+ int max_concurrent_requests; // in-flight request cap; dispatch answers 429 beyond it
+ int inflight_jobs; // loop-thread only: jobs created and not yet completed
+ int auto_truncate; // truncate over-long inputs instead of rejecting them
int batch_wait_us;
char *api_key;
loaded_model *models;
@@ -312,6 +325,8 @@ void embedding_request_free(embedding_request *r);
void contextual_request_free(contextual_request *r);
void late_tokens_free(late_tokens *t);
void rerank_request_free(rerank_request *r);
+int model_item_token_cap(const model_info *info, int max_batch_tokens);
+void truncate_token_buf(token_buf *t, int cap, int keep_tail);
/* Thin timing wrappers over libffwd's ffwd_tokenize/ffwd_tokenize_late. */
int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out);
int tokenize_input(
diff --git a/tools/server/test_models.c b/tools/server/test_models.c
index 3dce62b..48a2da1 100644
--- a/tools/server/test_models.c
+++ b/tools/server/test_models.c
@@ -1,4 +1,5 @@
-/* test_models.c - resolving a served model by its request label. */
+/* test_models.c - resolving a served model by its request label, and the
+ * per-input token cap with its truncation helper. */
#include "server_internal.h"
#include "test_util.h"
@@ -20,7 +21,46 @@ static void test_loaded_model_lookup(void) {
TEST_ASSERT(loaded_model_for_label(&s, "compiled-name") == NULL);
}
+static void test_model_item_token_cap(void) {
+ model_info info;
+ memset(&info, 0, sizeof(info));
+
+ /* No model cap: the API item cap applies, then the batch budget. */
+ TEST_ASSERT(model_item_token_cap(&info, 0) == FFWD_API_MAX_ITEM_TOKENS);
+ TEST_ASSERT(model_item_token_cap(&info, 16384) == 16384);
+
+ /* Model cap below both: it wins. */
+ info.max_seq_tokens = 512;
+ TEST_ASSERT(model_item_token_cap(&info, 16384) == 512);
+
+ /* Batch budget below the model cap: the budget wins. */
+ info.max_seq_tokens = 32768;
+ TEST_ASSERT(model_item_token_cap(&info, 16384) == 16384);
+ TEST_ASSERT(model_item_token_cap(&info, 0) == 32768);
+}
+
+static void test_truncate_token_buf(void) {
+ int ids[6] = {101, 7, 8, 9, 10, 102};
+ token_buf t = {.ids = ids, .n_tokens = 6};
+
+ /* Under the cap: untouched. */
+ truncate_token_buf(&t, 6, 1);
+ TEST_ASSERT(t.n_tokens == 6 && ids[5] == 102);
+
+ /* keep_tail moves the final special token to the new end. */
+ truncate_token_buf(&t, 4, 1);
+ TEST_ASSERT(t.n_tokens == 4);
+ TEST_ASSERT(ids[0] == 101 && ids[1] == 7 && ids[2] == 8 && ids[3] == 102);
+
+ /* Plain cut when the layout has no required tail. */
+ truncate_token_buf(&t, 2, 0);
+ TEST_ASSERT(t.n_tokens == 2);
+ TEST_ASSERT(ids[0] == 101 && ids[1] == 7);
+}
+
int main(void) {
test_loaded_model_lookup();
+ test_model_item_token_cap();
+ test_truncate_token_buf();
return TEST_REPORT("models");
}