From ed23a1571fe6911eba9d4727bef30c82df7be06d Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 4 Jul 2026 02:37:58 +0200 Subject: server: per-model input token cap, auto-truncate, and 429 admission --- tools/server/models.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'tools/server/models.c') diff --git a/tools/server/models.c b/tools/server/models.c index 2cb35a6..7420542 100644 --- a/tools/server/models.c +++ b/tools/server/models.c @@ -197,6 +197,31 @@ void rerank_request_free(rerank_request *r) { memset(r, 0, sizeof(*r)); } +/* Largest token count one input may carry for a model: the smallest of the + * model's positional capacity, the API item cap, and the per-batch token + * budget. Including the batch budget keeps --max-batch-tokens an honest bound + * on a single inference launch; without it a lone oversized input would run as + * a batch bigger than the configured budget. */ +int model_item_token_cap(const model_info *info, int max_batch_tokens) { + int cap = FFWD_API_MAX_ITEM_TOKENS; + if (info && info->max_seq_tokens > 0 && info->max_seq_tokens < cap) + cap = info->max_seq_tokens; + if (max_batch_tokens > 0 && max_batch_tokens < cap) + cap = max_batch_tokens; + return cap; +} + +/* Cut a tokenized input to cap tokens. When the layout ends with a required + * special token ([SEP], , or the Qwen terminal), that token moves to the + * new end so the truncated sequence keeps a valid layout. */ +void truncate_token_buf(token_buf *t, int cap, int keep_tail) { + if (!t || !t->ids || cap <= 0 || t->n_tokens <= cap) + return; + if (keep_tail) + t->ids[cap - 1] = t->ids[t->n_tokens - 1]; + t->n_tokens = cap; +} + /* Thin wrappers that add server per-stage timing to the tokenization job. * Request handlers call them unchanged. */ int tokenize_input( @@ -277,6 +302,15 @@ int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int tok m->info->dim = config->hidden_size; m->info->min_dim = min_dim; m->info->token_dim = token_dim; + /* Positional capacity: BERT-family positions live in a learned table of + * max_position_embeddings rows starting at position_id_offset; RoPE models + * state a trained context length. 0 (absent) means no model-side cap. */ + m->info->max_seq_tokens = 0; + if (config->max_position_embeddings > 0) { + int cap = config->max_position_embeddings - config->position_id_offset; + m->info->max_seq_tokens = cap > 0 ? cap : 0; + } + m->info->truncate_keep_tail = config->family == FFWD_FAMILY_BERT || config->append_terminal_token; m->info->attention_mode = config->attention_mode; m->info->pooling_mode = config->pooling_mode; m->info->normalize_embeddings = config->normalize_embeddings; -- cgit v1.3.1