diff options
Diffstat (limited to 'tools/server')
| -rw-r--r-- | tools/server/README.md | 17 | ||||
| -rw-r--r-- | tools/server/handlers.c | 61 | ||||
| -rw-r--r-- | tools/server/main.c | 20 | ||||
| -rw-r--r-- | tools/server/models.c | 34 | ||||
| -rw-r--r-- | tools/server/schedule.c | 2 | ||||
| -rw-r--r-- | tools/server/server.c | 30 | ||||
| -rw-r--r-- | tools/server/server.h | 4 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 17 | ||||
| -rw-r--r-- | tools/server/test_models.c | 42 |
9 files changed, 203 insertions, 24 deletions
diff --git a/tools/server/README.md b/tools/server/README.md index 594cabc..3098ed9 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -58,7 +58,9 @@ Run `./ffwd-server --help` for the exact, current list. Some important flags: | `--port PORT` | - | Listen port. | | `-t, --threads N` | backend | Worker thread count for CPU inference. | | `-b, --batch-size N` | `32` | Max texts or contextual documents per inference micro-batch. | -| `--max-batch-tokens N` | `16384` | Token budget per batch (packed for CPU, padded dense rows for MLX). | +| `--max-batch-tokens N` | `16384` | Token budget per batch (packed for CPU, padded dense rows for MLX). Also caps a single input. | +| `--max-concurrent-requests N` | `512` | In-flight request cap; excess requests are answered `429` immediately. | +| `--auto-truncate` | off | Truncate inputs that exceed the per-input token cap instead of rejecting them. | | `--batch-wait-us N` | backend | Micro-batch deadline from the first request's arrival (CPU/MLX `0`, CUDA `1000`). | | `--memory-utilization F` | `0.90` | Fraction of physical memory the MLX load preflight may plan for. | | `--gpu-quant-bits N` | off | MLX: quantize transformer linear weights at load. Opt-in, approximate. | @@ -236,7 +238,16 @@ curl http://127.0.0.1:8000/v1/rerank \ ### Limits and errors -- A request allows up to 512 inputs (1000 documents for `/v1/rerank`), 32768 - tokens per item, 120000 tokens total, and a 64 MiB body. +- A request allows up to 512 inputs (1000 documents for `/v1/rerank`), 120000 + tokens total, and a 64 MiB body. +- Each input (or packed contextual document, or rerank text) is capped at the + smallest of: the model's positional capacity (for example 512 for BERT + encoders), `--max-batch-tokens`, and 32768. Inputs past the cap return `422` + (`value_error.context_length`, the message states the cap), or are truncated + to it when the server runs with `--auto-truncate`. Truncation preserves the + model's trailing special token. The server logs each model's effective cap + at startup. - Invalid input returns `422` with a `detail` list naming the offending field. Unknown labels, and labels loaded for another endpoint, also return `422`. +- Beyond `--max-concurrent-requests` in-flight requests, new requests return + `429` immediately instead of queueing; retry with backoff. diff --git a/tools/server/handlers.c b/tools/server/handlers.c index 95eaad1..091ba01 100644 --- a/tools/server/handlers.c +++ b/tools/server/handlers.c @@ -288,6 +288,26 @@ static int validate_common(yyjson_val *root, return 0; } +/* Apply the per-item token cap after tokenization: truncate when the server + * policy allows it, otherwise record a 422 validation error. Returns 0 when + * the input is admitted. Enforced at admission because an over-capacity + * sequence fails inside the forward pass, which would fail every request + * merged into the same batch. */ +static int admit_input_tokens( + http_server *s, const loaded_model *m, token_buf *t, yyjson_mut_doc *detail, const char *loc) { + int cap = model_item_token_cap(m->info, s->max_batch_tokens); + if (t->n_tokens <= cap) + return 0; + if (s->auto_truncate) { + truncate_token_buf(t, cap, m->info->truncate_keep_tail); + return 0; + } + char msg[64]; + snprintf(msg, sizeof(msg), "input exceeds %d token limit", cap); + ve_add(detail, loc, msg, "value_error.context_length"); + return -1; +} + void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, embedding_request *out) { memset(out, 0, sizeof(*out)); out->j = j; @@ -353,13 +373,11 @@ void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, emb if (tokenize_input(m, j, single, query_instruct, &out->tokens[0]) != 0) { ve_add(detail, "[\"body\",\"input\"]", "tokenization failed", "value_error.tokenization"); - } else { + } else if (admit_input_tokens(s, m, &out->tokens[0], detail, "[\"body\",\"input\"]") == + 0) { out->inputs[0].ids = out->tokens[0].ids; out->inputs[0].n_tokens = out->tokens[0].n_tokens; out->total_tokens = out->tokens[0].n_tokens; - if (out->tokens[0].n_tokens > FFWD_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", - "value_error.context_length"); } } else { size_t idx, max; @@ -374,13 +392,10 @@ void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, emb } if (tokenize_input(m, j, text, query_instruct, &out->tokens[idx]) != 0) { ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); - } else { + } else if (admit_input_tokens(s, m, &out->tokens[idx], detail, loc) == 0) { out->inputs[idx].ids = out->tokens[idx].ids; out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; out->total_tokens += out->tokens[idx].n_tokens; - if (out->tokens[idx].n_tokens > FFWD_API_MAX_ITEM_TOKENS) - ve_add(detail, loc, "input exceeds 32768 token limit", - "value_error.context_length"); } } } @@ -474,6 +489,11 @@ void prepare_contextual_request(job *j, } if (ve_count(detail) == 0 && m) { + /* A packed document (chunks joined by separators) must fit the model + * and the batch budget like any other sequence. Truncation is not + * offered here: cutting the packed stream would drop chunks whose + * embeddings the response is required to contain. */ + int item_cap = model_item_token_cap(m->info, s->max_batch_tokens); out->n_docs = n_docs; out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); int64_t request_tokens = 0; @@ -501,11 +521,12 @@ void prepare_contextual_request(job *j, } } - if (doc_tokens > FFWD_API_MAX_ITEM_TOKENS) { + if (doc_tokens > item_cap) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", (int)di); - ve_add(detail, loc, "document exceeds 32768 token limit", - "value_error.context_length"); + char msg[64]; + snprintf(msg, sizeof(msg), "document exceeds %d token limit", item_cap); + ve_add(detail, loc, msg, "value_error.context_length"); doc_valid = 0; } request_tokens += doc_tokens; @@ -663,15 +684,20 @@ void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank } if (ve_count(detail) == 0 && out->model) { + /* Late-interaction sequences carry keep masks alongside the ids, so + * truncation is not offered; over-capacity inputs are rejected. */ + int item_cap = model_item_token_cap(out->model->info, s->max_batch_tokens); + char cap_msg[64]; out->n_documents = n_documents; out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) { ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization"); } else { out->query_tokens = out->query.n_tokens; - if (out->query_tokens > FFWD_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", - "value_error.context_length"); + if (out->query_tokens > item_cap) { + snprintf(cap_msg, sizeof(cap_msg), "query exceeds %d token limit", item_cap); + ve_add(detail, "[\"body\",\"query\"]", cap_msg, "value_error.context_length"); + } } size_t i, dmax; @@ -684,9 +710,10 @@ void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { int n = out->documents[i].n_tokens; - if (n > FFWD_API_MAX_ITEM_TOKENS) - ve_add(detail, loc, "document exceeds 32768 token limit", - "value_error.context_length"); + if (n > item_cap) { + snprintf(cap_msg, sizeof(cap_msg), "document exceeds %d token limit", item_cap); + ve_add(detail, loc, cap_msg, "value_error.context_length"); + } if (out->document_tokens <= INT_MAX - n) out->document_tokens += n; else diff --git a/tools/server/main.c b/tools/server/main.c index 4d79872..c0d01bc 100644 --- a/tools/server/main.c +++ b/tools/server/main.c @@ -39,9 +39,14 @@ static void print_usage(const char *prog) { " (default: 32)\n" " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" " --max-request-tokens N Max total tokens per API request (default: 120000)\n" + " --max-concurrent-requests N\n" + " In-flight request cap; excess requests get 429\n" + " (default: %d)\n" + " --auto-truncate Truncate inputs that exceed the per-input token\n" + " cap instead of rejecting them with 422\n" " --batch-wait-us N First-arrival micro-batch deadline in us\n" " (default: %d)\n", - prog, ffwd_default_batch_wait_us()); + prog, FFWD_SERVER_DEFAULT_MAX_CONCURRENT, ffwd_default_batch_wait_us()); ffwd_server_help(stderr); fprintf(stderr, " -v, --verbose Verbose (-vv for debug)\n" @@ -212,6 +217,8 @@ int main(int argc, char *argv[]) { int batch_size = 0; int max_batch_tokens = 0; int max_request_tokens = 0; + int max_concurrent_requests = 0; + int auto_truncate = 0; int batch_wait_us = -1; /* Backend tuning flags parse into opts regardless of backend; the linked * backend uses what it supports. */ @@ -288,6 +295,15 @@ int main(int argc, char *argv[]) { free_model_specs(&model_specs); return 1; } + } else if (!strcmp(f, "--max-concurrent-requests")) { + max_concurrent_requests = atoi(argv[++arg]); + if (max_concurrent_requests <= 0) { + fprintf(stderr, "--max-concurrent-requests must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--auto-truncate")) { + auto_truncate = 1; } else if (!strcmp(f, "--batch-wait-us")) { batch_wait_us = atoi(argv[++arg]); if (batch_wait_us < 0) { @@ -343,6 +359,8 @@ int main(int argc, char *argv[]) { .batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE, .max_batch_tokens = max_batch_tokens, .max_request_tokens = max_request_tokens, + .max_concurrent_requests = max_concurrent_requests, + .auto_truncate = auto_truncate, .batch_wait_us = batch_wait_us, .gpu_quantize_bits = opts.gpu_quant_bits, .gpu_quantize_group_size = opts.gpu_quant_group_size, diff --git a/tools/server/models.c b/tools/server/models.c index 2cb35a6..7420542 100644 --- a/tools/server/models.c +++ b/tools/server/models.c @@ -197,6 +197,31 @@ void rerank_request_free(rerank_request *r) { memset(r, 0, sizeof(*r)); } +/* Largest token count one input may carry for a model: the smallest of the + * model's positional capacity, the API item cap, and the per-batch token + * budget. Including the batch budget keeps --max-batch-tokens an honest bound + * on a single inference launch; without it a lone oversized input would run as + * a batch bigger than the configured budget. */ +int model_item_token_cap(const model_info *info, int max_batch_tokens) { + int cap = FFWD_API_MAX_ITEM_TOKENS; + if (info && info->max_seq_tokens > 0 && info->max_seq_tokens < cap) + cap = info->max_seq_tokens; + if (max_batch_tokens > 0 && max_batch_tokens < cap) + cap = max_batch_tokens; + return cap; +} + +/* Cut a tokenized input to cap tokens. When the layout ends with a required + * special token ([SEP], </s>, or the Qwen terminal), that token moves to the + * new end so the truncated sequence keeps a valid layout. */ +void truncate_token_buf(token_buf *t, int cap, int keep_tail) { + if (!t || !t->ids || cap <= 0 || t->n_tokens <= cap) + return; + if (keep_tail) + t->ids[cap - 1] = t->ids[t->n_tokens - 1]; + t->n_tokens = cap; +} + /* Thin wrappers that add server per-stage timing to the tokenization job. * Request handlers call them unchanged. */ int tokenize_input( @@ -277,6 +302,15 @@ int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int tok m->info->dim = config->hidden_size; m->info->min_dim = min_dim; m->info->token_dim = token_dim; + /* Positional capacity: BERT-family positions live in a learned table of + * max_position_embeddings rows starting at position_id_offset; RoPE models + * state a trained context length. 0 (absent) means no model-side cap. */ + m->info->max_seq_tokens = 0; + if (config->max_position_embeddings > 0) { + int cap = config->max_position_embeddings - config->position_id_offset; + m->info->max_seq_tokens = cap > 0 ? cap : 0; + } + m->info->truncate_keep_tail = config->family == FFWD_FAMILY_BERT || config->append_terminal_token; m->info->attention_mode = config->attention_mode; m->info->pooling_mode = config->pooling_mode; m->info->normalize_embeddings = config->normalize_embeddings; diff --git a/tools/server/schedule.c b/tools/server/schedule.c index 8254739..c71cad3 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -250,6 +250,8 @@ void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { job *j; while ((j = pop_done(s)) != NULL) { + /* Every admitted job ends here; release its admission slot. */ + s->inflight_jobs--; client *c = j->c; int refs_to_drop = 1; /* job reference */ if (!c->cancelled) { diff --git a/tools/server/server.c b/tools/server/server.c index 31eed36..9686019 100644 --- a/tools/server/server.c +++ b/tools/server/server.c @@ -94,6 +94,21 @@ void dispatch_request(client *c) { return; } + /* Admission control: beyond the in-flight cap, shed load with an immediate + * 429 instead of queueing without bound. The counter lives on the event + * loop thread (incremented here, decremented in completion_cb), so no lock + * is needed. */ + if (c->srv->inflight_jobs >= c->srv->max_concurrent_requests) { + size_t len; + char *body = json_error_body("server is overloaded", "overloaded_error", &len); + append_http_response(c, 429, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) + close_client(c); + return; + } + c->srv->inflight_jobs++; + job *j = xcalloc(1, sizeof(*j)); j->srv = c->srv; j->c = c; @@ -343,6 +358,9 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS; s.max_request_tokens = cfg->max_request_tokens > 0 ? cfg->max_request_tokens : FFWD_API_MAX_TOTAL_TOKENS; + s.max_concurrent_requests = cfg->max_concurrent_requests > 0 ? cfg->max_concurrent_requests + : FFWD_SERVER_DEFAULT_MAX_CONCURRENT; + s.auto_truncate = cfg->auto_truncate ? 1 : 0; s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us(); s.backend_opts = backend_opts; if (cfg->api_key && cfg->api_key[0]) @@ -460,6 +478,18 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { return 1; } + /* Capacity line per model, after worker init so a GPU build has finalized + * the config: the effective per-input token cap and the batch shape make + * the serving envelope visible at startup. */ + for (int i = 0; i < s.n_models; i++) { + loaded_model *m = &s.models[i]; + server_log("ffwd-server: model %s: max %d tokens per input%s, batch budget %d tokens x %d " + "rows, max %d concurrent requests", + m->info->id, model_item_token_cap(m->info, s.max_batch_tokens), + s.auto_truncate ? " (auto-truncate)" : "", s.max_batch_tokens, s.batch_size, + s.max_concurrent_requests); + } + /* Started after worker init so the tokenizer sees a finalized model config * (a GPU build sets dims/pooling on the worker, and request validation reads * them). Joined before the worker at shutdown so it drains raw jobs into the diff --git a/tools/server/server.h b/tools/server/server.h index 97fb57f..151a793 100644 --- a/tools/server/server.h +++ b/tools/server/server.h @@ -39,7 +39,9 @@ typedef struct { int port; int batch_size; int max_batch_tokens; - int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */ + int max_request_tokens; /* per-request total-token cap; 0 = default 120000 */ + int max_concurrent_requests; /* in-flight request cap before 429; 0 = default 512 */ + int auto_truncate; /* truncate over-long inputs instead of rejecting them */ int batch_wait_us; /* GPU build tuning; ignored by backends that do not use it. */ int gpu_quantize_bits; diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index d2829aa..51ef7dc 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -42,6 +42,9 @@ * was no faster and inflated long-document tail latency. */ #define FFWD_SERVER_DEFAULT_BATCH_SIZE 32 #define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 +/* In-flight request cap; dispatch answers 429 beyond it so overload sheds load + * instead of growing the job queues without bound. */ +#define FFWD_SERVER_DEFAULT_MAX_CONCURRENT 512 /* Microseconds the worker waits for more requests before dispatching a batch. * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU * busy; MLX and CPU dispatch immediately, where waiting only adds latency. @@ -65,6 +68,13 @@ typedef struct { int dim; int min_dim; int token_dim; + /* Positional capacity in tokens (max_position_embeddings minus the + * position-id offset); 0 when the config does not state one. Inputs past + * this fail in the forward pass, so admission enforces it up front. */ + int max_seq_tokens; + /* The token layout ends with a required special token ([SEP], </s>, or the + * Qwen terminal) that truncation must keep as the final token. */ + int truncate_keep_tail; ffwd_attention_mode_t attention_mode; ffwd_pooling_mode_t pooling_mode; int normalize_embeddings; @@ -92,7 +102,10 @@ typedef struct { int port; int batch_size; int max_batch_tokens; - int max_request_tokens; // per-request total-token cap (FFWD_API_MAX_TOTAL_TOKENS default) + int max_request_tokens; // per-request total-token cap (FFWD_API_MAX_TOTAL_TOKENS default) + int max_concurrent_requests; // in-flight request cap; dispatch answers 429 beyond it + int inflight_jobs; // loop-thread only: jobs created and not yet completed + int auto_truncate; // truncate over-long inputs instead of rejecting them int batch_wait_us; char *api_key; loaded_model *models; @@ -312,6 +325,8 @@ void embedding_request_free(embedding_request *r); void contextual_request_free(contextual_request *r); void late_tokens_free(late_tokens *t); void rerank_request_free(rerank_request *r); +int model_item_token_cap(const model_info *info, int max_batch_tokens); +void truncate_token_buf(token_buf *t, int cap, int keep_tail); /* Thin timing wrappers over libffwd's ffwd_tokenize/ffwd_tokenize_late. */ int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out); int tokenize_input( diff --git a/tools/server/test_models.c b/tools/server/test_models.c index 3dce62b..48a2da1 100644 --- a/tools/server/test_models.c +++ b/tools/server/test_models.c @@ -1,4 +1,5 @@ -/* test_models.c - resolving a served model by its request label. */ +/* test_models.c - resolving a served model by its request label, and the + * per-input token cap with its truncation helper. */ #include "server_internal.h" #include "test_util.h" @@ -20,7 +21,46 @@ static void test_loaded_model_lookup(void) { TEST_ASSERT(loaded_model_for_label(&s, "compiled-name") == NULL); } +static void test_model_item_token_cap(void) { + model_info info; + memset(&info, 0, sizeof(info)); + + /* No model cap: the API item cap applies, then the batch budget. */ + TEST_ASSERT(model_item_token_cap(&info, 0) == FFWD_API_MAX_ITEM_TOKENS); + TEST_ASSERT(model_item_token_cap(&info, 16384) == 16384); + + /* Model cap below both: it wins. */ + info.max_seq_tokens = 512; + TEST_ASSERT(model_item_token_cap(&info, 16384) == 512); + + /* Batch budget below the model cap: the budget wins. */ + info.max_seq_tokens = 32768; + TEST_ASSERT(model_item_token_cap(&info, 16384) == 16384); + TEST_ASSERT(model_item_token_cap(&info, 0) == 32768); +} + +static void test_truncate_token_buf(void) { + int ids[6] = {101, 7, 8, 9, 10, 102}; + token_buf t = {.ids = ids, .n_tokens = 6}; + + /* Under the cap: untouched. */ + truncate_token_buf(&t, 6, 1); + TEST_ASSERT(t.n_tokens == 6 && ids[5] == 102); + + /* keep_tail moves the final special token to the new end. */ + truncate_token_buf(&t, 4, 1); + TEST_ASSERT(t.n_tokens == 4); + TEST_ASSERT(ids[0] == 101 && ids[1] == 7 && ids[2] == 8 && ids[3] == 102); + + /* Plain cut when the layout has no required tail. */ + truncate_token_buf(&t, 2, 0); + TEST_ASSERT(t.n_tokens == 2); + TEST_ASSERT(ids[0] == 101 && ids[1] == 7); +} + int main(void) { test_loaded_model_lookup(); + test_model_item_token_cap(); + test_truncate_token_buf(); return TEST_REPORT("models"); } |
