diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 01:51:53 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 01:51:53 +0200 |
| commit | cb85a499f3d5178a752c85063bf9ff277ca3a71e (patch) | |
| tree | 78f68836e954d08dd6409e6d56cf44a7fb3c35e7 /tools/server/handlers.c | |
| parent | 716d2051de27123bb298dbe2536c23537ec9512d (diff) | |
| download | ffwd-cb85a499f3d5178a752c85063bf9ff277ca3a71e.tar.gz ffwd-cb85a499f3d5178a752c85063bf9ff277ca3a71e.zip | |
server: make tokenize/render overlap adaptive, drop env switches
Diffstat (limited to 'tools/server/handlers.c')
| -rw-r--r-- | tools/server/handlers.c | 53 |
1 files changed, 5 insertions, 48 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c index 4ca3157..fed4671 100644 --- a/tools/server/handlers.c +++ b/tools/server/handlers.c @@ -27,33 +27,11 @@ typedef struct { int output_index; } embedding_batch_item; -/* Render placement (auto by default). JSON serialization is deferred to the - * renderer thread only when another batch is already queued for the worker, so - * the worker can start the next GPU launch while results serialize; when the - * queue is empty (single-stream), the worker renders inline and skips the - * thread handoff, which otherwise costs ~1-2% with nothing to overlap. - * FFWD_SERVER_ASYNC_RENDER=0 forces inline, =1 forces always-deferred (A/B). */ -static int async_render_mode(void) { - static int init = 0, mode = -1; /* -1 auto, 0 inline, 1 deferred */ - if (!init) { - const char *e = getenv("FFWD_SERVER_ASYNC_RENDER"); - if (e) { - if (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")) - mode = 0; - else if (!strcmp(e, "1") || !strcmp(e, "on") || !strcmp(e, "true")) - mode = 1; - } - init = 1; - } - return mode; -} - -static int render_should_defer(http_server *s) { - int mode = async_render_mode(); - if (mode >= 0) - return mode; - return worker_has_pending_jobs(s); -} +/* Hand JSON serialization to the renderer thread only when another batch is + * already queued, so the worker can start that GPU launch while these results + * serialize. With nothing else queued the worker renders inline, which avoids a + * thread hand-off that buys no overlap and adds latency to the reply. */ +static int render_should_defer(http_server *s) { return worker_has_pending_jobs(s); } static int embedding_batch_item_cmp(const void *a, const void *b) { const embedding_batch_item *ia = a; @@ -65,17 +43,6 @@ static int embedding_batch_item_cmp(const void *a, const void *b) { return ia->output_index - ib->output_index; } -int embedding_request_fits_group(const embedding_request *r, - int group_inputs, - int group_tokens, - int max_batch, - int max_batch_tokens) { - if (group_inputs == 0) - return 1; - return r->n_inputs <= max_batch - group_inputs && - r->total_tokens <= max_batch_tokens - group_tokens; -} - void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { if (n_reqs <= 0) return; @@ -159,16 +126,6 @@ int contextual_request_compatible(const contextual_request *a, const contextual_ !strcmp(a->encoding, b->encoding); } -int contextual_request_fits_group(const contextual_request *r, - int group_docs, - int group_tokens, - int max_batch, - int max_batch_tokens) { - if (group_docs == 0) - return 1; - return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens; -} - typedef struct { ffwd_context_input_t input; int output_span_index; |
