diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 00:38:34 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 00:38:34 +0200 |
| commit | f182896ac7febe75d2ea81425782fc11d2117278 (patch) | |
| tree | ac05e50226abb62c5bc0124999dab019812befca | |
| parent | 8c1a141c9b7889fa2fa864a60d8773aa9630dc49 (diff) | |
| download | ffwd-f182896ac7febe75d2ea81425782fc11d2117278.tar.gz ffwd-f182896ac7febe75d2ea81425782fc11d2117278.zip | |
server: render inline when idle, defer only under backlog
| -rw-r--r-- | tools/server/handlers.c | 30 | ||||
| -rw-r--r-- | tools/server/schedule.c | 10 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 1 |
3 files changed, 35 insertions, 6 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c index 3d148b3..4ca3157 100644 --- a/tools/server/handlers.c +++ b/tools/server/handlers.c @@ -27,15 +27,32 @@ typedef struct { int output_index; } embedding_batch_item; -static int async_embedding_render_enabled(void) { - static int init = 0, enabled = 1; +/* Render placement (auto by default). JSON serialization is deferred to the + * renderer thread only when another batch is already queued for the worker, so + * the worker can start the next GPU launch while results serialize; when the + * queue is empty (single-stream), the worker renders inline and skips the + * thread handoff, which otherwise costs ~1-2% with nothing to overlap. + * FFWD_SERVER_ASYNC_RENDER=0 forces inline, =1 forces always-deferred (A/B). */ +static int async_render_mode(void) { + static int init = 0, mode = -1; /* -1 auto, 0 inline, 1 deferred */ if (!init) { const char *e = getenv("FFWD_SERVER_ASYNC_RENDER"); - if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false"))) - enabled = 0; + if (e) { + if (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")) + mode = 0; + else if (!strcmp(e, "1") || !strcmp(e, "on") || !strcmp(e, "true")) + mode = 1; + } init = 1; } - return enabled; + return mode; +} + +static int render_should_defer(http_server *s) { + int mode = async_render_mode(); + if (mode >= 0) + return mode; + return worker_has_pending_jobs(s); } static int embedding_batch_item_cmp(const void *a, const void *b) { @@ -114,12 +131,13 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { start += cur; } + int defer_render = render_should_defer(reqs[0]->j->srv); pos = 0; for (int i = 0; i < n_reqs; i++) { reqs[i]->j->infer_ns += infer_ns; if (failed) { job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); - } else if (!async_embedding_render_enabled()) { + } else if (!defer_render) { uint64_t t0 = nstime(); if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); diff --git a/tools/server/schedule.c b/tools/server/schedule.c index 37ded3b..fc63c91 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -144,6 +144,16 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) { return n; } +/* True when more jobs are already waiting for the worker. The render placement + * decision (handlers.c) uses this so it only hands a finished batch to the + * renderer thread when the worker has another batch to run. */ +int worker_has_pending_jobs(http_server *s) { + pthread_mutex_lock(&s->mu); + int pending = s->job_head != NULL; + pthread_mutex_unlock(&s->mu); + return pending; +} + void enqueue_job(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index 2f1067e..c79862f 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -324,6 +324,7 @@ void execute_rerank_request(rerank_request *r); /* ---- server_schedule.c: job queue, micro-batching, completion ---- */ void enqueue_job(job *j); +int worker_has_pending_jobs(http_server *s); void enqueue_render_job(job *j); job *dequeue_render_job(http_server *s); void finish_job(job *j); |
