aboutsummaryrefslogtreecommitdiff
path: root/tools/server
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-24 00:38:34 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-24 00:38:34 +0200
commitf182896ac7febe75d2ea81425782fc11d2117278 (patch)
treeac05e50226abb62c5bc0124999dab019812befca /tools/server
parent8c1a141c9b7889fa2fa864a60d8773aa9630dc49 (diff)
downloadffwd-f182896ac7febe75d2ea81425782fc11d2117278.tar.gz
ffwd-f182896ac7febe75d2ea81425782fc11d2117278.zip
server: render inline when idle, defer only under backlog
Diffstat (limited to 'tools/server')
-rw-r--r--tools/server/handlers.c30
-rw-r--r--tools/server/schedule.c10
-rw-r--r--tools/server/server_internal.h1
3 files changed, 35 insertions, 6 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c
index 3d148b3..4ca3157 100644
--- a/tools/server/handlers.c
+++ b/tools/server/handlers.c
@@ -27,15 +27,32 @@ typedef struct {
int output_index;
} embedding_batch_item;
-static int async_embedding_render_enabled(void) {
- static int init = 0, enabled = 1;
+/* Render placement (auto by default). JSON serialization is deferred to the
+ * renderer thread only when another batch is already queued for the worker, so
+ * the worker can start the next GPU launch while results serialize; when the
+ * queue is empty (single-stream), the worker renders inline and skips the
+ * thread handoff, which otherwise costs ~1-2% with nothing to overlap.
+ * FFWD_SERVER_ASYNC_RENDER=0 forces inline, =1 forces always-deferred (A/B). */
+static int async_render_mode(void) {
+ static int init = 0, mode = -1; /* -1 auto, 0 inline, 1 deferred */
if (!init) {
const char *e = getenv("FFWD_SERVER_ASYNC_RENDER");
- if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
- enabled = 0;
+ if (e) {
+ if (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false"))
+ mode = 0;
+ else if (!strcmp(e, "1") || !strcmp(e, "on") || !strcmp(e, "true"))
+ mode = 1;
+ }
init = 1;
}
- return enabled;
+ return mode;
+}
+
+static int render_should_defer(http_server *s) {
+ int mode = async_render_mode();
+ if (mode >= 0)
+ return mode;
+ return worker_has_pending_jobs(s);
}
static int embedding_batch_item_cmp(const void *a, const void *b) {
@@ -114,12 +131,13 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
start += cur;
}
+ int defer_render = render_should_defer(reqs[0]->j->srv);
pos = 0;
for (int i = 0; i < n_reqs; i++) {
reqs[i]->j->infer_ns += infer_ns;
if (failed) {
job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
- } else if (!async_embedding_render_enabled()) {
+ } else if (!defer_render) {
uint64_t t0 = nstime();
if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index 37ded3b..fc63c91 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -144,6 +144,16 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
return n;
}
+/* True when more jobs are already waiting for the worker. The render placement
+ * decision (handlers.c) uses this so it only hands a finished batch to the
+ * renderer thread when the worker has another batch to run. */
+int worker_has_pending_jobs(http_server *s) {
+ pthread_mutex_lock(&s->mu);
+ int pending = s->job_head != NULL;
+ pthread_mutex_unlock(&s->mu);
+ return pending;
+}
+
void enqueue_job(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index 2f1067e..c79862f 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -324,6 +324,7 @@ void execute_rerank_request(rerank_request *r);
/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
void enqueue_job(job *j);
+int worker_has_pending_jobs(http_server *s);
void enqueue_render_job(job *j);
job *dequeue_render_job(http_server *s);
void finish_job(job *j);