aboutsummaryrefslogtreecommitdiff
path: root/tools/server/handlers.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-23 23:54:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-23 23:54:56 +0200
commita4cfca86f2b3374fdb170023b29e86bef5d3e5d4 (patch)
treee8c17ff7ba9fe66cb3ed7bfa05483cbb9dac1618 /tools/server/handlers.c
parentf76b9fa8e52845eebb51552dcd1474b2a7e381b0 (diff)
downloadffwd-a4cfca86f2b3374fdb170023b29e86bef5d3e5d4.tar.gz
ffwd-a4cfca86f2b3374fdb170023b29e86bef5d3e5d4.zip
server: render embedding responses off worker
Diffstat (limited to 'tools/server/handlers.c')
-rw-r--r--tools/server/handlers.c15
1 files changed, 14 insertions, 1 deletions
diff --git a/tools/server/handlers.c b/tools/server/handlers.c
index 6720784..3d148b3 100644
--- a/tools/server/handlers.c
+++ b/tools/server/handlers.c
@@ -27,6 +27,17 @@ typedef struct {
int output_index;
} embedding_batch_item;
+static int async_embedding_render_enabled(void) {
+ static int init = 0, enabled = 1;
+ if (!init) {
+ const char *e = getenv("FFWD_SERVER_ASYNC_RENDER");
+ if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
+ enabled = 0;
+ init = 1;
+ }
+ return enabled;
+}
+
static int embedding_batch_item_cmp(const void *a, const void *b) {
const embedding_batch_item *ia = a;
const embedding_batch_item *ib = b;
@@ -108,11 +119,13 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
reqs[i]->j->infer_ns += infer_ns;
if (failed) {
job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
- } else {
+ } else if (!async_embedding_render_enabled()) {
uint64_t t0 = nstime();
if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
reqs[i]->j->encode_ns += nstime() - t0;
+ } else {
+ job_set_embedding_render(reqs[i]->j, reqs[i], embs + (size_t)pos * dim);
}
pos += reqs[i]->n_inputs;
}