diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-23 23:54:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-23 23:54:56 +0200 |
| commit | a4cfca86f2b3374fdb170023b29e86bef5d3e5d4 (patch) | |
| tree | e8c17ff7ba9fe66cb3ed7bfa05483cbb9dac1618 | |
| parent | f76b9fa8e52845eebb51552dcd1474b2a7e381b0 (diff) | |
| download | ffwd-a4cfca86f2b3374fdb170023b29e86bef5d3e5d4.tar.gz ffwd-a4cfca86f2b3374fdb170023b29e86bef5d3e5d4.zip | |
server: render embedding responses off worker
| -rw-r--r-- | tools/server/encode.c | 42 | ||||
| -rw-r--r-- | tools/server/handlers.c | 15 | ||||
| -rw-r--r-- | tools/server/schedule.c | 52 | ||||
| -rw-r--r-- | tools/server/server.c | 75 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 30 | ||||
| -rw-r--r-- | tools/server/test_encode.c | 40 |
6 files changed, 238 insertions, 16 deletions
diff --git a/tools/server/encode.c b/tools/server/encode.c index faa58a0..61ff4ea 100644 --- a/tools/server/encode.c +++ b/tools/server/encode.c @@ -129,6 +129,48 @@ int render_embedding_response(embedding_request *r, const float *embs) { return 0; } +void job_render_free(job *j) { + if (!j) + return; + free(j->embedding_render.encoding); + free(j->embedding_render.embs); + memset(&j->embedding_render, 0, sizeof(j->embedding_render)); + j->render_kind = 0; +} + +void job_set_embedding_render(job *j, const embedding_request *r, const float *embs) { + if (!j || !r || !r->model || !embs) + return; + job_render_free(j); + size_t count = (size_t)r->n_inputs * (size_t)r->model->info->dim; + j->embedding_render.model = r->model; + j->embedding_render.encoding = xstrdup(r->encoding); + j->embedding_render.embs = xmalloc(count * sizeof(*j->embedding_render.embs)); + memcpy(j->embedding_render.embs, embs, count * sizeof(*j->embedding_render.embs)); + j->embedding_render.dims = r->dims; + j->embedding_render.n_inputs = r->n_inputs; + j->embedding_render.total_tokens = r->total_tokens; + j->render_kind = 1; +} + +int render_job_response(job *j) { + if (!j || j->render_kind == 0) + return 0; + if (j->render_kind != 1 || !j->embedding_render.model || !j->embedding_render.encoding || + !j->embedding_render.embs) + return -1; + + embedding_request r; + memset(&r, 0, sizeof(r)); + r.j = j; + r.model = j->embedding_render.model; + r.dims = j->embedding_render.dims; + r.encoding = j->embedding_render.encoding; + r.n_inputs = j->embedding_render.n_inputs; + r.total_tokens = j->embedding_render.total_tokens; + return render_embedding_response(&r, j->embedding_render.embs); +} + void render_contextual_response(contextual_request *r, const float *embs) { sbuf b = {0}; sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); diff --git a/tools/server/handlers.c b/tools/server/handlers.c index 6720784..3d148b3 100644 --- a/tools/server/handlers.c +++ b/tools/server/handlers.c @@ -27,6 +27,17 @@ typedef struct { int output_index; } embedding_batch_item; +static int async_embedding_render_enabled(void) { + static int init = 0, enabled = 1; + if (!init) { + const char *e = getenv("FFWD_SERVER_ASYNC_RENDER"); + if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false"))) + enabled = 0; + init = 1; + } + return enabled; +} + static int embedding_batch_item_cmp(const void *a, const void *b) { const embedding_batch_item *ia = a; const embedding_batch_item *ib = b; @@ -108,11 +119,13 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { reqs[i]->j->infer_ns += infer_ns; if (failed) { job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); - } else { + } else if (!async_embedding_render_enabled()) { uint64_t t0 = nstime(); if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); reqs[i]->j->encode_ns += nstime() - t0; + } else { + job_set_embedding_render(reqs[i]->j, reqs[i], embs + (size_t)pos * dim); } pos += reqs[i]->n_inputs; } diff --git a/tools/server/schedule.c b/tools/server/schedule.c index c13e0cf..01538cc 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -36,6 +36,34 @@ static void enqueue_done(job *j) { } } +void enqueue_render_job(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->render_tail) + s->render_tail->next = j; + else + s->render_head = j; + s->render_tail = j; + pthread_cond_signal(&s->render_cv); + pthread_mutex_unlock(&s->mu); +} + +job *dequeue_render_job(http_server *s) { + pthread_mutex_lock(&s->mu); + while (!s->render_head && !s->render_stopping) + pthread_cond_wait(&s->render_cv, &s->mu); + + job *j = s->render_head; + if (j) { + s->render_head = j->next; + if (!s->render_head) + s->render_tail = NULL; + j->next = NULL; + } + pthread_mutex_unlock(&s->mu); + return j; +} + static job *pop_job_locked(http_server *s) { job *j = s->job_head; if (j) { @@ -143,6 +171,7 @@ static job *pop_done(http_server *s) { static void job_free(job *j) { if (!j) return; + job_render_free(j); free(j->body); free(j->auth); free(j->extra_headers); @@ -192,6 +221,11 @@ static void job_set_timing_header(job *j) { j->extra_headers = xstrdup(buf); } +void finish_job(job *j) { + job_set_timing_header(j); + enqueue_done(j); +} + static int parse_job_root(job *j, cJSON **out_root) { cJSON *detail = cJSON_CreateArray(); cJSON *root = parse_json_body(j, detail); @@ -260,24 +294,21 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) { if (!kind[i]) { process_unknown_job(jobs[i]); - job_set_timing_header(jobs[i]); - enqueue_done(jobs[i]); + finish_job(jobs[i]); done[i] = 1; continue; } if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) || (kind[i] == 3 && !rerank_reqs[i].ready)) { - job_set_timing_header(jobs[i]); - enqueue_done(jobs[i]); + finish_job(jobs[i]); done[i] = 1; continue; } if (kind[i] == 3) { execute_rerank_request(&rerank_reqs[i]); - job_set_timing_header(jobs[i]); - enqueue_done(jobs[i]); + finish_job(jobs[i]); done[i] = 1; } else if (kind[i] == 1) { int group_n = 1; @@ -297,8 +328,10 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) { execute_embedding_request_list(std_group, group_n); for (int k = 0; k < group_n; k++) { int idx = (int)(std_group[k] - std_reqs); - job_set_timing_header(jobs[idx]); - enqueue_done(jobs[idx]); + if (jobs[idx]->render_kind) + enqueue_render_job(jobs[idx]); + else + finish_job(jobs[idx]); done[idx] = 1; } } else { @@ -319,8 +352,7 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) { execute_contextual_request_list(ctx_group, group_n); for (int k = 0; k < group_n; k++) { int idx = (int)(ctx_group[k] - ctx_reqs); - job_set_timing_header(jobs[idx]); - enqueue_done(jobs[idx]); + finish_job(jobs[idx]); done[idx] = 1; } } diff --git a/tools/server/server.c b/tools/server/server.c index abff2c2..8762c4f 100644 --- a/tools/server/server.c +++ b/tools/server/server.c @@ -215,6 +215,23 @@ static void *worker_main(void *arg) { return NULL; } +static void *renderer_main(void *arg) { + http_server *s = arg; + for (;;) { + job *j = dequeue_render_job(s); + if (!j) + break; + + uint64_t t0 = nstime(); + if (render_job_response(j) != 0) + job_set_error(j, 500, "embedding normalization failed", "server_error"); + j->encode_ns += nstime() - t0; + job_render_free(j); + finish_job(j); + } + return NULL; +} + static int listen_on(const char *host, int port) { char portbuf[32]; snprintf(portbuf, sizeof(portbuf), "%d", port); @@ -310,10 +327,15 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { pthread_mutex_init(&s.mu, NULL); pthread_cond_init(&s.cv, NULL); + pthread_cond_init(&s.render_cv, NULL); for (int i = 0; i < cfg->n_models; i++) { if (load_one_model(&s, &cfg->models[i]) != 0) { free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); return 1; } } @@ -321,6 +343,10 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { perror("pipe"); free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); return 1; } set_nonblock(s.completion_pipe[0]); @@ -341,13 +367,37 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { if (!s.loop) { fprintf(stderr, "ffwd-server: failed to create event loop\n"); free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 1; + } + + if (pthread_create(&s.renderer, NULL, renderer_main, &s) != 0) { + fprintf(stderr, "ffwd-server: failed to start renderer\n"); + aeDeleteEventLoop(s.loop); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); return 1; } if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { fprintf(stderr, "ffwd-server: failed to start worker\n"); + pthread_mutex_lock(&s.mu); + s.render_stopping = 1; + pthread_cond_signal(&s.render_cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.renderer, NULL); aeDeleteEventLoop(s.loop); free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); return 1; } pthread_mutex_lock(&s.mu); @@ -356,10 +406,20 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { int worker_init_rc = s.worker_init_rc; pthread_mutex_unlock(&s.mu); if (worker_init_rc) { + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); pthread_join(s.worker, NULL); + pthread_mutex_lock(&s.mu); + s.render_stopping = 1; + pthread_cond_signal(&s.render_cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.renderer, NULL); aeDeleteEventLoop(s.loop); free_models(&s); free(s.api_key); + pthread_cond_destroy(&s.render_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -372,8 +432,17 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { pthread_cond_signal(&s.cv); pthread_mutex_unlock(&s.mu); pthread_join(s.worker, NULL); + pthread_mutex_lock(&s.mu); + s.render_stopping = 1; + pthread_cond_signal(&s.render_cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.renderer, NULL); aeDeleteEventLoop(s.loop); free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); return 1; } @@ -400,6 +469,11 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { pthread_cond_signal(&s.cv); pthread_mutex_unlock(&s.mu); pthread_join(s.worker, NULL); + pthread_mutex_lock(&s.mu); + s.render_stopping = 1; + pthread_cond_signal(&s.render_cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.renderer, NULL); aeDeleteEventLoop(s.loop); close(s.completion_pipe[0]); @@ -408,6 +482,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { close(s.signal_pipe[1]); free_models(&s); free(s.api_key); + pthread_cond_destroy(&s.render_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 0; diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index 91b51c5..2f1067e 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -74,8 +74,8 @@ typedef struct { typedef struct { model_info *info; char *path; - ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */ - ffwd_t *backend; /* compute handle */ + ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */ + ffwd_t *backend; /* compute handle */ int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */ } loaded_model; @@ -105,6 +105,11 @@ typedef struct { int worker_init_rc; job *job_head; job *job_tail; + pthread_t renderer; + pthread_cond_t render_cv; + int render_stopping; + job *render_head; + job *render_tail; job *done_head; job *done_tail; @@ -159,6 +164,15 @@ struct job { uint64_t tokenize_ns; uint64_t infer_ns; uint64_t encode_ns; + int render_kind; + struct { + loaded_model *model; + char *encoding; + float *embs; + int dims; + int n_inputs; + int total_tokens; + } embedding_render; job *next; }; @@ -260,6 +274,9 @@ void append_embedding_value( void set_response_from_buf(job *j, sbuf *b); int render_embedding_response(embedding_request *r, const float *embs); void render_contextual_response(contextual_request *r, const float *embs); +void job_render_free(job *j); +void job_set_embedding_render(job *j, const embedding_request *r, const float *embs); +int render_job_response(job *j); /* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */ loaded_model *loaded_model_for_label(http_server *s, const char *label); @@ -275,9 +292,9 @@ int tokenize_input( int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out); int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out); int model_ffwd_spans_batch(loaded_model *m, - const ffwd_context_input_t *inputs, - int batch, - float *out); + const ffwd_context_input_t *inputs, + int batch, + float *out); int inference_batch_accepts_input( const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens); int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim); @@ -307,6 +324,9 @@ void execute_rerank_request(rerank_request *r); /* ---- server_schedule.c: job queue, micro-batching, completion ---- */ void enqueue_job(job *j); +void enqueue_render_job(job *j); +job *dequeue_render_job(http_server *s); +void finish_job(job *j); int collect_job_batch(http_server *s, job **jobs, int max_jobs); void process_job_group(http_server *s, job **jobs, int n_jobs); void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask); diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c index a7db9c4..91a4479 100644 --- a/tools/server/test_encode.c +++ b/tools/server/test_encode.c @@ -80,11 +80,51 @@ static void test_append_embedding_value_float(void) { sbuf_free(&b2); } +static void test_job_embedding_render_payload(void) { + model_info info; + memset(&info, 0, sizeof(info)); + info.id = "m"; + info.dim = 2; + info.api = FFWD_API_OPENAI; + + loaded_model model; + memset(&model, 0, sizeof(model)); + model.info = &info; + + job j; + memset(&j, 0, sizeof(j)); + embedding_request r; + memset(&r, 0, sizeof(r)); + r.j = &j; + r.model = &model; + r.dims = 2; + r.encoding = "float"; + r.n_inputs = 2; + r.total_tokens = 7; + + float embs[4] = {0.5f, 1.0f, -0.25f, 0.0f}; + job_set_embedding_render(&j, &r, embs); + TEST_ASSERT(j.render_kind == 1); + TEST_ASSERT(j.embedding_render.embs != embs); + TEST_ASSERT(render_job_response(&j) == 0); + TEST_ASSERT(j.status == 200); + TEST_ASSERT(j.response != NULL); + TEST_ASSERT(strstr(j.response, "\"model\":\"m\"") != NULL); + TEST_ASSERT(strstr(j.response, "\"prompt_tokens\":7") != NULL); + TEST_ASSERT(strstr(j.response, "\"embedding\":[0.5,1]") != NULL); + TEST_ASSERT(strstr(j.response, "\"embedding\":[-0.25,0]") != NULL); + + free(j.response); + j.response = NULL; + job_render_free(&j); +} + int main(void) { test_quantize_int8_tanh(); test_encode_embedding_int8(); test_encode_embedding_binary(); test_encode_embedding_base64_float32(); test_append_embedding_value_float(); + test_job_embedding_render_payload(); return TEST_REPORT("encode"); } |
