diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 00:53:21 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-24 00:53:21 +0200 |
| commit | 214d6aa4e641296dba6040ab776d91baefdca82e (patch) | |
| tree | cd185930075edf792fc014d30ce6df9c87a36418 /tools | |
| parent | f182896ac7febe75d2ea81425782fc11d2117278 (diff) | |
| download | ffwd-214d6aa4e641296dba6040ab776d91baefdca82e.tar.gz ffwd-214d6aa4e641296dba6040ab776d91baefdca82e.zip | |
server: tokenize requests off the worker thread
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/server/schedule.c | 124 | ||||
| -rw-r--r-- | tools/server/server.c | 89 | ||||
| -rw-r--r-- | tools/server/server_internal.h | 19 |
3 files changed, 209 insertions, 23 deletions
diff --git a/tools/server/schedule.c b/tools/server/schedule.c index fc63c91..34db038 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -65,6 +65,34 @@ job *dequeue_render_job(http_server *s) { return j; } +void enqueue_raw_job(job *j) { + http_server *s = j->srv; + pthread_mutex_lock(&s->mu); + if (s->raw_tail) + s->raw_tail->next = j; + else + s->raw_head = j; + s->raw_tail = j; + pthread_cond_signal(&s->raw_cv); + pthread_mutex_unlock(&s->mu); +} + +job *dequeue_raw_job(http_server *s) { + pthread_mutex_lock(&s->mu); + while (!s->raw_head && !s->raw_stopping) + pthread_cond_wait(&s->raw_cv, &s->mu); + + job *j = s->raw_head; + if (j) { + s->raw_head = j->next; + if (!s->raw_head) + s->raw_tail = NULL; + j->next = NULL; + } + pthread_mutex_unlock(&s->mu); + return j; +} + static job *pop_job_locked(http_server *s) { job *j = s->job_head; if (j) { @@ -179,9 +207,26 @@ static job *pop_done(http_server *s) { return j; } +/* Free a tokenizer-stage request still attached to a job. In the normal flow + * the worker moves *prep out and clears it before the job is freed; this only + * fires if a tokenized job is discarded without being processed. */ +static void job_prep_free(job *j) { + if (!j->prep) + return; + if (j->prep_kind == 1) + embedding_request_free(j->prep); + else if (j->prep_kind == 2) + contextual_request_free(j->prep); + else if (j->prep_kind == 3) + rerank_request_free(j->prep); + free(j->prep); + j->prep = NULL; +} + static void job_free(job *j) { if (!j) return; + job_prep_free(j); job_render_free(j); free(j->body); free(j->auth); @@ -264,6 +309,46 @@ static void process_unknown_job(job *j) { cJSON_Delete(root); } +/* Parse and tokenize one job into a heap request attached to j->prep. Runs on + * the tokenizer thread when tokenize-off-worker is on, or inline on the worker + * otherwise; the worker moves *prep into its request array. An unknown endpoint + * leaves prep_kind 0 (process_unknown_job handles it); a parse failure leaves + * prep NULL with the 422 already set, so the worker finishes it as not-ready. */ +void tokenize_job(http_server *s, job *j) { + j->tokenized = 1; + if (!j->started_ns) + j->started_ns = nstime(); + if (!strcmp(j->path, "/v1/embeddings")) + j->prep_kind = 1; + else if (!strcmp(j->path, "/v1/contextualizedembeddings")) + j->prep_kind = 2; + else if (!strcmp(j->path, "/v1/rerank")) + j->prep_kind = 3; + else + return; + + cJSON *root = NULL; + uint64_t t0 = nstime(); + int parse_rc = parse_job_root(j, &root); + j->parse_ns += nstime() - t0; + if (parse_rc != 0) + return; + + if (j->prep_kind == 1) { + embedding_request *r = xcalloc(1, sizeof(*r)); + prepare_embedding_request(j, root, s, r); + j->prep = r; + } else if (j->prep_kind == 2) { + contextual_request *r = xcalloc(1, sizeof(*r)); + prepare_contextual_request(j, root, s, r); + j->prep = r; + } else { + rerank_request *r = xcalloc(1, sizeof(*r)); + prepare_rerank_request(j, root, s, r); + j->prep = r; + } +} + /* Cross-request batch merging (default on). When enabled, every compatible * ready request in the arrival window is flattened into one pool and re-packed * into token-budget batches by execute_*_request_list, so mixed client batch @@ -291,30 +376,25 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) { embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group)); + /* Tokenize on the worker only for jobs the tokenizer stage did not already + * prepare (tokenize-off-worker disabled), then move each prepared request + * out of its job into the contiguous array the grouping below indexes. The + * move transfers the heap pointers; freeing the wrapper leaves the array + * slot owning them, freed once at the end. */ for (int i = 0; i < n_jobs; i++) { - cJSON *root = NULL; - if (!jobs[i]->started_ns) - jobs[i]->started_ns = nstime(); - if (!strcmp(jobs[i]->path, "/v1/embeddings")) { - kind[i] = 1; - } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { - kind[i] = 2; - } else if (!strcmp(jobs[i]->path, "/v1/rerank")) { - kind[i] = 3; - } else { - continue; + if (!jobs[i]->tokenized) + tokenize_job(s, jobs[i]); + kind[i] = jobs[i]->prep_kind; + if (jobs[i]->prep) { + if (kind[i] == 1) + std_reqs[i] = *(embedding_request *)jobs[i]->prep; + else if (kind[i] == 2) + ctx_reqs[i] = *(contextual_request *)jobs[i]->prep; + else if (kind[i] == 3) + rerank_reqs[i] = *(rerank_request *)jobs[i]->prep; + free(jobs[i]->prep); + jobs[i]->prep = NULL; } - uint64_t t0 = nstime(); - int parse_rc = parse_job_root(jobs[i], &root); - jobs[i]->parse_ns += nstime() - t0; - if (parse_rc != 0) - continue; - if (kind[i] == 1) - prepare_embedding_request(jobs[i], root, s, &std_reqs[i]); - else if (kind[i] == 2) - prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]); - else - prepare_rerank_request(jobs[i], root, s, &rerank_reqs[i]); } for (int i = 0; i < n_jobs; i++) { diff --git a/tools/server/server.c b/tools/server/server.c index 8762c4f..63438b9 100644 --- a/tools/server/server.c +++ b/tools/server/server.c @@ -66,6 +66,17 @@ static bool auth_ok(http_server *s, const char *auth) { !strcmp(auth + sizeof(prefix) - 1, s->api_key); } +/* Tokenize-off-worker (default on). When set, dispatch hands raw jobs to the + * tokenizer thread so parsing and tokenization overlap the GPU instead of + * running in front of it on the worker. FFWD_SERVER_TOKENIZE_OFF_WORKER=0 + * keeps tokenization on the worker for an A/B comparison. */ +static int tokenize_off_worker_default(void) { + const char *e = getenv("FFWD_SERVER_TOKENIZE_OFF_WORKER"); + if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false"))) + return 0; + return 1; +} + void dispatch_request(client *c) { aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); @@ -106,7 +117,10 @@ void dispatch_request(client *c) { j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; j->created_ns = nstime(); client_incref(c); - enqueue_job(j); + if (c->srv->tokenize_off_worker) + enqueue_raw_job(j); + else + enqueue_job(j); } static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { @@ -215,6 +229,30 @@ static void *worker_main(void *arg) { return NULL; } +/* Tokenizer stage: parse + tokenize raw jobs off the worker, then feed the + * worker's job queue. Waits for the worker to finish model init first, because + * a GPU build finalizes the model config (dims, pooling) on the worker and + * request validation reads it. Idle when tokenize-off-worker is disabled. */ +static void *tokenizer_main(void *arg) { + http_server *s = arg; + pthread_mutex_lock(&s->mu); + while (!s->worker_ready) + pthread_cond_wait(&s->cv, &s->mu); + int rc = s->worker_init_rc; + pthread_mutex_unlock(&s->mu); + if (rc) + return NULL; + + for (;;) { + job *j = dequeue_raw_job(s); + if (!j) + break; + tokenize_job(s, j); + enqueue_job(j); + } + return NULL; +} + static void *renderer_main(void *arg) { http_server *s = arg; for (;;) { @@ -316,6 +354,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { s.max_batch_tokens = cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS; s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us(); + s.tokenize_off_worker = tokenize_off_worker_default(); s.backend_opts = backend_opts; if (cfg->api_key && cfg->api_key[0]) s.api_key = xstrdup(cfg->api_key); @@ -327,6 +366,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { pthread_mutex_init(&s.mu, NULL); pthread_cond_init(&s.cv, NULL); + pthread_cond_init(&s.raw_cv, NULL); pthread_cond_init(&s.render_cv, NULL); for (int i = 0; i < cfg->n_models; i++) { @@ -334,6 +374,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -345,6 +386,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -369,6 +411,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -380,6 +423,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -396,6 +440,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -420,6 +465,33 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 1; + } + + /* Started after worker init so the tokenizer sees a finalized model config + * (a GPU build sets dims/pooling on the worker, and request validation reads + * them). Joined before the worker at shutdown so it drains raw jobs into the + * job queue first. */ + if (pthread_create(&s.tokenizer, NULL, tokenizer_main, &s) != 0) { + fprintf(stderr, "ffwd-server: failed to start tokenizer\n"); + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + pthread_mutex_lock(&s.mu); + s.render_stopping = 1; + pthread_cond_signal(&s.render_cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.renderer, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -428,6 +500,11 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { s.listen_fd = listen_on(s.host, s.port); if (s.listen_fd < 0) { pthread_mutex_lock(&s.mu); + s.raw_stopping = 1; + pthread_cond_signal(&s.raw_cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.tokenizer, NULL); + pthread_mutex_lock(&s.mu); s.stopping = 1; pthread_cond_signal(&s.cv); pthread_mutex_unlock(&s.mu); @@ -441,6 +518,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 1; @@ -464,6 +542,14 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { c = next; } + /* Drain stages front to back: the tokenizer flushes raw jobs into the job + * queue, then the worker drains that, then the renderer drains finished + * jobs, so nothing is stranded mid-pipeline. */ + pthread_mutex_lock(&s.mu); + s.raw_stopping = 1; + pthread_cond_signal(&s.raw_cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.tokenizer, NULL); pthread_mutex_lock(&s.mu); s.stopping = 1; pthread_cond_signal(&s.cv); @@ -483,6 +569,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) { free_models(&s); free(s.api_key); pthread_cond_destroy(&s.render_cv); + pthread_cond_destroy(&s.raw_cv); pthread_cond_destroy(&s.cv); pthread_mutex_destroy(&s.mu); return 0; diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index c79862f..3ec4e8d 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -105,6 +105,15 @@ typedef struct { int worker_init_rc; job *job_head; job *job_tail; + /* Tokenizer stage: when tokenize_off_worker is set, dispatch enqueues raw + * jobs here and a dedicated thread parses + tokenizes them off the worker so + * tokenization overlaps the GPU instead of serializing in front of it. */ + int tokenize_off_worker; + pthread_t tokenizer; + pthread_cond_t raw_cv; + int raw_stopping; + job *raw_head; + job *raw_tail; pthread_t renderer; pthread_cond_t render_cv; int render_stopping; @@ -158,6 +167,13 @@ struct job { char *extra_headers; char *response; size_t response_len; + /* Filled by the tokenizer stage (tokenize_job): prep points at a heap + * request of type prep_kind (1 embedding, 2 contextual, 3 rerank), which the + * worker moves into its request array. tokenized guards re-tokenizing when + * the worker runs tokenize_job inline (tokenize-off-worker disabled). */ + int tokenized; + int prep_kind; + void *prep; uint64_t created_ns; uint64_t started_ns; uint64_t parse_ns; @@ -325,6 +341,9 @@ void execute_rerank_request(rerank_request *r); /* ---- server_schedule.c: job queue, micro-batching, completion ---- */ void enqueue_job(job *j); int worker_has_pending_jobs(http_server *s); +void enqueue_raw_job(job *j); +job *dequeue_raw_job(http_server *s); +void tokenize_job(http_server *s, job *j); void enqueue_render_job(job *j); job *dequeue_render_job(http_server *s); void finish_job(job *j); |
