aboutsummaryrefslogtreecommitdiff
path: root/tools/server
diff options
context:
space:
mode:
Diffstat (limited to 'tools/server')
-rw-r--r--tools/server/encode.c42
-rw-r--r--tools/server/handlers.c15
-rw-r--r--tools/server/schedule.c52
-rw-r--r--tools/server/server.c75
-rw-r--r--tools/server/server_internal.h30
-rw-r--r--tools/server/test_encode.c40
6 files changed, 238 insertions, 16 deletions
diff --git a/tools/server/encode.c b/tools/server/encode.c
index faa58a0..61ff4ea 100644
--- a/tools/server/encode.c
+++ b/tools/server/encode.c
@@ -129,6 +129,48 @@ int render_embedding_response(embedding_request *r, const float *embs) {
return 0;
}
+void job_render_free(job *j) {
+ if (!j)
+ return;
+ free(j->embedding_render.encoding);
+ free(j->embedding_render.embs);
+ memset(&j->embedding_render, 0, sizeof(j->embedding_render));
+ j->render_kind = 0;
+}
+
+void job_set_embedding_render(job *j, const embedding_request *r, const float *embs) {
+ if (!j || !r || !r->model || !embs)
+ return;
+ job_render_free(j);
+ size_t count = (size_t)r->n_inputs * (size_t)r->model->info->dim;
+ j->embedding_render.model = r->model;
+ j->embedding_render.encoding = xstrdup(r->encoding);
+ j->embedding_render.embs = xmalloc(count * sizeof(*j->embedding_render.embs));
+ memcpy(j->embedding_render.embs, embs, count * sizeof(*j->embedding_render.embs));
+ j->embedding_render.dims = r->dims;
+ j->embedding_render.n_inputs = r->n_inputs;
+ j->embedding_render.total_tokens = r->total_tokens;
+ j->render_kind = 1;
+}
+
+int render_job_response(job *j) {
+ if (!j || j->render_kind == 0)
+ return 0;
+ if (j->render_kind != 1 || !j->embedding_render.model || !j->embedding_render.encoding ||
+ !j->embedding_render.embs)
+ return -1;
+
+ embedding_request r;
+ memset(&r, 0, sizeof(r));
+ r.j = j;
+ r.model = j->embedding_render.model;
+ r.dims = j->embedding_render.dims;
+ r.encoding = j->embedding_render.encoding;
+ r.n_inputs = j->embedding_render.n_inputs;
+ r.total_tokens = j->embedding_render.total_tokens;
+ return render_embedding_response(&r, j->embedding_render.embs);
+}
+
void render_contextual_response(contextual_request *r, const float *embs) {
sbuf b = {0};
sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
diff --git a/tools/server/handlers.c b/tools/server/handlers.c
index 6720784..3d148b3 100644
--- a/tools/server/handlers.c
+++ b/tools/server/handlers.c
@@ -27,6 +27,17 @@ typedef struct {
int output_index;
} embedding_batch_item;
+static int async_embedding_render_enabled(void) {
+ static int init = 0, enabled = 1;
+ if (!init) {
+ const char *e = getenv("FFWD_SERVER_ASYNC_RENDER");
+ if (e && (!strcmp(e, "0") || !strcmp(e, "off") || !strcmp(e, "false")))
+ enabled = 0;
+ init = 1;
+ }
+ return enabled;
+}
+
static int embedding_batch_item_cmp(const void *a, const void *b) {
const embedding_batch_item *ia = a;
const embedding_batch_item *ib = b;
@@ -108,11 +119,13 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
reqs[i]->j->infer_ns += infer_ns;
if (failed) {
job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
- } else {
+ } else if (!async_embedding_render_enabled()) {
uint64_t t0 = nstime();
if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
reqs[i]->j->encode_ns += nstime() - t0;
+ } else {
+ job_set_embedding_render(reqs[i]->j, reqs[i], embs + (size_t)pos * dim);
}
pos += reqs[i]->n_inputs;
}
diff --git a/tools/server/schedule.c b/tools/server/schedule.c
index c13e0cf..01538cc 100644
--- a/tools/server/schedule.c
+++ b/tools/server/schedule.c
@@ -36,6 +36,34 @@ static void enqueue_done(job *j) {
}
}
+void enqueue_render_job(job *j) {
+ http_server *s = j->srv;
+ pthread_mutex_lock(&s->mu);
+ if (s->render_tail)
+ s->render_tail->next = j;
+ else
+ s->render_head = j;
+ s->render_tail = j;
+ pthread_cond_signal(&s->render_cv);
+ pthread_mutex_unlock(&s->mu);
+}
+
+job *dequeue_render_job(http_server *s) {
+ pthread_mutex_lock(&s->mu);
+ while (!s->render_head && !s->render_stopping)
+ pthread_cond_wait(&s->render_cv, &s->mu);
+
+ job *j = s->render_head;
+ if (j) {
+ s->render_head = j->next;
+ if (!s->render_head)
+ s->render_tail = NULL;
+ j->next = NULL;
+ }
+ pthread_mutex_unlock(&s->mu);
+ return j;
+}
+
static job *pop_job_locked(http_server *s) {
job *j = s->job_head;
if (j) {
@@ -143,6 +171,7 @@ static job *pop_done(http_server *s) {
static void job_free(job *j) {
if (!j)
return;
+ job_render_free(j);
free(j->body);
free(j->auth);
free(j->extra_headers);
@@ -192,6 +221,11 @@ static void job_set_timing_header(job *j) {
j->extra_headers = xstrdup(buf);
}
+void finish_job(job *j) {
+ job_set_timing_header(j);
+ enqueue_done(j);
+}
+
static int parse_job_root(job *j, cJSON **out_root) {
cJSON *detail = cJSON_CreateArray();
cJSON *root = parse_json_body(j, detail);
@@ -260,24 +294,21 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
if (!kind[i]) {
process_unknown_job(jobs[i]);
- job_set_timing_header(jobs[i]);
- enqueue_done(jobs[i]);
+ finish_job(jobs[i]);
done[i] = 1;
continue;
}
if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) ||
(kind[i] == 3 && !rerank_reqs[i].ready)) {
- job_set_timing_header(jobs[i]);
- enqueue_done(jobs[i]);
+ finish_job(jobs[i]);
done[i] = 1;
continue;
}
if (kind[i] == 3) {
execute_rerank_request(&rerank_reqs[i]);
- job_set_timing_header(jobs[i]);
- enqueue_done(jobs[i]);
+ finish_job(jobs[i]);
done[i] = 1;
} else if (kind[i] == 1) {
int group_n = 1;
@@ -297,8 +328,10 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
execute_embedding_request_list(std_group, group_n);
for (int k = 0; k < group_n; k++) {
int idx = (int)(std_group[k] - std_reqs);
- job_set_timing_header(jobs[idx]);
- enqueue_done(jobs[idx]);
+ if (jobs[idx]->render_kind)
+ enqueue_render_job(jobs[idx]);
+ else
+ finish_job(jobs[idx]);
done[idx] = 1;
}
} else {
@@ -319,8 +352,7 @@ void process_job_group(http_server *s, job **jobs, int n_jobs) {
execute_contextual_request_list(ctx_group, group_n);
for (int k = 0; k < group_n; k++) {
int idx = (int)(ctx_group[k] - ctx_reqs);
- job_set_timing_header(jobs[idx]);
- enqueue_done(jobs[idx]);
+ finish_job(jobs[idx]);
done[idx] = 1;
}
}
diff --git a/tools/server/server.c b/tools/server/server.c
index abff2c2..8762c4f 100644
--- a/tools/server/server.c
+++ b/tools/server/server.c
@@ -215,6 +215,23 @@ static void *worker_main(void *arg) {
return NULL;
}
+static void *renderer_main(void *arg) {
+ http_server *s = arg;
+ for (;;) {
+ job *j = dequeue_render_job(s);
+ if (!j)
+ break;
+
+ uint64_t t0 = nstime();
+ if (render_job_response(j) != 0)
+ job_set_error(j, 500, "embedding normalization failed", "server_error");
+ j->encode_ns += nstime() - t0;
+ job_render_free(j);
+ finish_job(j);
+ }
+ return NULL;
+}
+
static int listen_on(const char *host, int port) {
char portbuf[32];
snprintf(portbuf, sizeof(portbuf), "%d", port);
@@ -310,10 +327,15 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
pthread_mutex_init(&s.mu, NULL);
pthread_cond_init(&s.cv, NULL);
+ pthread_cond_init(&s.render_cv, NULL);
for (int i = 0; i < cfg->n_models; i++) {
if (load_one_model(&s, &cfg->models[i]) != 0) {
free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
return 1;
}
}
@@ -321,6 +343,10 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) {
perror("pipe");
free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
return 1;
}
set_nonblock(s.completion_pipe[0]);
@@ -341,13 +367,37 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
if (!s.loop) {
fprintf(stderr, "ffwd-server: failed to create event loop\n");
free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 1;
+ }
+
+ if (pthread_create(&s.renderer, NULL, renderer_main, &s) != 0) {
+ fprintf(stderr, "ffwd-server: failed to start renderer\n");
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
return 1;
}
if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) {
fprintf(stderr, "ffwd-server: failed to start worker\n");
+ pthread_mutex_lock(&s.mu);
+ s.render_stopping = 1;
+ pthread_cond_signal(&s.render_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.renderer, NULL);
aeDeleteEventLoop(s.loop);
free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
return 1;
}
pthread_mutex_lock(&s.mu);
@@ -356,10 +406,20 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
int worker_init_rc = s.worker_init_rc;
pthread_mutex_unlock(&s.mu);
if (worker_init_rc) {
+ pthread_mutex_lock(&s.mu);
+ s.stopping = 1;
+ pthread_cond_signal(&s.cv);
+ pthread_mutex_unlock(&s.mu);
pthread_join(s.worker, NULL);
+ pthread_mutex_lock(&s.mu);
+ s.render_stopping = 1;
+ pthread_cond_signal(&s.render_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.renderer, NULL);
aeDeleteEventLoop(s.loop);
free_models(&s);
free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 1;
@@ -372,8 +432,17 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
pthread_cond_signal(&s.cv);
pthread_mutex_unlock(&s.mu);
pthread_join(s.worker, NULL);
+ pthread_mutex_lock(&s.mu);
+ s.render_stopping = 1;
+ pthread_cond_signal(&s.render_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.renderer, NULL);
aeDeleteEventLoop(s.loop);
free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
return 1;
}
@@ -400,6 +469,11 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
pthread_cond_signal(&s.cv);
pthread_mutex_unlock(&s.mu);
pthread_join(s.worker, NULL);
+ pthread_mutex_lock(&s.mu);
+ s.render_stopping = 1;
+ pthread_cond_signal(&s.render_cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.renderer, NULL);
aeDeleteEventLoop(s.loop);
close(s.completion_pipe[0]);
@@ -408,6 +482,7 @@ int ffwd_run_server(const ffwd_server_config_t *cfg) {
close(s.signal_pipe[1]);
free_models(&s);
free(s.api_key);
+ pthread_cond_destroy(&s.render_cv);
pthread_cond_destroy(&s.cv);
pthread_mutex_destroy(&s.mu);
return 0;
diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h
index 91b51c5..2f1067e 100644
--- a/tools/server/server_internal.h
+++ b/tools/server/server_internal.h
@@ -74,8 +74,8 @@ typedef struct {
typedef struct {
model_info *info;
char *path;
- ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */
- ffwd_t *backend; /* compute handle */
+ ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */
+ ffwd_t *backend; /* compute handle */
int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */
} loaded_model;
@@ -105,6 +105,11 @@ typedef struct {
int worker_init_rc;
job *job_head;
job *job_tail;
+ pthread_t renderer;
+ pthread_cond_t render_cv;
+ int render_stopping;
+ job *render_head;
+ job *render_tail;
job *done_head;
job *done_tail;
@@ -159,6 +164,15 @@ struct job {
uint64_t tokenize_ns;
uint64_t infer_ns;
uint64_t encode_ns;
+ int render_kind;
+ struct {
+ loaded_model *model;
+ char *encoding;
+ float *embs;
+ int dims;
+ int n_inputs;
+ int total_tokens;
+ } embedding_render;
job *next;
};
@@ -260,6 +274,9 @@ void append_embedding_value(
void set_response_from_buf(job *j, sbuf *b);
int render_embedding_response(embedding_request *r, const float *embs);
void render_contextual_response(contextual_request *r, const float *embs);
+void job_render_free(job *j);
+void job_set_embedding_render(job *j, const embedding_request *r, const float *embs);
+int render_job_response(job *j);
/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */
loaded_model *loaded_model_for_label(http_server *s, const char *label);
@@ -275,9 +292,9 @@ int tokenize_input(
int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out);
int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out);
int model_ffwd_spans_batch(loaded_model *m,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out);
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out);
int inference_batch_accepts_input(
const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens);
int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim);
@@ -307,6 +324,9 @@ void execute_rerank_request(rerank_request *r);
/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
void enqueue_job(job *j);
+void enqueue_render_job(job *j);
+job *dequeue_render_job(http_server *s);
+void finish_job(job *j);
int collect_job_batch(http_server *s, job **jobs, int max_jobs);
void process_job_group(http_server *s, job **jobs, int n_jobs);
void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c
index a7db9c4..91a4479 100644
--- a/tools/server/test_encode.c
+++ b/tools/server/test_encode.c
@@ -80,11 +80,51 @@ static void test_append_embedding_value_float(void) {
sbuf_free(&b2);
}
+static void test_job_embedding_render_payload(void) {
+ model_info info;
+ memset(&info, 0, sizeof(info));
+ info.id = "m";
+ info.dim = 2;
+ info.api = FFWD_API_OPENAI;
+
+ loaded_model model;
+ memset(&model, 0, sizeof(model));
+ model.info = &info;
+
+ job j;
+ memset(&j, 0, sizeof(j));
+ embedding_request r;
+ memset(&r, 0, sizeof(r));
+ r.j = &j;
+ r.model = &model;
+ r.dims = 2;
+ r.encoding = "float";
+ r.n_inputs = 2;
+ r.total_tokens = 7;
+
+ float embs[4] = {0.5f, 1.0f, -0.25f, 0.0f};
+ job_set_embedding_render(&j, &r, embs);
+ TEST_ASSERT(j.render_kind == 1);
+ TEST_ASSERT(j.embedding_render.embs != embs);
+ TEST_ASSERT(render_job_response(&j) == 0);
+ TEST_ASSERT(j.status == 200);
+ TEST_ASSERT(j.response != NULL);
+ TEST_ASSERT(strstr(j.response, "\"model\":\"m\"") != NULL);
+ TEST_ASSERT(strstr(j.response, "\"prompt_tokens\":7") != NULL);
+ TEST_ASSERT(strstr(j.response, "\"embedding\":[0.5,1]") != NULL);
+ TEST_ASSERT(strstr(j.response, "\"embedding\":[-0.25,0]") != NULL);
+
+ free(j.response);
+ j.response = NULL;
+ job_render_free(&j);
+}
+
int main(void) {
test_quantize_int8_tanh();
test_encode_embedding_int8();
test_encode_embedding_binary();
test_encode_embedding_base64_float32();
test_append_embedding_value_float();
+ test_job_embedding_render_payload();
return TEST_REPORT("encode");
}