aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-30 04:49:22 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:59 +0200
commit5bab6ce54b51d019c480e6f88a446d9c8a127602 (patch)
tree680b4efd1d97463cef456c74443d7ba935a61bf7
parentab50d908ba2eae14929a07341b1b716cde681c1c (diff)
downloadffwd-5bab6ce54b51d019c480e6f88a446d9c8a127602.tar.gz
ffwd-5bab6ce54b51d019c480e6f88a446d9c8a127602.zip
server: add embeddings micro-batching
-rw-r--r--main.c16
-rw-r--r--pplx_server.c441
-rw-r--r--pplx_server.h1
3 files changed, 358 insertions, 100 deletions
diff --git a/main.c b/main.c
index 1eba511..6c9f25f 100644
--- a/main.c
+++ b/main.c
@@ -40,7 +40,9 @@ static void print_usage(const char *prog)
" --port N Server bind port (default: 8000)\n"
" --cors Enable CORS headers in --serve mode\n"
" --api-key K Require Authorization: Bearer K in --serve mode\n"
- " -b <n> Max texts per engine batch (default: all; stdin/server: 1)\n"
+ " -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n"
+ " --batch-wait-us N\n"
+ " Server micro-batch wait in microseconds (default: 1000; 0 disables)\n"
" -t <n> CPU threads (default: all cores)\n"
" -e Print raw embeddings (with multiple texts)\n"
" -v Verbose (-vv for debug)\n"
@@ -466,6 +468,7 @@ int main(int argc, char *argv[])
int stdin_mode = 0;
int serve_mode = 0;
int batch_size = 0;
+ int batch_wait_us = -1;
const char *backend = "cpu";
const char *host = "127.0.0.1";
const char *api_key = NULL;
@@ -481,6 +484,14 @@ int main(int argc, char *argv[])
else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
batch_size = atoi(argv[++arg_start]);
}
+ else if (!strcmp(f, "--batch-wait-us")) {
+ batch_wait_us = atoi(argv[++arg_start]);
+ if (batch_wait_us < 0) {
+ fprintf(stderr, "--batch-wait-us must be >= 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ }
else if (!strcmp(f, "--model")) {
if (append_model_spec(&model_specs, argv[++arg_start]) != 0) return 1;
}
@@ -557,7 +568,8 @@ int main(int argc, char *argv[])
.n_models = model_specs.n,
.host = host,
.port = port,
- .batch_size = batch_size > 0 ? batch_size : 1,
+ .batch_size = batch_size > 0 ? batch_size : 8,
+ .batch_wait_us = batch_wait_us,
.use_mlx = use_mlx,
.enable_cors = cors,
.api_key = api_key,
diff --git a/pplx_server.c b/pplx_server.c
index 825d448..d1b76b0 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -51,6 +51,9 @@
#define PPLX_API_MAX_CONTEXT_CHUNKS 16000
#define PPLX_API_MAX_ITEM_TOKENS 32768
#define PPLX_API_MAX_TOTAL_TOKENS 120000
+#define PPLX_SERVER_DEFAULT_BATCH_SIZE 8
+#define PPLX_SERVER_BATCH_WAIT_US 1000
+#define PPLX_SERVER_MICROBATCH_MAX_JOBS 128
typedef struct {
char *ptr;
@@ -214,6 +217,7 @@ typedef struct {
const char *host;
int port;
int batch_size;
+ int batch_wait_us;
int enable_cors;
char *api_key;
loaded_model models[4];
@@ -223,6 +227,8 @@ typedef struct {
pthread_mutex_t mu;
pthread_cond_t cv;
int stopping;
+ int worker_ready;
+ int worker_init_rc;
job *job_head;
job *job_tail;
job *done_head;
@@ -542,6 +548,20 @@ static job *dequeue_job(http_server *s) {
return j;
}
+static int drain_ready_jobs(http_server *s, job **jobs, int max_jobs) {
+ int n = 0;
+ pthread_mutex_lock(&s->mu);
+ while (n < max_jobs && s->job_head) {
+ job *j = s->job_head;
+ s->job_head = j->next;
+ if (!s->job_head) s->job_tail = NULL;
+ j->next = NULL;
+ jobs[n++] = j;
+ }
+ pthread_mutex_unlock(&s->mu);
+ return n;
+}
+
static void enqueue_job(job *j) {
http_server *s = j->srv;
pthread_mutex_lock(&s->mu);
@@ -890,11 +910,33 @@ typedef struct {
int n_tokens;
} token_buf;
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ int dims;
+ const char *encoding;
+ int n_inputs;
+ token_buf *tokens;
+ pplx_input_t *inputs;
+ int total_tokens;
+ int ready;
+} embedding_request;
+
static void free_token_bufs(token_buf *t, int n) {
if (!t) return;
for (int i = 0; i < n; i++) free(t[i].ids);
}
+static void embedding_request_free(embedding_request *r) {
+ if (!r) return;
+ free_token_bufs(r->tokens, r->n_inputs);
+ free(r->tokens);
+ free(r->inputs);
+ if (r->root) cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
static int tokenize_one(loaded_model *m, const char *text, token_buf *out) {
memset(out, 0, sizeof(*out));
out->ids = qwen_tokenizer_encode(m->tok, text, &out->n_tokens);
@@ -939,73 +981,91 @@ static void set_response_from_buf(job *j, sbuf *b) {
memset(b, 0, sizeof(*b));
}
-static void handle_embeddings_job(job *j, cJSON *root, loaded_model *m,
- int dims, const char *encoding) {
- cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
- int n_inputs = cJSON_IsString(input) ? 1 : cJSON_GetArraySize(input);
- token_buf *tokens = xcalloc((size_t)n_inputs, sizeof(*tokens));
- pplx_input_t *inputs = xmalloc((size_t)n_inputs * sizeof(*inputs));
- float *embs = xmalloc((size_t)n_inputs * m->info->dim * sizeof(float));
- char **encoded = xcalloc((size_t)n_inputs, sizeof(*encoded));
-
- int total_tokens = 0;
- int idx = 0;
- if (cJSON_IsString(input)) {
- tokenize_one(m, input->valuestring, &tokens[0]);
- inputs[0].ids = tokens[0].ids;
- inputs[0].n_tokens = tokens[0].n_tokens;
- total_tokens = tokens[0].n_tokens;
- } else {
- cJSON *item;
- cJSON_ArrayForEach(item, input) {
- tokenize_one(m, item->valuestring, &tokens[idx]);
- inputs[idx].ids = tokens[idx].ids;
- inputs[idx].n_tokens = tokens[idx].n_tokens;
- total_tokens += tokens[idx].n_tokens;
- idx++;
- }
- }
-
- for (int i = 0; i < n_inputs; i++) {
- if (!tokens[i].ids) {
- job_set_error(j, 500, "tokenization failed", "server_error");
- goto done;
- }
- }
- int max_batch = j->srv->batch_size > 0 ? j->srv->batch_size : n_inputs;
- for (int start = 0; start < n_inputs; start += max_batch) {
- int cur = n_inputs - start;
- if (cur > max_batch) cur = max_batch;
- if (model_embed_batch(m, inputs + start, cur,
- embs + (size_t)start * m->info->dim) != 0) {
- job_set_error(j, 500, "embedding failed", "server_error");
- goto done;
- }
- }
- for (int i = 0; i < n_inputs; i++)
- encoded[i] = encode_embedding(embs + (size_t)i * m->info->dim, dims, encoding);
+static void render_embedding_response(embedding_request *r, const float *embs) {
+ char **encoded = xcalloc((size_t)r->n_inputs, sizeof(*encoded));
+ for (int i = 0; i < r->n_inputs; i++)
+ encoded[i] = encode_embedding(embs + (size_t)i * r->model->info->dim,
+ r->dims, r->encoding);
sbuf b = {0};
sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
- for (int i = 0; i < n_inputs; i++) {
+ for (int i = 0; i < r->n_inputs; i++) {
if (i) sbuf_putc(&b, ',');
append_embedding_object(&b, i, encoded[i]);
}
- double cost = ((double)total_tokens / 1000000.0) * m->info->price_per_mtok;
+ double cost = ((double)r->total_tokens / 1000000.0) *
+ r->model->info->price_per_mtok;
sbuf_printf(&b,
"],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,"
"\"total_tokens\":%d,\"cost\":{\"input_cost\":%.10g,"
"\"total_cost\":%.10g,\"currency\":\"USD\"}}}",
- m->info->id, total_tokens, total_tokens, cost, cost);
- set_response_from_buf(j, &b);
+ r->model->info->id, r->total_tokens, r->total_tokens, cost, cost);
+ set_response_from_buf(r->j, &b);
-done:
- for (int i = 0; i < n_inputs; i++) free(encoded[i]);
+ for (int i = 0; i < r->n_inputs; i++) free(encoded[i]);
free(encoded);
+}
+
+static int embedding_request_compatible(const embedding_request *a,
+ const embedding_request *b) {
+ return a->ready && b->ready &&
+ a->model == b->model &&
+ a->dims == b->dims &&
+ !strcmp(a->encoding, b->encoding);
+}
+
+static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
+ if (n_reqs <= 0) return;
+ loaded_model *m = reqs[0]->model;
+ int dim = m->info->dim;
+ int total_inputs = 0;
+ for (int i = 0; i < n_reqs; i++)
+ total_inputs += reqs[i]->n_inputs;
+
+ pplx_input_t *inputs = xmalloc((size_t)total_inputs * sizeof(*inputs));
+ float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float));
+
+ int pos = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ memcpy(inputs + pos, reqs[i]->inputs,
+ (size_t)reqs[i]->n_inputs * sizeof(*inputs));
+ pos += reqs[i]->n_inputs;
+ }
+
+ int max_batch = reqs[0]->j->srv->batch_size > 0
+ ? reqs[0]->j->srv->batch_size : total_inputs;
+ int failed = 0;
+ for (int start = 0; start < total_inputs; start += max_batch) {
+ int cur = total_inputs - start;
+ if (cur > max_batch) cur = max_batch;
+ if (model_embed_batch(m, inputs + start, cur,
+ embs + (size_t)start * dim) != 0) {
+ failed = 1;
+ break;
+ }
+ }
+
+ pos = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ if (failed) {
+ job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
+ } else {
+ render_embedding_response(reqs[i], embs + (size_t)pos * dim);
+ }
+ pos += reqs[i]->n_inputs;
+ }
+
free(embs);
free(inputs);
- free_token_bufs(tokens, n_inputs);
- free(tokens);
+}
+
+static void execute_embedding_requests(embedding_request *reqs, int n_reqs) {
+ if (n_reqs <= 0) return;
+ embedding_request **list = xmalloc((size_t)n_reqs * sizeof(*list));
+ for (int i = 0; i < n_reqs; i++)
+ list[i] = &reqs[i];
+ execute_embedding_request_list(list, n_reqs);
+ free(list);
}
static void handle_context_job(job *j, cJSON *root, loaded_model *m,
@@ -1097,20 +1157,6 @@ done:
free(chunks);
}
-static void validate_text_tokens(cJSON *detail, loaded_model *m, const char *loc,
- const char *text, int *total_tokens) {
- token_buf t;
- if (tokenize_one(m, text, &t) != 0) {
- ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
- return;
- }
- if (t.n_tokens > PPLX_API_MAX_ITEM_TOKENS)
- ve_add(detail, loc, "input exceeds 32768 token limit", "value_error.context_length");
- if (*total_tokens <= PPLX_API_MAX_TOTAL_TOKENS)
- *total_tokens += t.n_tokens;
- free(t.ids);
-}
-
static int validate_common(cJSON *root, cJSON *detail, int expect_context,
loaded_model **out_model, int *out_dims,
const char **out_encoding, http_server *s) {
@@ -1145,25 +1191,29 @@ static int validate_common(cJSON *root, cJSON *detail, int expect_context,
!s->models[model_slot_for_id(model_id)].info ? 503 : 0;
}
-static void process_embeddings(job *j, cJSON *root, http_server *s) {
+static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
+ embedding_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
+
cJSON *detail = cJSON_CreateArray();
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
int unloaded = validate_common(root, detail, 0, &m, &dims, &encoding, s);
+ out->model = m;
+ out->dims = dims;
+ out->encoding = encoding;
cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
int n_inputs = 0;
- int total_tokens = 0;
if (!input) {
ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
} else if (cJSON_IsString(input)) {
if (!input->valuestring || input->valuestring[0] == '\0')
ve_add(detail, "[\"body\",\"input\"]", "input must not be empty",
"value_error.empty");
- else if (m)
- validate_text_tokens(detail, m, "[\"body\",\"input\"]",
- input->valuestring, &total_tokens);
n_inputs = 1;
} else if (cJSON_IsArray(input)) {
n_inputs = cJSON_GetArraySize(input);
@@ -1182,8 +1232,6 @@ static void process_embeddings(job *j, cJSON *root, http_server *s) {
ve_add(detail, loc, "input item must be a string", "type_error.string");
} else if (!item->valuestring || item->valuestring[0] == '\0') {
ve_add(detail, loc, "input item must not be empty", "value_error.empty");
- } else if (m) {
- validate_text_tokens(detail, m, loc, item->valuestring, &total_tokens);
}
i++;
}
@@ -1191,21 +1239,76 @@ static void process_embeddings(job *j, cJSON *root, http_server *s) {
ve_add(detail, "[\"body\",\"input\"]",
"input must be a string or an array of strings", "type_error");
}
- if (total_tokens > PPLX_API_MAX_TOTAL_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
- "value_error.context_length");
- if (cJSON_GetArraySize(detail) > 0) {
- job_set_422(j, detail);
- } else if (unloaded) {
+ if (cJSON_GetArraySize(detail) == 0 && unloaded) {
job_set_error(j, 503, "requested model is valid but not loaded",
"model_not_loaded");
+ cJSON_Delete(detail);
+ return;
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && m) {
+ out->n_inputs = n_inputs;
+ out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens));
+ out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs));
+
+ int idx = 0;
+ if (cJSON_IsString(input)) {
+ if (tokenize_one(m, input->valuestring, &out->tokens[0]) != 0) {
+ ve_add(detail, "[\"body\",\"input\"]", "tokenization failed",
+ "value_error.tokenization");
+ } else {
+ out->inputs[0].ids = out->tokens[0].ids;
+ out->inputs[0].n_tokens = out->tokens[0].n_tokens;
+ out->total_tokens = out->tokens[0].n_tokens;
+ if (out->tokens[0].n_tokens > PPLX_API_MAX_ITEM_TOKENS)
+ ve_add(detail, "[\"body\",\"input\"]",
+ "input exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+ } else {
+ cJSON *item;
+ cJSON_ArrayForEach(item, input) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx);
+ if (tokenize_one(m, item->valuestring, &out->tokens[idx]) != 0) {
+ ve_add(detail, loc, "tokenization failed",
+ "value_error.tokenization");
+ } else {
+ out->inputs[idx].ids = out->tokens[idx].ids;
+ out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
+ out->total_tokens += out->tokens[idx].n_tokens;
+ if (out->tokens[idx].n_tokens > PPLX_API_MAX_ITEM_TOKENS)
+ ve_add(detail, loc, "input exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+ idx++;
+ }
+ }
+
+ if (out->total_tokens > PPLX_API_MAX_TOTAL_TOKENS)
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ "value_error.context_length");
+ }
+
+ if (cJSON_GetArraySize(detail) > 0) {
+ job_set_422(j, detail);
+ embedding_request_free(out);
+ out->j = j;
} else {
- handle_embeddings_job(j, root, m, dims, encoding);
+ out->ready = 1;
}
cJSON_Delete(detail);
}
+static void process_embeddings(job *j, cJSON *root, http_server *s) {
+ embedding_request r;
+ prepare_embedding_request(j, root, s, &r);
+ if (r.ready)
+ execute_embedding_requests(&r, 1);
+ embedding_request_free(&r);
+}
+
static void process_contextual(job *j, cJSON *root, http_server *s) {
cJSON *detail = cJSON_CreateArray();
loaded_model *m = NULL;
@@ -1300,33 +1403,165 @@ static void process_contextual(job *j, cJSON *root, http_server *s) {
cJSON_Delete(detail);
}
-static void process_job(job *j) {
+static int parse_job_root(job *j, cJSON **out_root) {
cJSON *detail = cJSON_CreateArray();
cJSON *root = parse_json_body(j, detail);
if (!root) {
job_set_422(j, detail);
cJSON_Delete(detail);
- return;
+ *out_root = NULL;
+ return -1;
}
cJSON_Delete(detail);
+ *out_root = root;
+ return 0;
+}
- if (!strcmp(j->path, "/v1/embeddings"))
+static void process_job(job *j) {
+ cJSON *root = NULL;
+ if (parse_job_root(j, &root) != 0)
+ return;
+ if (!strcmp(j->path, "/v1/embeddings")) {
process_embeddings(j, root, j->srv);
- else if (!strcmp(j->path, "/v1/contextualizedembeddings"))
+ return;
+ } else if (!strcmp(j->path, "/v1/contextualizedembeddings"))
process_contextual(j, root, j->srv);
else
job_set_error(j, 404, "unknown endpoint", "invalid_request_error");
cJSON_Delete(root);
}
+static void process_job_group(http_server *s, job **jobs, int n_jobs) {
+ embedding_request *reqs = xcalloc((size_t)n_jobs, sizeof(*reqs));
+ int *is_standard = xcalloc((size_t)n_jobs, sizeof(*is_standard));
+ int *done = xcalloc((size_t)n_jobs, sizeof(*done));
+ embedding_request **group = xmalloc((size_t)n_jobs * sizeof(*group));
+
+ for (int i = 0; i < n_jobs; i++) {
+ if (strcmp(jobs[i]->path, "/v1/embeddings"))
+ continue;
+ cJSON *root = NULL;
+ is_standard[i] = 1;
+ if (parse_job_root(jobs[i], &root) == 0)
+ prepare_embedding_request(jobs[i], root, s, &reqs[i]);
+ }
+
+ for (int i = 0; i < n_jobs; i++) {
+ if (done[i])
+ continue;
+
+ if (!is_standard[i]) {
+ process_job(jobs[i]);
+ enqueue_done(jobs[i]);
+ done[i] = 1;
+ continue;
+ }
+
+ if (!reqs[i].ready) {
+ enqueue_done(jobs[i]);
+ done[i] = 1;
+ continue;
+ }
+
+ int group_n = 1;
+ group[0] = &reqs[i];
+ for (int k = i + 1; k < n_jobs; k++) {
+ if (!done[k] && is_standard[k] && reqs[k].ready &&
+ embedding_request_compatible(&reqs[i], &reqs[k]))
+ group[group_n++] = &reqs[k];
+ }
+
+ execute_embedding_request_list(group, group_n);
+ for (int k = 0; k < group_n; k++) {
+ int idx = (int)(group[k] - reqs);
+ enqueue_done(jobs[idx]);
+ done[idx] = 1;
+ }
+ }
+
+ for (int i = 0; i < n_jobs; i++)
+ embedding_request_free(&reqs[i]);
+ free(group);
+ free(done);
+ free(is_standard);
+ free(reqs);
+}
+
static void *worker_main(void *arg) {
http_server *s = arg;
+#ifdef USE_MLX
+ if (s->use_mlx) {
+ int rc = 0;
+ for (int i = 0; i < 4; i++) {
+ loaded_model *m = &s->models[i];
+ if (!m->info) continue;
+ m->mlx_ctx = pplx_mlx_load(m->path);
+ if (!m->mlx_ctx) {
+ server_log("pplx-serve: failed to load MLX model on worker: %s",
+ m->path);
+ rc = 1;
+ break;
+ }
+ if (pplx_mlx_config(m->mlx_ctx)->hidden_size != m->info->dim) {
+ server_log("pplx-serve: model %s has unexpected dimension %d",
+ m->info->id,
+ pplx_mlx_config(m->mlx_ctx)->hidden_size);
+ rc = 1;
+ break;
+ }
+ }
+ pthread_mutex_lock(&s->mu);
+ s->worker_init_rc = rc;
+ s->worker_ready = 1;
+ if (rc) s->stopping = 1;
+ pthread_cond_broadcast(&s->cv);
+ pthread_mutex_unlock(&s->mu);
+ if (rc) {
+ for (int i = 0; i < 4; i++) {
+ loaded_model *m = &s->models[i];
+ if (m->mlx_ctx) {
+ pplx_mlx_free(m->mlx_ctx);
+ m->mlx_ctx = NULL;
+ }
+ }
+ return NULL;
+ }
+ } else
+#endif
+ {
+ pthread_mutex_lock(&s->mu);
+ s->worker_ready = 1;
+ pthread_cond_broadcast(&s->cv);
+ pthread_mutex_unlock(&s->mu);
+ }
+
for (;;) {
job *j = dequeue_job(s);
if (!j) break;
- process_job(j);
- enqueue_done(j);
+ job *jobs[PPLX_SERVER_MICROBATCH_MAX_JOBS];
+ int n_jobs = 1;
+ jobs[0] = j;
+
+ if (!strcmp(j->path, "/v1/embeddings") && s->batch_size > 1) {
+ if (s->batch_wait_us > 0)
+ usleep((useconds_t)s->batch_wait_us);
+ n_jobs += drain_ready_jobs(s, jobs + 1,
+ PPLX_SERVER_MICROBATCH_MAX_JOBS - 1);
+ }
+
+ process_job_group(s, jobs, n_jobs);
}
+#ifdef USE_MLX
+ if (s->use_mlx) {
+ for (int i = 0; i < 4; i++) {
+ loaded_model *m = &s->models[i];
+ if (m->mlx_ctx) {
+ pplx_mlx_free(m->mlx_ctx);
+ m->mlx_ctx = NULL;
+ }
+ }
+ }
+#endif
return NULL;
}
@@ -1362,16 +1597,9 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
}
#ifdef USE_MLX
if (s->use_mlx) {
- m->mlx_ctx = pplx_mlx_load(path);
- if (!m->mlx_ctx) {
- server_log("pplx-serve: failed to load MLX model: %s", path);
- return -1;
- }
- if (pplx_mlx_config(m->mlx_ctx)->hidden_size != m->info->dim) {
- server_log("pplx-serve: model %s has unexpected dimension %d",
- m->info->id, pplx_mlx_config(m->mlx_ctx)->hidden_size);
- return -1;
- }
+ /* MLX streams are thread-local. The inference worker loads the MLX
+ * model so all MLX arrays/streams are created, used, and freed by the
+ * same thread. */
return 0;
}
#endif
@@ -1417,7 +1645,10 @@ int pplx_run_server(const pplx_server_config_t *cfg) {
memset(&s, 0, sizeof(s));
s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1";
s.port = cfg->port > 0 ? cfg->port : 8000;
- s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : 1;
+ s.batch_size = cfg->batch_size > 0
+ ? cfg->batch_size : PPLX_SERVER_DEFAULT_BATCH_SIZE;
+ s.batch_wait_us = cfg->batch_wait_us >= 0
+ ? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US;
s.enable_cors = cfg->enable_cors;
s.use_mlx = cfg->use_mlx;
if (cfg->api_key && cfg->api_key[0])
@@ -1475,6 +1706,20 @@ int pplx_run_server(const pplx_server_config_t *cfg) {
free_models(&s);
return 1;
}
+ pthread_mutex_lock(&s.mu);
+ while (!s.worker_ready)
+ pthread_cond_wait(&s.cv, &s.mu);
+ int worker_init_rc = s.worker_init_rc;
+ pthread_mutex_unlock(&s.mu);
+ if (worker_init_rc) {
+ pthread_join(s.worker, NULL);
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 1;
+ }
s.listen_fd = listen_on(s.host, s.port);
if (s.listen_fd < 0) {
diff --git a/pplx_server.h b/pplx_server.h
index c46f4f9..a15483d 100644
--- a/pplx_server.h
+++ b/pplx_server.h
@@ -14,6 +14,7 @@ typedef struct {
const char *host;
int port;
int batch_size;
+ int batch_wait_us;
int use_mlx;
int enable_cors;
const char *api_key;