diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-05-31 00:01:27 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 2ceffe319f42bbf966f085a581c71eb9e369cf19 (patch) | |
| tree | b652bc7a302c2be91369716f51e841abc2728dc7 | |
| parent | 6c60a776d5f2f2f494d20072a595fa0e10dd51d1 (diff) | |
| download | ffwd-2ceffe319f42bbf966f085a581c71eb9e369cf19.tar.gz ffwd-2ceffe319f42bbf966f085a581c71eb9e369cf19.zip | |
server: batch standard requests by token budget
| -rw-r--r-- | main.c | 12 | ||||
| -rw-r--r-- | pplx_server.c | 75 | ||||
| -rw-r--r-- | pplx_server.h | 1 |
3 files changed, 77 insertions, 11 deletions
@@ -47,6 +47,8 @@ static void print_usage(const char *prog) " --cors Enable CORS headers in --serve mode\n" " --api-key K Require Authorization: Bearer K in --serve mode\n" " -b <n> Max texts per engine batch (default: all; stdin: 1; server: 8)\n" + " --max-batch-tokens N\n" + " Server max tokens per standard inference batch (default: 16384)\n" " --batch-wait-us N\n" " Server micro-batch wait in microseconds (default: 1000; 0 disables)\n" " -t <n> CPU threads (default: all cores)\n" @@ -509,6 +511,7 @@ int main(int argc, char *argv[]) int layer_start = -1; int layer_end = -1; int batch_size = 0; + int max_batch_tokens = 0; int batch_wait_us = -1; const char *backend = "cpu"; const char *host = "127.0.0.1"; @@ -533,6 +536,14 @@ int main(int argc, char *argv[]) return 1; } } + else if (!strcmp(f, "--max-batch-tokens")) { + max_batch_tokens = atoi(argv[++arg_start]); + if (max_batch_tokens <= 0) { + fprintf(stderr, "--max-batch-tokens must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--model")) { if (append_model_spec(&model_specs, argv[++arg_start]) != 0) return 1; } @@ -619,6 +630,7 @@ int main(int argc, char *argv[]) .host = host, .port = port, .batch_size = batch_size > 0 ? batch_size : 8, + .max_batch_tokens = max_batch_tokens, .batch_wait_us = batch_wait_us, .use_mlx = use_mlx, .enable_cors = cors, diff --git a/pplx_server.c b/pplx_server.c index 08a8dd2..8968ddc 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -52,6 +52,7 @@ #define PPLX_API_MAX_ITEM_TOKENS 32768 #define PPLX_API_MAX_TOTAL_TOKENS 120000 #define PPLX_SERVER_DEFAULT_BATCH_SIZE 8 +#define PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 #define PPLX_SERVER_BATCH_WAIT_US 1000 #define PPLX_SERVER_MICROBATCH_MAX_JOBS 128 @@ -217,6 +218,7 @@ typedef struct { const char *host; int port; int batch_size; + int max_batch_tokens; int batch_wait_us; int enable_cors; char *api_key; @@ -1030,6 +1032,27 @@ static int embedding_request_compatible(const embedding_request *a, !strcmp(a->encoding, b->encoding); } +typedef struct { + pplx_input_t input; + int output_index; +} embedding_batch_item; + +static int embedding_batch_item_cmp(const void *a, const void *b) { + const embedding_batch_item *ia = a; + const embedding_batch_item *ib = b; + if (ia->input.n_tokens < ib->input.n_tokens) return -1; + if (ia->input.n_tokens > ib->input.n_tokens) return 1; + return ia->output_index - ib->output_index; +} + +static int embedding_request_fits_group(const embedding_request *r, + int group_inputs, int group_tokens, + int max_batch, int max_batch_tokens) { + if (group_inputs == 0) return 1; + return r->n_inputs <= max_batch - group_inputs && + r->total_tokens <= max_batch_tokens - group_tokens; +} + static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { if (n_reqs <= 0) return; loaded_model *m = reqs[0]->model; @@ -1038,27 +1061,46 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) for (int i = 0; i < n_reqs; i++) total_inputs += reqs[i]->n_inputs; - pplx_input_t *inputs = xmalloc((size_t)total_inputs * sizeof(*inputs)); + embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items)); float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float)); int pos = 0; for (int i = 0; i < n_reqs; i++) { - memcpy(inputs + pos, reqs[i]->inputs, - (size_t)reqs[i]->n_inputs * sizeof(*inputs)); - pos += reqs[i]->n_inputs; + for (int k = 0; k < reqs[i]->n_inputs; k++) { + items[pos].input = reqs[i]->inputs[k]; + items[pos].output_index = pos; + pos++; + } } + qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs; + if (max_batch > total_inputs) max_batch = total_inputs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + pplx_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); + int failed = 0; - for (int start = 0; start < total_inputs; start += max_batch) { - int cur = total_inputs - start; - if (cur > max_batch) cur = max_batch; - if (model_embed_batch(m, inputs + start, cur, - embs + (size_t)start * dim) != 0) { + for (int start = 0; start < total_inputs;) { + int cur = 0; + int tokens = 0; + while (start + cur < total_inputs && cur < max_batch) { + int n_tokens = items[start + cur].input.n_tokens; + if (cur > 0 && n_tokens > max_batch_tokens - tokens) + break; + inputs[cur] = items[start + cur].input; + tokens += n_tokens; + cur++; + } + if (model_embed_batch(m, inputs, cur, batch_embs) != 0) { failed = 1; break; } + for (int i = 0; i < cur; i++) + memcpy(embs + (size_t)items[start + i].output_index * dim, + batch_embs + (size_t)i * dim, (size_t)dim * sizeof(float)); + start += cur; } pos = 0; @@ -1071,8 +1113,10 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) pos += reqs[i]->n_inputs; } - free(embs); + free(batch_embs); free(inputs); + free(embs); + free(items); } static void execute_embedding_requests(embedding_request *reqs, int n_reqs) { @@ -1496,11 +1540,18 @@ static void process_job_group(http_server *s, job **jobs, int n_jobs) { } int group_n = 1; + int group_inputs = reqs[i].n_inputs; + int group_tokens = reqs[i].total_tokens; group[0] = &reqs[i]; for (int k = i + 1; k < n_jobs; k++) { if (!done[k] && is_standard[k] && reqs[k].ready && - embedding_request_compatible(&reqs[i], &reqs[k])) + embedding_request_compatible(&reqs[i], &reqs[k]) && + embedding_request_fits_group(&reqs[k], group_inputs, + group_tokens, s->batch_size, s->max_batch_tokens)) { group[group_n++] = &reqs[k]; + group_inputs += reqs[k].n_inputs; + group_tokens += reqs[k].total_tokens; + } } execute_embedding_request_list(group, group_n); @@ -1679,6 +1730,8 @@ int pplx_run_server(const pplx_server_config_t *cfg) { s.port = cfg->port > 0 ? cfg->port : 8000; s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : PPLX_SERVER_DEFAULT_BATCH_SIZE; + s.max_batch_tokens = cfg->max_batch_tokens > 0 + ? cfg->max_batch_tokens : PPLX_SERVER_DEFAULT_MAX_BATCH_TOKENS; s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : PPLX_SERVER_BATCH_WAIT_US; s.enable_cors = cfg->enable_cors; diff --git a/pplx_server.h b/pplx_server.h index a15483d..08a3a0d 100644 --- a/pplx_server.h +++ b/pplx_server.h @@ -14,6 +14,7 @@ typedef struct { const char *host; int port; int batch_size; + int max_batch_tokens; int batch_wait_us; int use_mlx; int enable_cors; |
