diff options
| -rw-r--r-- | pplx_embed.c | 81 | ||||
| -rw-r--r-- | pplx_embed.h | 17 | ||||
| -rw-r--r-- | pplx_embed_mlx.c | 154 | ||||
| -rw-r--r-- | pplx_embed_mlx.h | 8 | ||||
| -rw-r--r-- | pplx_server.c | 533 |
5 files changed, 567 insertions, 226 deletions
diff --git a/pplx_embed.c b/pplx_embed.c index d4b2d02..4aaeda1 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -1044,23 +1044,80 @@ int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens, return 0; } -int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, - const int *token_ids, int n_tokens, - const pplx_span_t *spans, int n_spans, - float *out_embeddings) +int pplx_model_embed_spans_batch(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings) { - if (!model || !ws || !token_ids || n_tokens <= 0 || !spans || - n_spans <= 0 || !out_embeddings) + if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; - pplx_input_t input = { token_ids, n_tokens }; - int offsets[2] = { 0, n_tokens }; - if (forward_packed_inplace(model, ws, &input, 1, offsets, - n_tokens, n_tokens, 1) != 0) + if (ensure_offsets(ws, batch) != 0) return -1; + if ((size_t)batch > SIZE_MAX / sizeof(pplx_input_t)) return -1; + pplx_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); + if (!packed_inputs) return -1; + int *offsets = ws->offsets; + int total_seq = 0; + int max_seq = 0; + int total_spans = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || + input->n_spans <= 0 || total_seq > INT_MAX - n_tokens || + total_spans > INT_MAX - input->n_spans) { + free(packed_inputs); + return -1; + } + packed_inputs[b] = input->input; + offsets[b] = total_seq; + total_seq += n_tokens; + total_spans += input->n_spans; + if (n_tokens > max_seq) max_seq = n_tokens; + for (int s = 0; s < input->n_spans; s++) { + int start = input->spans[s].start; + int len = input->spans[s].n_tokens; + if (start < 0 || len <= 0 || start > n_tokens || + len > n_tokens - start) { + free(packed_inputs); + return -1; + } + } + } + offsets[batch] = total_seq; + + if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, + total_seq, max_seq, 1) != 0) { + free(packed_inputs); return -1; + } + free(packed_inputs); + + int span_offset = 0; + int hidden = model->config.hidden_size; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + if (pplx_pool_spans(&model->config, + ws->x + (size_t)offsets[b] * hidden, + input->input.n_tokens, input->spans, input->n_spans, + out_embeddings + (size_t)span_offset * hidden) != 0) + return -1; + span_offset += input->n_spans; + } + return 0; +} - return pplx_pool_spans(&model->config, ws->x, n_tokens, spans, n_spans, - out_embeddings); +int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, + const int *token_ids, int n_tokens, + const pplx_span_t *spans, int n_spans, + float *out_embeddings) +{ + pplx_context_input_t input = { + .input = { token_ids, n_tokens }, + .spans = spans, + .n_spans = n_spans, + }; + return pplx_model_embed_spans_batch(model, ws, &input, 1, out_embeddings); } float *pplx_model_forward(const pplx_model_t *model, pplx_workspace_t *ws, diff --git a/pplx_embed.h b/pplx_embed.h index 6a159de..3aa4059 100644 --- a/pplx_embed.h +++ b/pplx_embed.h @@ -53,6 +53,12 @@ typedef struct { int n_tokens; } pplx_span_t; +typedef struct { + pplx_input_t input; + const pplx_span_t *spans; + int n_spans; +} pplx_context_input_t; + /* ======================================================================== * API * ======================================================================== */ @@ -208,6 +214,17 @@ int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws, float *out_embeddings); /* + * Run a packed/ragged contextual document batch and pool every selected span. + * + * Documents attend independently, RoPE positions restart for each document, + * and out_embeddings contains spans in document order. + */ +int pplx_model_embed_spans_batch(const pplx_model_t *model, + pplx_workspace_t *ws, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings); + +/* * L2-normalize vec[dim] in place. * * Returns 0 on success. Returns -1 for invalid arguments or a zero-length diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c index c50c647..3157c17 100644 --- a/pplx_embed_mlx.c +++ b/pplx_embed_mlx.c @@ -695,12 +695,12 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx, return 0; } -int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, - int n_tokens, const pplx_span_t *spans, - int n_spans, float *out_embeddings) +int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings) { - if (!ctx || !token_ids || n_tokens <= 0 || !spans || n_spans <= 0 || - !out_embeddings || ctx->layer_start != 0 || + if (!ctx || !inputs || batch <= 0 || !out_embeddings || + ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) return -1; @@ -708,33 +708,95 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, const pplx_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int hidden = c->hidden_size; - for (int t = 0; t < n_tokens; t++) { - if (token_ids[t] < 0 || token_ids[t] >= c->vocab_size) { - fprintf(stderr, "mlx: invalid token id %d at position %d\n", - token_ids[t], t); + int max_seq = 0; + int total_spans = 0; + int has_padding = 0; + for (int b = 0; b < batch; b++) { + const pplx_context_input_t *input = &inputs[b]; + int n_tokens = input->input.n_tokens; + if (!input->input.ids || n_tokens <= 0 || !input->spans || + input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans) return -1; + total_spans += input->n_spans; + if (n_tokens > max_seq) max_seq = n_tokens; + for (int t = 0; t < n_tokens; t++) { + if (input->input.ids[t] < 0 || + input->input.ids[t] >= c->vocab_size) { + fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", + input->input.ids[t], b, t); + return -1; + } + } + for (int s = 0; s < input->n_spans; s++) { + if (input->spans[s].start < 0 || + input->spans[s].start > n_tokens || + input->spans[s].n_tokens <= 0 || + input->spans[s].n_tokens > n_tokens - input->spans[s].start) + return -1; } } - for (int s = 0; s < n_spans; s++) { - if (spans[s].start < 0 || spans[s].start > n_tokens || - spans[s].n_tokens <= 0 || - spans[s].n_tokens > n_tokens - spans[s].start) - return -1; + for (int b = 0; b < batch; b++) + if (inputs[b].input.n_tokens != max_seq) has_padding = 1; + + if ((size_t)batch > SIZE_MAX / (size_t)max_seq) return -1; + size_t rows = (size_t)batch * (size_t)max_seq; + if (rows > SIZE_MAX / sizeof(int) || + (has_padding && rows > SIZE_MAX / sizeof(float))) + return -1; + int *padded_ids = (int *)malloc(rows * sizeof(*padded_ids)); + float *attn_mask_data = has_padding + ? (float *)malloc(rows * sizeof(*attn_mask_data)) : NULL; + if (!padded_ids || (has_padding && !attn_mask_data)) { + free(padded_ids); + free(attn_mask_data); + return -1; + } + for (int b = 0; b < batch; b++) { + const pplx_input_t *input = &inputs[b].input; + for (int t = 0; t < max_seq; t++) { + size_t row = (size_t)b * max_seq + t; + if (t < input->n_tokens) { + padded_ids[row] = input->ids[t]; + if (has_padding) attn_mask_data[row] = 0.0f; + } else { + padded_ids[row] = 0; + if (has_padding) attn_mask_data[row] = -1.0e9f; + } + } } mlx_array ids = (mlx_array){0}; + mlx_array attn_mask = (mlx_array){0}; mlx_array x = (mlx_array){0}; mlx_array x_normed = (mlx_array){0}; mlx_array x_f32 = (mlx_array){0}; mlx_array embeddings = (mlx_array){0}; - mlx_array *pooled = (mlx_array *)calloc((size_t)n_spans, sizeof(*pooled)); + mlx_array *pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled)); mlx_vector_array pooled_vec = (mlx_vector_array){0}; int rc = -1; - if (!pooled) return -1; + if (!pooled) { + free(padded_ids); + free(attn_mask_data); + return -1; + } - int ids_shape[] = {1, n_tokens}; - ids = mlx_array_new_data(token_ids, ids_shape, 2, MLX_INT32); + int ids_shape[] = {batch, max_seq}; + int attn_mask_shape[] = {batch, 1, 1, max_seq}; + ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32); + attn_mask = has_padding + ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32) + : (mlx_array){0}; + free(padded_ids); + free(attn_mask_data); if (!arr_ok(ids)) goto cleanup; + if (has_padding && !arr_ok(attn_mask)) goto cleanup; + if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) { + mlx_array attn_mask_bf16 = mlx_array_new(); + mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S); + mlx_array_free(attn_mask); + attn_mask = attn_mask_bf16; + if (!arr_ok(attn_mask)) goto cleanup; + } x = mlx_array_new(); mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S); @@ -742,8 +804,12 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, ids = (mlx_array){0}; if (!arr_ok(x)) goto cleanup; - x = mlx_forward_layers(ctx, x, 1, n_tokens, 0, (mlx_array){0}, + x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers); + if (has_padding) { + mlx_array_free(attn_mask); + attn_mask = (mlx_array){0}; + } if (!arr_ok(x)) goto cleanup; x_normed = mlx_array_new(); @@ -758,24 +824,29 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, x_normed = (mlx_array){0}; if (!arr_ok(x_f32)) goto cleanup; - for (int s = 0; s < n_spans; s++) { - int start[] = {0, spans[s].start, 0}; - int stop[] = {1, spans[s].start + spans[s].n_tokens, hidden}; - int strides[] = {1, 1, 1}; - mlx_array slice = mlx_array_new(); - mlx_slice(&slice, x_f32, start, 3, stop, 3, strides, 3, S); - if (!arr_ok(slice)) { + int pooled_count = 0; + for (int b = 0; b < batch; b++) { + for (int s = 0; s < inputs[b].n_spans; s++) { + const pplx_span_t *span = &inputs[b].spans[s]; + int start[] = {b, span->start, 0}; + int stop[] = {b + 1, span->start + span->n_tokens, hidden}; + int strides[] = {1, 1, 1}; + mlx_array slice = mlx_array_new(); + mlx_slice(&slice, x_f32, start, 3, stop, 3, strides, 3, S); + if (!arr_ok(slice)) { + mlx_array_free(slice); + goto cleanup; + } + + pooled[pooled_count] = mlx_array_new(); + mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S); mlx_array_free(slice); - goto cleanup; + if (!arr_ok(pooled[pooled_count])) goto cleanup; + pooled_count++; } - - pooled[s] = mlx_array_new(); - mlx_mean_axis(&pooled[s], slice, 1, false, S); - mlx_array_free(slice); - if (!arr_ok(pooled[s])) goto cleanup; } - pooled_vec = mlx_vector_array_new_data(pooled, (size_t)n_spans); + pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans); if (!pooled_vec.ctx) goto cleanup; embeddings = mlx_array_new(); mlx_concatenate_axis(&embeddings, pooled_vec, 0, S); @@ -785,22 +856,35 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, const float *data = mlx_array_data_float32(embeddings); if (data) { memcpy(out_embeddings, data, - (size_t)n_spans * hidden * sizeof(float)); + (size_t)total_spans * hidden * sizeof(float)); rc = 0; } cleanup: mlx_array_free(embeddings); if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec); - for (int s = 0; s < n_spans; s++) mlx_array_free(pooled[s]); + for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]); free(pooled); mlx_array_free(x_f32); mlx_array_free(x_normed); mlx_array_free(x); + mlx_array_free(attn_mask); mlx_array_free(ids); return rc; } +int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, + int n_tokens, const pplx_span_t *spans, + int n_spans, float *out_embeddings) +{ + pplx_context_input_t input = { + .input = { token_ids, n_tokens }, + .spans = spans, + .n_spans = n_spans, + }; + return pplx_mlx_embed_spans_batch(ctx, &input, 1, out_embeddings); +} + float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { if (!ctx || !token_ids || n_tokens <= 0) return NULL; diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h index b45fc8a..2b7164d 100644 --- a/pplx_embed_mlx.h +++ b/pplx_embed_mlx.h @@ -74,6 +74,14 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, const pplx_span_t *spans, int n_spans, float *out_embeddings); +/* + * Run a padded dense contextual document batch and pool every selected span. + * out_embeddings contains spans in document order. + */ +int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx, + const pplx_context_input_t *inputs, int batch, + float *out_embeddings); + /* Get the config (so main.c can read hidden_size etc.) */ const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx); diff --git a/pplx_server.c b/pplx_server.c index 8968ddc..61dc1eb 100644 --- a/pplx_server.c +++ b/pplx_server.c @@ -923,6 +923,26 @@ typedef struct { int ready; } embedding_request; +typedef struct { + int *ids; + int n_tokens; + pplx_span_t *spans; + int n_spans; +} contextual_doc; + +typedef struct { + job *j; + cJSON *root; + loaded_model *model; + int dims; + const char *encoding; + contextual_doc *docs; + int n_docs; + int total_chunks; + int total_tokens; + int ready; +} contextual_request; + static void free_token_bufs(token_buf *t, int n) { if (!t) return; for (int i = 0; i < n; i++) free(t[i].ids); @@ -937,6 +957,17 @@ static void embedding_request_free(embedding_request *r) { memset(r, 0, sizeof(*r)); } +static void contextual_request_free(contextual_request *r) { + if (!r) return; + for (int i = 0; i < r->n_docs; i++) { + free(r->docs[i].ids); + free(r->docs[i].spans); + } + free(r->docs); + if (r->root) cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + static int tokenize_one(loaded_model *m, const char *text, token_buf *out) { memset(out, 0, sizeof(*out)); out->ids = qwen_tokenizer_encode(m->tok, text, &out->n_tokens); @@ -957,14 +988,15 @@ static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs, return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); } -static int model_embed_spans(loaded_model *m, const int *ids, int n_tokens, - const pplx_span_t *spans, int n_spans, float *out) { +static int model_embed_spans_batch(loaded_model *m, + const pplx_context_input_t *inputs, + int batch, float *out) { #ifdef USE_MLX if (m->mlx_ctx) - return pplx_mlx_embed_spans(m->mlx_ctx, ids, n_tokens, spans, n_spans, out); + return pplx_mlx_embed_spans_batch(m->mlx_ctx, inputs, batch, out); #endif - return pplx_model_embed_spans(m->cpu_model, m->cpu_ws, ids, n_tokens, - spans, n_spans, out); + return pplx_model_embed_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, + out); } static void append_embedding_object(sbuf *b, int index, const char *embedding) { @@ -1119,118 +1151,169 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) free(items); } -static void execute_embedding_requests(embedding_request *reqs, int n_reqs) { - if (n_reqs <= 0) return; - embedding_request **list = xmalloc((size_t)n_reqs * sizeof(*list)); - for (int i = 0; i < n_reqs; i++) - list[i] = &reqs[i]; - execute_embedding_request_list(list, n_reqs); - free(list); -} - -static void handle_context_job(job *j, cJSON *root, loaded_model *m, - int dims, const char *encoding) { - cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); - int n_docs = cJSON_GetArraySize(input); - int *chunks = xcalloc((size_t)n_docs, sizeof(*chunks)); - char ***encoded = strcmp(encoding, "float") - ? xcalloc((size_t)n_docs, sizeof(*encoded)) : NULL; - float **float_embs = !strcmp(encoding, "float") - ? xcalloc((size_t)n_docs, sizeof(*float_embs)) : NULL; - int total_tokens = 0; - - cJSON *doc_arr; - int di = 0; - cJSON_ArrayForEach(doc_arr, input) { - chunks[di] = cJSON_GetArraySize(doc_arr); - if (encoded) encoded[di] = xcalloc((size_t)chunks[di], sizeof(char *)); - token_buf *chunk_tokens = xcalloc((size_t)chunks[di], sizeof(*chunk_tokens)); - pplx_span_t *spans = xcalloc((size_t)chunks[di], sizeof(*spans)); - int doc_tokens = 0; - cJSON *chunk; - int ci = 0; - cJSON_ArrayForEach(chunk, doc_arr) { - if (tokenize_one(m, chunk->valuestring, &chunk_tokens[ci]) != 0) { - job_set_error(j, 500, "tokenization failed", "server_error"); - free_token_bufs(chunk_tokens, chunks[di]); - free(chunk_tokens); free(spans); - goto done; - } - doc_tokens += chunk_tokens[ci].n_tokens; - if (ci + 1 < chunks[di]) doc_tokens += m->newline_n; - ci++; - } - int *doc_ids = xmalloc((size_t)doc_tokens * sizeof(int)); - int pos = 0; - for (ci = 0; ci < chunks[di]; ci++) { - spans[ci].start = pos; - spans[ci].n_tokens = chunk_tokens[ci].n_tokens; - memcpy(doc_ids + pos, chunk_tokens[ci].ids, - (size_t)chunk_tokens[ci].n_tokens * sizeof(int)); - pos += chunk_tokens[ci].n_tokens; - if (ci + 1 < chunks[di] && m->newline_n > 0) { - memcpy(doc_ids + pos, m->newline_ids, - (size_t)m->newline_n * sizeof(int)); - pos += m->newline_n; - } - } - total_tokens += doc_tokens; - float *embs = xmalloc((size_t)chunks[di] * m->info->dim * sizeof(float)); - if (model_embed_spans(m, doc_ids, doc_tokens, spans, chunks[di], embs) != 0) { - free(embs); free(doc_ids); - job_set_error(j, 500, "contextual embedding failed", "server_error"); - free_token_bufs(chunk_tokens, chunks[di]); - free(chunk_tokens); free(spans); - goto done; - } - if (encoded) { - for (ci = 0; ci < chunks[di]; ci++) - encoded[di][ci] = encode_embedding(embs + (size_t)ci * m->info->dim, - dims, encoding); - free(embs); - } else { - float_embs[di] = embs; - } - free(doc_ids); - free_token_bufs(chunk_tokens, chunks[di]); - free(chunk_tokens); - free(spans); - di++; - } - +static void render_contextual_response(contextual_request *r, + const float *embs) { sbuf b = {0}; sbuf_puts(&b, "{\"object\":\"list\",\"data\":["); - for (di = 0; di < n_docs; di++) { + int span_offset = 0; + for (int di = 0; di < r->n_docs; di++) { if (di) sbuf_putc(&b, ','); sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di); - for (int ci = 0; ci < chunks[di]; ci++) { + for (int ci = 0; ci < r->docs[di].n_spans; ci++) { if (ci) sbuf_putc(&b, ','); - if (encoded) - append_embedding_object(&b, ci, encoded[di][ci]); - else + const float *emb = embs + (size_t)span_offset * r->model->info->dim; + if (!strcmp(r->encoding, "float")) { append_float_embedding_object(&b, ci, - float_embs[di] + (size_t)ci * m->info->dim, dims); + emb, r->dims); + } else { + char *encoded = encode_embedding(emb, r->dims, r->encoding); + append_embedding_object(&b, ci, encoded); + free(encoded); + } + span_offset++; } sbuf_puts(&b, "]}"); } sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}", - m->info->id, total_tokens, total_tokens); - set_response_from_buf(j, &b); + r->model->info->id, r->total_tokens, r->total_tokens); + set_response_from_buf(r->j, &b); +} + +static int contextual_request_compatible(const contextual_request *a, + const contextual_request *b) { + return a->ready && b->ready && + a->model == b->model && + a->dims == b->dims && + !strcmp(a->encoding, b->encoding); +} + +static int contextual_request_fits_group(const contextual_request *r, + int group_docs, int group_tokens, + int max_batch, int max_batch_tokens) { + if (group_docs == 0) return 1; + return r->n_docs <= max_batch - group_docs && + r->total_tokens <= max_batch_tokens - group_tokens; +} + +typedef struct { + pplx_context_input_t input; + int output_span_index; + int order; +} contextual_batch_item; + +static int contextual_batch_item_cmp(const void *a, const void *b) { + const contextual_batch_item *ia = a; + const contextual_batch_item *ib = b; + if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1; + if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1; + return ia->order - ib->order; +} -done: - for (di = 0; di < n_docs; di++) { - if (encoded) { - if (!encoded[di]) continue; - for (int ci = 0; ci < chunks[di]; ci++) free(encoded[di][ci]); - free(encoded[di]); - } else if (float_embs) { - free(float_embs[di]); +static void execute_contextual_request_list(contextual_request **reqs, + int n_reqs) { + if (n_reqs <= 0) return; + loaded_model *m = reqs[0]->model; + int dim = m->info->dim; + int total_docs = 0; + int total_chunks = 0; + for (int i = 0; i < n_reqs; i++) { + if (total_docs > INT_MAX - reqs[i]->n_docs || + total_chunks > INT_MAX - reqs[i]->total_chunks) { + for (int k = 0; k < n_reqs; k++) + job_set_error(reqs[k]->j, 500, "contextual batch is too large", + "server_error"); + return; } + total_docs += reqs[i]->n_docs; + total_chunks += reqs[i]->total_chunks; } - free(encoded); - free(float_embs); - free(chunks); + if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { + for (int i = 0; i < n_reqs; i++) + job_set_error(reqs[i]->j, 500, "contextual batch is too large", + "server_error"); + return; + } + + contextual_batch_item *items = + xmalloc((size_t)total_docs * sizeof(*items)); + float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); + int pos = 0; + int span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + for (int d = 0; d < reqs[i]->n_docs; d++) { + contextual_doc *doc = &reqs[i]->docs[d]; + items[pos].input.input.ids = doc->ids; + items[pos].input.input.n_tokens = doc->n_tokens; + items[pos].input.spans = doc->spans; + items[pos].input.n_spans = doc->n_spans; + items[pos].output_span_index = span_offset; + items[pos].order = pos; + span_offset += doc->n_spans; + pos++; + } + } + qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); + + int max_batch = reqs[0]->j->srv->batch_size > 0 + ? reqs[0]->j->srv->batch_size : total_docs; + if (max_batch > total_docs) max_batch = total_docs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + pplx_context_input_t *inputs = + xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = NULL; + size_t batch_emb_cap = 0; + + int failed = 0; + for (int start = 0; start < total_docs;) { + int cur = 0; + int tokens = 0; + int chunks = 0; + while (start + cur < total_docs && cur < max_batch) { + const pplx_context_input_t *input = &items[start + cur].input; + if (cur > 0 && input->input.n_tokens > max_batch_tokens - tokens) + break; + inputs[cur] = *input; + tokens += input->input.n_tokens; + chunks += input->n_spans; + cur++; + } + size_t needed = (size_t)chunks * dim; + if (needed > batch_emb_cap) { + batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs)); + batch_emb_cap = needed; + } + if (model_embed_spans_batch(m, inputs, cur, batch_embs) != 0) { + failed = 1; + break; + } + int batch_span = 0; + for (int i = 0; i < cur; i++) { + int n_spans = inputs[i].n_spans; + memcpy(embs + (size_t)items[start + i].output_span_index * dim, + batch_embs + (size_t)batch_span * dim, + (size_t)n_spans * dim * sizeof(*embs)); + batch_span += n_spans; + } + start += cur; + } + + span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + if (failed) { + job_set_error(reqs[i]->j, 500, "contextual embedding failed", + "server_error"); + } else { + render_contextual_response(reqs[i], + embs + (size_t)span_offset * dim); + } + span_offset += reqs[i]->total_chunks; + } + + free(batch_embs); + free(inputs); + free(embs); + free(items); } static int validate_common(cJSON *root, cJSON *detail, int expect_context, @@ -1377,23 +1460,24 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s, cJSON_Delete(detail); } -static void process_embeddings(job *j, cJSON *root, http_server *s) { - embedding_request r; - prepare_embedding_request(j, root, s, &r); - if (r.ready) - execute_embedding_requests(&r, 1); - embedding_request_free(&r); -} +static void prepare_contextual_request(job *j, cJSON *root, http_server *s, + contextual_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; -static void process_contextual(job *j, cJSON *root, http_server *s) { cJSON *detail = cJSON_CreateArray(); loaded_model *m = NULL; int dims = 0; const char *encoding = NULL; int unloaded = validate_common(root, detail, 1, &m, &dims, &encoding, s); + out->model = m; + out->dims = dims; + out->encoding = encoding; cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); - int n_docs = 0, total_chunks = 0, total_tokens = 0; + int n_docs = 0; + int total_chunks = 0; if (!input) { ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); } else if (!cJSON_IsArray(input)) { @@ -1419,14 +1503,16 @@ static void process_contextual(job *j, cJSON *root, http_server *s) { continue; } int doc_chunks = cJSON_GetArraySize(doc_arr); - total_chunks += doc_chunks; + if (doc_chunks > PPLX_API_MAX_CONTEXT_CHUNKS - total_chunks) + total_chunks = PPLX_API_MAX_CONTEXT_CHUNKS + 1; + else + total_chunks += doc_chunks; if (doc_chunks < 1) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); ve_add(detail, loc, "document must contain at least 1 chunk", "value_error.list.min_items"); } - int doc_tokens = 0; cJSON *chunk; int ci = 0; cJSON_ArrayForEach(chunk, doc_arr) { @@ -1436,45 +1522,101 @@ static void process_contextual(job *j, cJSON *root, http_server *s) { ve_add(detail, loc, "chunk must be a string", "type_error.string"); } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); - } else if (m) { - token_buf t; - if (tokenize_one(m, chunk->valuestring, &t) != 0) { - ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); - } else { - doc_tokens += t.n_tokens; - total_tokens += t.n_tokens; - free(t.ids); - } } ci++; } - if (m && doc_chunks > 1) { - doc_tokens += (doc_chunks - 1) * m->newline_n; - total_tokens += (doc_chunks - 1) * m->newline_n; + di++; + } + if (total_chunks > PPLX_API_MAX_CONTEXT_CHUNKS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", + "value_error.list.max_items"); + } + + if (cJSON_GetArraySize(detail) == 0 && unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", + "model_not_loaded"); + cJSON_Delete(detail); + return; + } + + if (cJSON_GetArraySize(detail) == 0 && m) { + out->n_docs = n_docs; + out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); + int64_t request_tokens = 0; + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + contextual_doc *doc = &out->docs[di]; + int n_chunks = cJSON_GetArraySize(doc_arr); + token_buf *chunk_tokens = + xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); + doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); + doc->n_spans = n_chunks; + int64_t doc_tokens = n_chunks > 1 + ? (int64_t)(n_chunks - 1) * m->newline_n : 0; + int doc_valid = 1; + + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (tokenize_one(m, chunk->valuestring, &chunk_tokens[ci]) != 0) { + ve_add(detail, loc, "tokenization failed", + "value_error.tokenization"); + doc_valid = 0; + } else { + doc_tokens += chunk_tokens[ci].n_tokens; + } + ci++; } + if (doc_tokens > PPLX_API_MAX_ITEM_TOKENS) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); ve_add(detail, loc, "document exceeds 32768 token limit", "value_error.context_length"); + doc_valid = 0; } + request_tokens += doc_tokens; + + if (doc_valid) { + doc->n_tokens = (int)doc_tokens; + doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); + int pos = 0; + for (ci = 0; ci < n_chunks; ci++) { + doc->spans[ci].start = pos; + doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens; + memcpy(doc->ids + pos, chunk_tokens[ci].ids, + (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); + pos += chunk_tokens[ci].n_tokens; + if (ci + 1 < n_chunks) { + memcpy(doc->ids + pos, m->newline_ids, + (size_t)m->newline_n * sizeof(*doc->ids)); + pos += m->newline_n; + } + } + } + free_token_bufs(chunk_tokens, n_chunks); + free(chunk_tokens); di++; } - if (total_chunks > PPLX_API_MAX_CONTEXT_CHUNKS) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", - "value_error.list.max_items"); - if (total_tokens > PPLX_API_MAX_TOTAL_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + if (request_tokens > PPLX_API_MAX_TOTAL_TOKENS) { + ve_add(detail, "[\"body\",\"input\"]", + "request exceeds 120000 token limit", "value_error.context_length"); + } else { + out->total_tokens = (int)request_tokens; + out->total_chunks = total_chunks; + } } if (cJSON_GetArraySize(detail) > 0) { job_set_422(j, detail); - } else if (unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", - "model_not_loaded"); + contextual_request_free(out); + out->j = j; } else { - handle_context_job(j, root, m, dims, encoding); + out->ready = 1; } cJSON_Delete(detail); } @@ -1493,81 +1635,112 @@ static int parse_job_root(job *j, cJSON **out_root) { return 0; } -static void process_job(job *j) { +static void process_unknown_job(job *j) { cJSON *root = NULL; if (parse_job_root(j, &root) != 0) return; - if (!strcmp(j->path, "/v1/embeddings")) { - process_embeddings(j, root, j->srv); - return; - } else if (!strcmp(j->path, "/v1/contextualizedembeddings")) - process_contextual(j, root, j->srv); - else - job_set_error(j, 404, "unknown endpoint", "invalid_request_error"); + job_set_error(j, 404, "unknown endpoint", "invalid_request_error"); cJSON_Delete(root); } static void process_job_group(http_server *s, job **jobs, int n_jobs) { - embedding_request *reqs = xcalloc((size_t)n_jobs, sizeof(*reqs)); - int *is_standard = xcalloc((size_t)n_jobs, sizeof(*is_standard)); + embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs)); + contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs)); + int *kind = xcalloc((size_t)n_jobs, sizeof(*kind)); int *done = xcalloc((size_t)n_jobs, sizeof(*done)); - embedding_request **group = xmalloc((size_t)n_jobs * sizeof(*group)); + embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group)); + contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group)); for (int i = 0; i < n_jobs; i++) { - if (strcmp(jobs[i]->path, "/v1/embeddings")) - continue; cJSON *root = NULL; - is_standard[i] = 1; - if (parse_job_root(jobs[i], &root) == 0) - prepare_embedding_request(jobs[i], root, s, &reqs[i]); + if (!strcmp(jobs[i]->path, "/v1/embeddings")) { + kind[i] = 1; + } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) { + kind[i] = 2; + } else { + continue; + } + if (parse_job_root(jobs[i], &root) != 0) + continue; + if (kind[i] == 1) + prepare_embedding_request(jobs[i], root, s, &std_reqs[i]); + else + prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]); } for (int i = 0; i < n_jobs; i++) { if (done[i]) continue; - if (!is_standard[i]) { - process_job(jobs[i]); + if (!kind[i]) { + process_unknown_job(jobs[i]); enqueue_done(jobs[i]); done[i] = 1; continue; } - if (!reqs[i].ready) { + if ((kind[i] == 1 && !std_reqs[i].ready) || + (kind[i] == 2 && !ctx_reqs[i].ready)) { enqueue_done(jobs[i]); done[i] = 1; continue; } - int group_n = 1; - int group_inputs = reqs[i].n_inputs; - int group_tokens = reqs[i].total_tokens; - group[0] = &reqs[i]; - for (int k = i + 1; k < n_jobs; k++) { - if (!done[k] && is_standard[k] && reqs[k].ready && - embedding_request_compatible(&reqs[i], &reqs[k]) && - embedding_request_fits_group(&reqs[k], group_inputs, - group_tokens, s->batch_size, s->max_batch_tokens)) { - group[group_n++] = &reqs[k]; - group_inputs += reqs[k].n_inputs; - group_tokens += reqs[k].total_tokens; + if (kind[i] == 1) { + int group_n = 1; + int group_inputs = std_reqs[i].n_inputs; + int group_tokens = std_reqs[i].total_tokens; + std_group[0] = &std_reqs[i]; + for (int k = i + 1; k < n_jobs; k++) { + if (!done[k] && kind[k] == 1 && std_reqs[k].ready && + embedding_request_compatible(&std_reqs[i], &std_reqs[k]) && + embedding_request_fits_group(&std_reqs[k], group_inputs, + group_tokens, s->batch_size, s->max_batch_tokens)) { + std_group[group_n++] = &std_reqs[k]; + group_inputs += std_reqs[k].n_inputs; + group_tokens += std_reqs[k].total_tokens; + } + } + execute_embedding_request_list(std_group, group_n); + for (int k = 0; k < group_n; k++) { + int idx = (int)(std_group[k] - std_reqs); + enqueue_done(jobs[idx]); + done[idx] = 1; + } + } else { + int group_n = 1; + int group_docs = ctx_reqs[i].n_docs; + int group_tokens = ctx_reqs[i].total_tokens; + ctx_group[0] = &ctx_reqs[i]; + for (int k = i + 1; k < n_jobs; k++) { + if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready && + contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) && + contextual_request_fits_group(&ctx_reqs[k], group_docs, + group_tokens, s->batch_size, s->max_batch_tokens)) { + ctx_group[group_n++] = &ctx_reqs[k]; + group_docs += ctx_reqs[k].n_docs; + group_tokens += ctx_reqs[k].total_tokens; + } + } + execute_contextual_request_list(ctx_group, group_n); + for (int k = 0; k < group_n; k++) { + int idx = (int)(ctx_group[k] - ctx_reqs); + enqueue_done(jobs[idx]); + done[idx] = 1; } - } - - execute_embedding_request_list(group, group_n); - for (int k = 0; k < group_n; k++) { - int idx = (int)(group[k] - reqs); - enqueue_done(jobs[idx]); - done[idx] = 1; } } - for (int i = 0; i < n_jobs; i++) - embedding_request_free(&reqs[i]); - free(group); + for (int i = 0; i < n_jobs; i++) { + embedding_request_free(&std_reqs[i]); + contextual_request_free(&ctx_reqs[i]); + } + free(ctx_group); + free(std_group); free(done); - free(is_standard); - free(reqs); + free(kind); + free(ctx_reqs); + free(std_reqs); } static void *worker_main(void *arg) { @@ -1625,7 +1798,9 @@ static void *worker_main(void *arg) { int n_jobs = 1; jobs[0] = j; - if (!strcmp(j->path, "/v1/embeddings") && s->batch_size > 1) { + if ((!strcmp(j->path, "/v1/embeddings") || + !strcmp(j->path, "/v1/contextualizedembeddings")) && + s->batch_size > 1) { if (s->batch_wait_us > 0) usleep((useconds_t)s->batch_wait_us); n_jobs += drain_ready_jobs(s, jobs + 1, |
