aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--pplx_embed.c81
-rw-r--r--pplx_embed.h17
-rw-r--r--pplx_embed_mlx.c154
-rw-r--r--pplx_embed_mlx.h8
-rw-r--r--pplx_server.c533
5 files changed, 567 insertions, 226 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index d4b2d02..4aaeda1 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -1044,23 +1044,80 @@ int pplx_pool_spans(const pplx_config_t *cfg, const float *states, int n_tokens,
return 0;
}
-int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws,
- const int *token_ids, int n_tokens,
- const pplx_span_t *spans, int n_spans,
- float *out_embeddings)
+int pplx_model_embed_spans_batch(const pplx_model_t *model,
+ pplx_workspace_t *ws,
+ const pplx_context_input_t *inputs, int batch,
+ float *out_embeddings)
{
- if (!model || !ws || !token_ids || n_tokens <= 0 || !spans ||
- n_spans <= 0 || !out_embeddings)
+ if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
return -1;
- pplx_input_t input = { token_ids, n_tokens };
- int offsets[2] = { 0, n_tokens };
- if (forward_packed_inplace(model, ws, &input, 1, offsets,
- n_tokens, n_tokens, 1) != 0)
+ if (ensure_offsets(ws, batch) != 0) return -1;
+ if ((size_t)batch > SIZE_MAX / sizeof(pplx_input_t)) return -1;
+ pplx_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs));
+ if (!packed_inputs) return -1;
+ int *offsets = ws->offsets;
+ int total_seq = 0;
+ int max_seq = 0;
+ int total_spans = 0;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ int n_tokens = input->input.n_tokens;
+ if (!input->input.ids || n_tokens <= 0 || !input->spans ||
+ input->n_spans <= 0 || total_seq > INT_MAX - n_tokens ||
+ total_spans > INT_MAX - input->n_spans) {
+ free(packed_inputs);
+ return -1;
+ }
+ packed_inputs[b] = input->input;
+ offsets[b] = total_seq;
+ total_seq += n_tokens;
+ total_spans += input->n_spans;
+ if (n_tokens > max_seq) max_seq = n_tokens;
+ for (int s = 0; s < input->n_spans; s++) {
+ int start = input->spans[s].start;
+ int len = input->spans[s].n_tokens;
+ if (start < 0 || len <= 0 || start > n_tokens ||
+ len > n_tokens - start) {
+ free(packed_inputs);
+ return -1;
+ }
+ }
+ }
+ offsets[batch] = total_seq;
+
+ if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets,
+ total_seq, max_seq, 1) != 0) {
+ free(packed_inputs);
return -1;
+ }
+ free(packed_inputs);
+
+ int span_offset = 0;
+ int hidden = model->config.hidden_size;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ if (pplx_pool_spans(&model->config,
+ ws->x + (size_t)offsets[b] * hidden,
+ input->input.n_tokens, input->spans, input->n_spans,
+ out_embeddings + (size_t)span_offset * hidden) != 0)
+ return -1;
+ span_offset += input->n_spans;
+ }
+ return 0;
+}
- return pplx_pool_spans(&model->config, ws->x, n_tokens, spans, n_spans,
- out_embeddings);
+int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws,
+ const int *token_ids, int n_tokens,
+ const pplx_span_t *spans, int n_spans,
+ float *out_embeddings)
+{
+ pplx_context_input_t input = {
+ .input = { token_ids, n_tokens },
+ .spans = spans,
+ .n_spans = n_spans,
+ };
+ return pplx_model_embed_spans_batch(model, ws, &input, 1, out_embeddings);
}
float *pplx_model_forward(const pplx_model_t *model, pplx_workspace_t *ws,
diff --git a/pplx_embed.h b/pplx_embed.h
index 6a159de..3aa4059 100644
--- a/pplx_embed.h
+++ b/pplx_embed.h
@@ -53,6 +53,12 @@ typedef struct {
int n_tokens;
} pplx_span_t;
+typedef struct {
+ pplx_input_t input;
+ const pplx_span_t *spans;
+ int n_spans;
+} pplx_context_input_t;
+
/* ========================================================================
* API
* ======================================================================== */
@@ -208,6 +214,17 @@ int pplx_model_embed_spans(const pplx_model_t *model, pplx_workspace_t *ws,
float *out_embeddings);
/*
+ * Run a packed/ragged contextual document batch and pool every selected span.
+ *
+ * Documents attend independently, RoPE positions restart for each document,
+ * and out_embeddings contains spans in document order.
+ */
+int pplx_model_embed_spans_batch(const pplx_model_t *model,
+ pplx_workspace_t *ws,
+ const pplx_context_input_t *inputs, int batch,
+ float *out_embeddings);
+
+/*
* L2-normalize vec[dim] in place.
*
* Returns 0 on success. Returns -1 for invalid arguments or a zero-length
diff --git a/pplx_embed_mlx.c b/pplx_embed_mlx.c
index c50c647..3157c17 100644
--- a/pplx_embed_mlx.c
+++ b/pplx_embed_mlx.c
@@ -695,12 +695,12 @@ int pplx_mlx_forward_slice_batch(pplx_mlx_ctx_t *ctx,
return 0;
}
-int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
- int n_tokens, const pplx_span_t *spans,
- int n_spans, float *out_embeddings)
+int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx,
+ const pplx_context_input_t *inputs, int batch,
+ float *out_embeddings)
{
- if (!ctx || !token_ids || n_tokens <= 0 || !spans || n_spans <= 0 ||
- !out_embeddings || ctx->layer_start != 0 ||
+ if (!ctx || !inputs || batch <= 0 || !out_embeddings ||
+ ctx->layer_start != 0 ||
ctx->layer_end != ctx->config.n_layers ||
!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
return -1;
@@ -708,33 +708,95 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
const pplx_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
int hidden = c->hidden_size;
- for (int t = 0; t < n_tokens; t++) {
- if (token_ids[t] < 0 || token_ids[t] >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at position %d\n",
- token_ids[t], t);
+ int max_seq = 0;
+ int total_spans = 0;
+ int has_padding = 0;
+ for (int b = 0; b < batch; b++) {
+ const pplx_context_input_t *input = &inputs[b];
+ int n_tokens = input->input.n_tokens;
+ if (!input->input.ids || n_tokens <= 0 || !input->spans ||
+ input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans)
return -1;
+ total_spans += input->n_spans;
+ if (n_tokens > max_seq) max_seq = n_tokens;
+ for (int t = 0; t < n_tokens; t++) {
+ if (input->input.ids[t] < 0 ||
+ input->input.ids[t] >= c->vocab_size) {
+ fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
+ input->input.ids[t], b, t);
+ return -1;
+ }
+ }
+ for (int s = 0; s < input->n_spans; s++) {
+ if (input->spans[s].start < 0 ||
+ input->spans[s].start > n_tokens ||
+ input->spans[s].n_tokens <= 0 ||
+ input->spans[s].n_tokens > n_tokens - input->spans[s].start)
+ return -1;
}
}
- for (int s = 0; s < n_spans; s++) {
- if (spans[s].start < 0 || spans[s].start > n_tokens ||
- spans[s].n_tokens <= 0 ||
- spans[s].n_tokens > n_tokens - spans[s].start)
- return -1;
+ for (int b = 0; b < batch; b++)
+ if (inputs[b].input.n_tokens != max_seq) has_padding = 1;
+
+ if ((size_t)batch > SIZE_MAX / (size_t)max_seq) return -1;
+ size_t rows = (size_t)batch * (size_t)max_seq;
+ if (rows > SIZE_MAX / sizeof(int) ||
+ (has_padding && rows > SIZE_MAX / sizeof(float)))
+ return -1;
+ int *padded_ids = (int *)malloc(rows * sizeof(*padded_ids));
+ float *attn_mask_data = has_padding
+ ? (float *)malloc(rows * sizeof(*attn_mask_data)) : NULL;
+ if (!padded_ids || (has_padding && !attn_mask_data)) {
+ free(padded_ids);
+ free(attn_mask_data);
+ return -1;
+ }
+ for (int b = 0; b < batch; b++) {
+ const pplx_input_t *input = &inputs[b].input;
+ for (int t = 0; t < max_seq; t++) {
+ size_t row = (size_t)b * max_seq + t;
+ if (t < input->n_tokens) {
+ padded_ids[row] = input->ids[t];
+ if (has_padding) attn_mask_data[row] = 0.0f;
+ } else {
+ padded_ids[row] = 0;
+ if (has_padding) attn_mask_data[row] = -1.0e9f;
+ }
+ }
}
mlx_array ids = (mlx_array){0};
+ mlx_array attn_mask = (mlx_array){0};
mlx_array x = (mlx_array){0};
mlx_array x_normed = (mlx_array){0};
mlx_array x_f32 = (mlx_array){0};
mlx_array embeddings = (mlx_array){0};
- mlx_array *pooled = (mlx_array *)calloc((size_t)n_spans, sizeof(*pooled));
+ mlx_array *pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled));
mlx_vector_array pooled_vec = (mlx_vector_array){0};
int rc = -1;
- if (!pooled) return -1;
+ if (!pooled) {
+ free(padded_ids);
+ free(attn_mask_data);
+ return -1;
+ }
- int ids_shape[] = {1, n_tokens};
- ids = mlx_array_new_data(token_ids, ids_shape, 2, MLX_INT32);
+ int ids_shape[] = {batch, max_seq};
+ int attn_mask_shape[] = {batch, 1, 1, max_seq};
+ ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
+ attn_mask = has_padding
+ ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
+ : (mlx_array){0};
+ free(padded_ids);
+ free(attn_mask_data);
if (!arr_ok(ids)) goto cleanup;
+ if (has_padding && !arr_ok(attn_mask)) goto cleanup;
+ if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
+ mlx_array attn_mask_bf16 = mlx_array_new();
+ mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
+ mlx_array_free(attn_mask);
+ attn_mask = attn_mask_bf16;
+ if (!arr_ok(attn_mask)) goto cleanup;
+ }
x = mlx_array_new();
mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
@@ -742,8 +804,12 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
ids = (mlx_array){0};
if (!arr_ok(x)) goto cleanup;
- x = mlx_forward_layers(ctx, x, 1, n_tokens, 0, (mlx_array){0},
+ x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask,
0, c->n_layers);
+ if (has_padding) {
+ mlx_array_free(attn_mask);
+ attn_mask = (mlx_array){0};
+ }
if (!arr_ok(x)) goto cleanup;
x_normed = mlx_array_new();
@@ -758,24 +824,29 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
x_normed = (mlx_array){0};
if (!arr_ok(x_f32)) goto cleanup;
- for (int s = 0; s < n_spans; s++) {
- int start[] = {0, spans[s].start, 0};
- int stop[] = {1, spans[s].start + spans[s].n_tokens, hidden};
- int strides[] = {1, 1, 1};
- mlx_array slice = mlx_array_new();
- mlx_slice(&slice, x_f32, start, 3, stop, 3, strides, 3, S);
- if (!arr_ok(slice)) {
+ int pooled_count = 0;
+ for (int b = 0; b < batch; b++) {
+ for (int s = 0; s < inputs[b].n_spans; s++) {
+ const pplx_span_t *span = &inputs[b].spans[s];
+ int start[] = {b, span->start, 0};
+ int stop[] = {b + 1, span->start + span->n_tokens, hidden};
+ int strides[] = {1, 1, 1};
+ mlx_array slice = mlx_array_new();
+ mlx_slice(&slice, x_f32, start, 3, stop, 3, strides, 3, S);
+ if (!arr_ok(slice)) {
+ mlx_array_free(slice);
+ goto cleanup;
+ }
+
+ pooled[pooled_count] = mlx_array_new();
+ mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S);
mlx_array_free(slice);
- goto cleanup;
+ if (!arr_ok(pooled[pooled_count])) goto cleanup;
+ pooled_count++;
}
-
- pooled[s] = mlx_array_new();
- mlx_mean_axis(&pooled[s], slice, 1, false, S);
- mlx_array_free(slice);
- if (!arr_ok(pooled[s])) goto cleanup;
}
- pooled_vec = mlx_vector_array_new_data(pooled, (size_t)n_spans);
+ pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans);
if (!pooled_vec.ctx) goto cleanup;
embeddings = mlx_array_new();
mlx_concatenate_axis(&embeddings, pooled_vec, 0, S);
@@ -785,22 +856,35 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
const float *data = mlx_array_data_float32(embeddings);
if (data) {
memcpy(out_embeddings, data,
- (size_t)n_spans * hidden * sizeof(float));
+ (size_t)total_spans * hidden * sizeof(float));
rc = 0;
}
cleanup:
mlx_array_free(embeddings);
if (pooled_vec.ctx) mlx_vector_array_free(pooled_vec);
- for (int s = 0; s < n_spans; s++) mlx_array_free(pooled[s]);
+ for (int s = 0; s < total_spans; s++) mlx_array_free(pooled[s]);
free(pooled);
mlx_array_free(x_f32);
mlx_array_free(x_normed);
mlx_array_free(x);
+ mlx_array_free(attn_mask);
mlx_array_free(ids);
return rc;
}
+int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
+ int n_tokens, const pplx_span_t *spans,
+ int n_spans, float *out_embeddings)
+{
+ pplx_context_input_t input = {
+ .input = { token_ids, n_tokens },
+ .spans = spans,
+ .n_spans = n_spans,
+ };
+ return pplx_mlx_embed_spans_batch(ctx, &input, 1, out_embeddings);
+}
+
float *pplx_mlx_embed(pplx_mlx_ctx_t *ctx, const int *token_ids, int n_tokens)
{
if (!ctx || !token_ids || n_tokens <= 0) return NULL;
diff --git a/pplx_embed_mlx.h b/pplx_embed_mlx.h
index b45fc8a..2b7164d 100644
--- a/pplx_embed_mlx.h
+++ b/pplx_embed_mlx.h
@@ -74,6 +74,14 @@ int pplx_mlx_embed_spans(pplx_mlx_ctx_t *ctx, const int *token_ids,
int n_tokens, const pplx_span_t *spans,
int n_spans, float *out_embeddings);
+/*
+ * Run a padded dense contextual document batch and pool every selected span.
+ * out_embeddings contains spans in document order.
+ */
+int pplx_mlx_embed_spans_batch(pplx_mlx_ctx_t *ctx,
+ const pplx_context_input_t *inputs, int batch,
+ float *out_embeddings);
+
/* Get the config (so main.c can read hidden_size etc.) */
const pplx_config_t *pplx_mlx_config(const pplx_mlx_ctx_t *ctx);
diff --git a/pplx_server.c b/pplx_server.c
index 8968ddc..61dc1eb 100644
--- a/pplx_server.c
+++ b/pplx_server.c
@@ -923,6 +923,26 @@ typedef struct {
int ready;
} embedding_request;
+typedef struct {
+ int *ids;
+ int n_tokens;
+ pplx_span_t *spans;
+ int n_spans;
+} contextual_doc;
+
+typedef struct {
+ job *j;
+ cJSON *root;
+ loaded_model *model;
+ int dims;
+ const char *encoding;
+ contextual_doc *docs;
+ int n_docs;
+ int total_chunks;
+ int total_tokens;
+ int ready;
+} contextual_request;
+
static void free_token_bufs(token_buf *t, int n) {
if (!t) return;
for (int i = 0; i < n; i++) free(t[i].ids);
@@ -937,6 +957,17 @@ static void embedding_request_free(embedding_request *r) {
memset(r, 0, sizeof(*r));
}
+static void contextual_request_free(contextual_request *r) {
+ if (!r) return;
+ for (int i = 0; i < r->n_docs; i++) {
+ free(r->docs[i].ids);
+ free(r->docs[i].spans);
+ }
+ free(r->docs);
+ if (r->root) cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
static int tokenize_one(loaded_model *m, const char *text, token_buf *out) {
memset(out, 0, sizeof(*out));
out->ids = qwen_tokenizer_encode(m->tok, text, &out->n_tokens);
@@ -957,14 +988,15 @@ static int model_embed_batch(loaded_model *m, const pplx_input_t *inputs,
return pplx_model_embed_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
}
-static int model_embed_spans(loaded_model *m, const int *ids, int n_tokens,
- const pplx_span_t *spans, int n_spans, float *out) {
+static int model_embed_spans_batch(loaded_model *m,
+ const pplx_context_input_t *inputs,
+ int batch, float *out) {
#ifdef USE_MLX
if (m->mlx_ctx)
- return pplx_mlx_embed_spans(m->mlx_ctx, ids, n_tokens, spans, n_spans, out);
+ return pplx_mlx_embed_spans_batch(m->mlx_ctx, inputs, batch, out);
#endif
- return pplx_model_embed_spans(m->cpu_model, m->cpu_ws, ids, n_tokens,
- spans, n_spans, out);
+ return pplx_model_embed_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch,
+ out);
}
static void append_embedding_object(sbuf *b, int index, const char *embedding) {
@@ -1119,118 +1151,169 @@ static void execute_embedding_request_list(embedding_request **reqs, int n_reqs)
free(items);
}
-static void execute_embedding_requests(embedding_request *reqs, int n_reqs) {
- if (n_reqs <= 0) return;
- embedding_request **list = xmalloc((size_t)n_reqs * sizeof(*list));
- for (int i = 0; i < n_reqs; i++)
- list[i] = &reqs[i];
- execute_embedding_request_list(list, n_reqs);
- free(list);
-}
-
-static void handle_context_job(job *j, cJSON *root, loaded_model *m,
- int dims, const char *encoding) {
- cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
- int n_docs = cJSON_GetArraySize(input);
- int *chunks = xcalloc((size_t)n_docs, sizeof(*chunks));
- char ***encoded = strcmp(encoding, "float")
- ? xcalloc((size_t)n_docs, sizeof(*encoded)) : NULL;
- float **float_embs = !strcmp(encoding, "float")
- ? xcalloc((size_t)n_docs, sizeof(*float_embs)) : NULL;
- int total_tokens = 0;
-
- cJSON *doc_arr;
- int di = 0;
- cJSON_ArrayForEach(doc_arr, input) {
- chunks[di] = cJSON_GetArraySize(doc_arr);
- if (encoded) encoded[di] = xcalloc((size_t)chunks[di], sizeof(char *));
- token_buf *chunk_tokens = xcalloc((size_t)chunks[di], sizeof(*chunk_tokens));
- pplx_span_t *spans = xcalloc((size_t)chunks[di], sizeof(*spans));
- int doc_tokens = 0;
- cJSON *chunk;
- int ci = 0;
- cJSON_ArrayForEach(chunk, doc_arr) {
- if (tokenize_one(m, chunk->valuestring, &chunk_tokens[ci]) != 0) {
- job_set_error(j, 500, "tokenization failed", "server_error");
- free_token_bufs(chunk_tokens, chunks[di]);
- free(chunk_tokens); free(spans);
- goto done;
- }
- doc_tokens += chunk_tokens[ci].n_tokens;
- if (ci + 1 < chunks[di]) doc_tokens += m->newline_n;
- ci++;
- }
- int *doc_ids = xmalloc((size_t)doc_tokens * sizeof(int));
- int pos = 0;
- for (ci = 0; ci < chunks[di]; ci++) {
- spans[ci].start = pos;
- spans[ci].n_tokens = chunk_tokens[ci].n_tokens;
- memcpy(doc_ids + pos, chunk_tokens[ci].ids,
- (size_t)chunk_tokens[ci].n_tokens * sizeof(int));
- pos += chunk_tokens[ci].n_tokens;
- if (ci + 1 < chunks[di] && m->newline_n > 0) {
- memcpy(doc_ids + pos, m->newline_ids,
- (size_t)m->newline_n * sizeof(int));
- pos += m->newline_n;
- }
- }
- total_tokens += doc_tokens;
- float *embs = xmalloc((size_t)chunks[di] * m->info->dim * sizeof(float));
- if (model_embed_spans(m, doc_ids, doc_tokens, spans, chunks[di], embs) != 0) {
- free(embs); free(doc_ids);
- job_set_error(j, 500, "contextual embedding failed", "server_error");
- free_token_bufs(chunk_tokens, chunks[di]);
- free(chunk_tokens); free(spans);
- goto done;
- }
- if (encoded) {
- for (ci = 0; ci < chunks[di]; ci++)
- encoded[di][ci] = encode_embedding(embs + (size_t)ci * m->info->dim,
- dims, encoding);
- free(embs);
- } else {
- float_embs[di] = embs;
- }
- free(doc_ids);
- free_token_bufs(chunk_tokens, chunks[di]);
- free(chunk_tokens);
- free(spans);
- di++;
- }
-
+static void render_contextual_response(contextual_request *r,
+ const float *embs) {
sbuf b = {0};
sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
- for (di = 0; di < n_docs; di++) {
+ int span_offset = 0;
+ for (int di = 0; di < r->n_docs; di++) {
if (di) sbuf_putc(&b, ',');
sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di);
- for (int ci = 0; ci < chunks[di]; ci++) {
+ for (int ci = 0; ci < r->docs[di].n_spans; ci++) {
if (ci) sbuf_putc(&b, ',');
- if (encoded)
- append_embedding_object(&b, ci, encoded[di][ci]);
- else
+ const float *emb = embs + (size_t)span_offset * r->model->info->dim;
+ if (!strcmp(r->encoding, "float")) {
append_float_embedding_object(&b, ci,
- float_embs[di] + (size_t)ci * m->info->dim, dims);
+ emb, r->dims);
+ } else {
+ char *encoded = encode_embedding(emb, r->dims, r->encoding);
+ append_embedding_object(&b, ci, encoded);
+ free(encoded);
+ }
+ span_offset++;
}
sbuf_puts(&b, "]}");
}
sbuf_printf(&b,
"],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}",
- m->info->id, total_tokens, total_tokens);
- set_response_from_buf(j, &b);
+ r->model->info->id, r->total_tokens, r->total_tokens);
+ set_response_from_buf(r->j, &b);
+}
+
+static int contextual_request_compatible(const contextual_request *a,
+ const contextual_request *b) {
+ return a->ready && b->ready &&
+ a->model == b->model &&
+ a->dims == b->dims &&
+ !strcmp(a->encoding, b->encoding);
+}
+
+static int contextual_request_fits_group(const contextual_request *r,
+ int group_docs, int group_tokens,
+ int max_batch, int max_batch_tokens) {
+ if (group_docs == 0) return 1;
+ return r->n_docs <= max_batch - group_docs &&
+ r->total_tokens <= max_batch_tokens - group_tokens;
+}
+
+typedef struct {
+ pplx_context_input_t input;
+ int output_span_index;
+ int order;
+} contextual_batch_item;
+
+static int contextual_batch_item_cmp(const void *a, const void *b) {
+ const contextual_batch_item *ia = a;
+ const contextual_batch_item *ib = b;
+ if (ia->input.input.n_tokens < ib->input.input.n_tokens) return -1;
+ if (ia->input.input.n_tokens > ib->input.input.n_tokens) return 1;
+ return ia->order - ib->order;
+}
-done:
- for (di = 0; di < n_docs; di++) {
- if (encoded) {
- if (!encoded[di]) continue;
- for (int ci = 0; ci < chunks[di]; ci++) free(encoded[di][ci]);
- free(encoded[di]);
- } else if (float_embs) {
- free(float_embs[di]);
+static void execute_contextual_request_list(contextual_request **reqs,
+ int n_reqs) {
+ if (n_reqs <= 0) return;
+ loaded_model *m = reqs[0]->model;
+ int dim = m->info->dim;
+ int total_docs = 0;
+ int total_chunks = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ if (total_docs > INT_MAX - reqs[i]->n_docs ||
+ total_chunks > INT_MAX - reqs[i]->total_chunks) {
+ for (int k = 0; k < n_reqs; k++)
+ job_set_error(reqs[k]->j, 500, "contextual batch is too large",
+ "server_error");
+ return;
}
+ total_docs += reqs[i]->n_docs;
+ total_chunks += reqs[i]->total_chunks;
}
- free(encoded);
- free(float_embs);
- free(chunks);
+ if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) {
+ for (int i = 0; i < n_reqs; i++)
+ job_set_error(reqs[i]->j, 500, "contextual batch is too large",
+ "server_error");
+ return;
+ }
+
+ contextual_batch_item *items =
+ xmalloc((size_t)total_docs * sizeof(*items));
+ float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs));
+ int pos = 0;
+ int span_offset = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ for (int d = 0; d < reqs[i]->n_docs; d++) {
+ contextual_doc *doc = &reqs[i]->docs[d];
+ items[pos].input.input.ids = doc->ids;
+ items[pos].input.input.n_tokens = doc->n_tokens;
+ items[pos].input.spans = doc->spans;
+ items[pos].input.n_spans = doc->n_spans;
+ items[pos].output_span_index = span_offset;
+ items[pos].order = pos;
+ span_offset += doc->n_spans;
+ pos++;
+ }
+ }
+ qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp);
+
+ int max_batch = reqs[0]->j->srv->batch_size > 0
+ ? reqs[0]->j->srv->batch_size : total_docs;
+ if (max_batch > total_docs) max_batch = total_docs;
+ int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
+ pplx_context_input_t *inputs =
+ xmalloc((size_t)max_batch * sizeof(*inputs));
+ float *batch_embs = NULL;
+ size_t batch_emb_cap = 0;
+
+ int failed = 0;
+ for (int start = 0; start < total_docs;) {
+ int cur = 0;
+ int tokens = 0;
+ int chunks = 0;
+ while (start + cur < total_docs && cur < max_batch) {
+ const pplx_context_input_t *input = &items[start + cur].input;
+ if (cur > 0 && input->input.n_tokens > max_batch_tokens - tokens)
+ break;
+ inputs[cur] = *input;
+ tokens += input->input.n_tokens;
+ chunks += input->n_spans;
+ cur++;
+ }
+ size_t needed = (size_t)chunks * dim;
+ if (needed > batch_emb_cap) {
+ batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs));
+ batch_emb_cap = needed;
+ }
+ if (model_embed_spans_batch(m, inputs, cur, batch_embs) != 0) {
+ failed = 1;
+ break;
+ }
+ int batch_span = 0;
+ for (int i = 0; i < cur; i++) {
+ int n_spans = inputs[i].n_spans;
+ memcpy(embs + (size_t)items[start + i].output_span_index * dim,
+ batch_embs + (size_t)batch_span * dim,
+ (size_t)n_spans * dim * sizeof(*embs));
+ batch_span += n_spans;
+ }
+ start += cur;
+ }
+
+ span_offset = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ if (failed) {
+ job_set_error(reqs[i]->j, 500, "contextual embedding failed",
+ "server_error");
+ } else {
+ render_contextual_response(reqs[i],
+ embs + (size_t)span_offset * dim);
+ }
+ span_offset += reqs[i]->total_chunks;
+ }
+
+ free(batch_embs);
+ free(inputs);
+ free(embs);
+ free(items);
}
static int validate_common(cJSON *root, cJSON *detail, int expect_context,
@@ -1377,23 +1460,24 @@ static void prepare_embedding_request(job *j, cJSON *root, http_server *s,
cJSON_Delete(detail);
}
-static void process_embeddings(job *j, cJSON *root, http_server *s) {
- embedding_request r;
- prepare_embedding_request(j, root, s, &r);
- if (r.ready)
- execute_embedding_requests(&r, 1);
- embedding_request_free(&r);
-}
+static void prepare_contextual_request(job *j, cJSON *root, http_server *s,
+ contextual_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
-static void process_contextual(job *j, cJSON *root, http_server *s) {
cJSON *detail = cJSON_CreateArray();
loaded_model *m = NULL;
int dims = 0;
const char *encoding = NULL;
int unloaded = validate_common(root, detail, 1, &m, &dims, &encoding, s);
+ out->model = m;
+ out->dims = dims;
+ out->encoding = encoding;
cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
- int n_docs = 0, total_chunks = 0, total_tokens = 0;
+ int n_docs = 0;
+ int total_chunks = 0;
if (!input) {
ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
} else if (!cJSON_IsArray(input)) {
@@ -1419,14 +1503,16 @@ static void process_contextual(job *j, cJSON *root, http_server *s) {
continue;
}
int doc_chunks = cJSON_GetArraySize(doc_arr);
- total_chunks += doc_chunks;
+ if (doc_chunks > PPLX_API_MAX_CONTEXT_CHUNKS - total_chunks)
+ total_chunks = PPLX_API_MAX_CONTEXT_CHUNKS + 1;
+ else
+ total_chunks += doc_chunks;
if (doc_chunks < 1) {
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
ve_add(detail, loc, "document must contain at least 1 chunk",
"value_error.list.min_items");
}
- int doc_tokens = 0;
cJSON *chunk;
int ci = 0;
cJSON_ArrayForEach(chunk, doc_arr) {
@@ -1436,45 +1522,101 @@ static void process_contextual(job *j, cJSON *root, http_server *s) {
ve_add(detail, loc, "chunk must be a string", "type_error.string");
} else if (!chunk->valuestring || chunk->valuestring[0] == '\0') {
ve_add(detail, loc, "chunk must not be empty", "value_error.empty");
- } else if (m) {
- token_buf t;
- if (tokenize_one(m, chunk->valuestring, &t) != 0) {
- ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
- } else {
- doc_tokens += t.n_tokens;
- total_tokens += t.n_tokens;
- free(t.ids);
- }
}
ci++;
}
- if (m && doc_chunks > 1) {
- doc_tokens += (doc_chunks - 1) * m->newline_n;
- total_tokens += (doc_chunks - 1) * m->newline_n;
+ di++;
+ }
+ if (total_chunks > PPLX_API_MAX_CONTEXT_CHUNKS)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks",
+ "value_error.list.max_items");
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && unloaded) {
+ job_set_error(j, 503, "requested model is valid but not loaded",
+ "model_not_loaded");
+ cJSON_Delete(detail);
+ return;
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && m) {
+ out->n_docs = n_docs;
+ out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs));
+ int64_t request_tokens = 0;
+ cJSON *doc_arr;
+ int di = 0;
+ cJSON_ArrayForEach(doc_arr, input) {
+ contextual_doc *doc = &out->docs[di];
+ int n_chunks = cJSON_GetArraySize(doc_arr);
+ token_buf *chunk_tokens =
+ xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
+ doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans));
+ doc->n_spans = n_chunks;
+ int64_t doc_tokens = n_chunks > 1
+ ? (int64_t)(n_chunks - 1) * m->newline_n : 0;
+ int doc_valid = 1;
+
+ cJSON *chunk;
+ int ci = 0;
+ cJSON_ArrayForEach(chunk, doc_arr) {
+ char loc[80];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
+ if (tokenize_one(m, chunk->valuestring, &chunk_tokens[ci]) != 0) {
+ ve_add(detail, loc, "tokenization failed",
+ "value_error.tokenization");
+ doc_valid = 0;
+ } else {
+ doc_tokens += chunk_tokens[ci].n_tokens;
+ }
+ ci++;
}
+
if (doc_tokens > PPLX_API_MAX_ITEM_TOKENS) {
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
ve_add(detail, loc, "document exceeds 32768 token limit",
"value_error.context_length");
+ doc_valid = 0;
}
+ request_tokens += doc_tokens;
+
+ if (doc_valid) {
+ doc->n_tokens = (int)doc_tokens;
+ doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids));
+ int pos = 0;
+ for (ci = 0; ci < n_chunks; ci++) {
+ doc->spans[ci].start = pos;
+ doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens;
+ memcpy(doc->ids + pos, chunk_tokens[ci].ids,
+ (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids));
+ pos += chunk_tokens[ci].n_tokens;
+ if (ci + 1 < n_chunks) {
+ memcpy(doc->ids + pos, m->newline_ids,
+ (size_t)m->newline_n * sizeof(*doc->ids));
+ pos += m->newline_n;
+ }
+ }
+ }
+ free_token_bufs(chunk_tokens, n_chunks);
+ free(chunk_tokens);
di++;
}
- if (total_chunks > PPLX_API_MAX_CONTEXT_CHUNKS)
- ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks",
- "value_error.list.max_items");
- if (total_tokens > PPLX_API_MAX_TOTAL_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ if (request_tokens > PPLX_API_MAX_TOTAL_TOKENS) {
+ ve_add(detail, "[\"body\",\"input\"]",
+ "request exceeds 120000 token limit",
"value_error.context_length");
+ } else {
+ out->total_tokens = (int)request_tokens;
+ out->total_chunks = total_chunks;
+ }
}
if (cJSON_GetArraySize(detail) > 0) {
job_set_422(j, detail);
- } else if (unloaded) {
- job_set_error(j, 503, "requested model is valid but not loaded",
- "model_not_loaded");
+ contextual_request_free(out);
+ out->j = j;
} else {
- handle_context_job(j, root, m, dims, encoding);
+ out->ready = 1;
}
cJSON_Delete(detail);
}
@@ -1493,81 +1635,112 @@ static int parse_job_root(job *j, cJSON **out_root) {
return 0;
}
-static void process_job(job *j) {
+static void process_unknown_job(job *j) {
cJSON *root = NULL;
if (parse_job_root(j, &root) != 0)
return;
- if (!strcmp(j->path, "/v1/embeddings")) {
- process_embeddings(j, root, j->srv);
- return;
- } else if (!strcmp(j->path, "/v1/contextualizedembeddings"))
- process_contextual(j, root, j->srv);
- else
- job_set_error(j, 404, "unknown endpoint", "invalid_request_error");
+ job_set_error(j, 404, "unknown endpoint", "invalid_request_error");
cJSON_Delete(root);
}
static void process_job_group(http_server *s, job **jobs, int n_jobs) {
- embedding_request *reqs = xcalloc((size_t)n_jobs, sizeof(*reqs));
- int *is_standard = xcalloc((size_t)n_jobs, sizeof(*is_standard));
+ embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs));
+ contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs));
+ int *kind = xcalloc((size_t)n_jobs, sizeof(*kind));
int *done = xcalloc((size_t)n_jobs, sizeof(*done));
- embedding_request **group = xmalloc((size_t)n_jobs * sizeof(*group));
+ embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group));
+ contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group));
for (int i = 0; i < n_jobs; i++) {
- if (strcmp(jobs[i]->path, "/v1/embeddings"))
- continue;
cJSON *root = NULL;
- is_standard[i] = 1;
- if (parse_job_root(jobs[i], &root) == 0)
- prepare_embedding_request(jobs[i], root, s, &reqs[i]);
+ if (!strcmp(jobs[i]->path, "/v1/embeddings")) {
+ kind[i] = 1;
+ } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) {
+ kind[i] = 2;
+ } else {
+ continue;
+ }
+ if (parse_job_root(jobs[i], &root) != 0)
+ continue;
+ if (kind[i] == 1)
+ prepare_embedding_request(jobs[i], root, s, &std_reqs[i]);
+ else
+ prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]);
}
for (int i = 0; i < n_jobs; i++) {
if (done[i])
continue;
- if (!is_standard[i]) {
- process_job(jobs[i]);
+ if (!kind[i]) {
+ process_unknown_job(jobs[i]);
enqueue_done(jobs[i]);
done[i] = 1;
continue;
}
- if (!reqs[i].ready) {
+ if ((kind[i] == 1 && !std_reqs[i].ready) ||
+ (kind[i] == 2 && !ctx_reqs[i].ready)) {
enqueue_done(jobs[i]);
done[i] = 1;
continue;
}
- int group_n = 1;
- int group_inputs = reqs[i].n_inputs;
- int group_tokens = reqs[i].total_tokens;
- group[0] = &reqs[i];
- for (int k = i + 1; k < n_jobs; k++) {
- if (!done[k] && is_standard[k] && reqs[k].ready &&
- embedding_request_compatible(&reqs[i], &reqs[k]) &&
- embedding_request_fits_group(&reqs[k], group_inputs,
- group_tokens, s->batch_size, s->max_batch_tokens)) {
- group[group_n++] = &reqs[k];
- group_inputs += reqs[k].n_inputs;
- group_tokens += reqs[k].total_tokens;
+ if (kind[i] == 1) {
+ int group_n = 1;
+ int group_inputs = std_reqs[i].n_inputs;
+ int group_tokens = std_reqs[i].total_tokens;
+ std_group[0] = &std_reqs[i];
+ for (int k = i + 1; k < n_jobs; k++) {
+ if (!done[k] && kind[k] == 1 && std_reqs[k].ready &&
+ embedding_request_compatible(&std_reqs[i], &std_reqs[k]) &&
+ embedding_request_fits_group(&std_reqs[k], group_inputs,
+ group_tokens, s->batch_size, s->max_batch_tokens)) {
+ std_group[group_n++] = &std_reqs[k];
+ group_inputs += std_reqs[k].n_inputs;
+ group_tokens += std_reqs[k].total_tokens;
+ }
+ }
+ execute_embedding_request_list(std_group, group_n);
+ for (int k = 0; k < group_n; k++) {
+ int idx = (int)(std_group[k] - std_reqs);
+ enqueue_done(jobs[idx]);
+ done[idx] = 1;
+ }
+ } else {
+ int group_n = 1;
+ int group_docs = ctx_reqs[i].n_docs;
+ int group_tokens = ctx_reqs[i].total_tokens;
+ ctx_group[0] = &ctx_reqs[i];
+ for (int k = i + 1; k < n_jobs; k++) {
+ if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready &&
+ contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) &&
+ contextual_request_fits_group(&ctx_reqs[k], group_docs,
+ group_tokens, s->batch_size, s->max_batch_tokens)) {
+ ctx_group[group_n++] = &ctx_reqs[k];
+ group_docs += ctx_reqs[k].n_docs;
+ group_tokens += ctx_reqs[k].total_tokens;
+ }
+ }
+ execute_contextual_request_list(ctx_group, group_n);
+ for (int k = 0; k < group_n; k++) {
+ int idx = (int)(ctx_group[k] - ctx_reqs);
+ enqueue_done(jobs[idx]);
+ done[idx] = 1;
}
- }
-
- execute_embedding_request_list(group, group_n);
- for (int k = 0; k < group_n; k++) {
- int idx = (int)(group[k] - reqs);
- enqueue_done(jobs[idx]);
- done[idx] = 1;
}
}
- for (int i = 0; i < n_jobs; i++)
- embedding_request_free(&reqs[i]);
- free(group);
+ for (int i = 0; i < n_jobs; i++) {
+ embedding_request_free(&std_reqs[i]);
+ contextual_request_free(&ctx_reqs[i]);
+ }
+ free(ctx_group);
+ free(std_group);
free(done);
- free(is_standard);
- free(reqs);
+ free(kind);
+ free(ctx_reqs);
+ free(std_reqs);
}
static void *worker_main(void *arg) {
@@ -1625,7 +1798,9 @@ static void *worker_main(void *arg) {
int n_jobs = 1;
jobs[0] = j;
- if (!strcmp(j->path, "/v1/embeddings") && s->batch_size > 1) {
+ if ((!strcmp(j->path, "/v1/embeddings") ||
+ !strcmp(j->path, "/v1/contextualizedembeddings")) &&
+ s->batch_size > 1) {
if (s->batch_wait_us > 0)
usleep((useconds_t)s->batch_wait_us);
n_jobs += drain_ready_jobs(s, jobs + 1,