aboutsummaryrefslogtreecommitdiff
path: root/src/server_handlers.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/server_handlers.c')
-rw-r--r--src/server_handlers.c934
1 files changed, 934 insertions, 0 deletions
diff --git a/src/server_handlers.c b/src/server_handlers.c
new file mode 100644
index 0000000..95df1de
--- /dev/null
+++ b/src/server_handlers.c
@@ -0,0 +1,934 @@
+#include "server_internal.h"
+#include "server_util.h"
+#include "embed.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* ========================================================================
+ * Endpoint handlers: prepare, batch, execute, render
+ *
+ * One prepare_* per endpoint validates the request JSON and tokenizes its
+ * inputs into a *_request; the worker (server_schedule.c) groups compatible
+ * ready requests and calls the matching execute_*_request_list, which runs the
+ * batched forward pass through model_embed_* and renders each response via
+ * server_encode.c. Rerank runs MaxSim over late-interaction token vectors.
+ * ======================================================================== */
+
+int embedding_request_compatible(const embedding_request *a, const embedding_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
+ !strcmp(a->encoding, b->encoding);
+}
+
+typedef struct {
+ embed_input_t input;
+ int output_index;
+} embedding_batch_item;
+
+static int embedding_batch_item_cmp(const void *a, const void *b) {
+ const embedding_batch_item *ia = a;
+ const embedding_batch_item *ib = b;
+ if (ia->input.n_tokens < ib->input.n_tokens)
+ return -1;
+ if (ia->input.n_tokens > ib->input.n_tokens)
+ return 1;
+ return ia->output_index - ib->output_index;
+}
+
+int embedding_request_fits_group(const embedding_request *r,
+ int group_inputs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_inputs == 0)
+ return 1;
+ return r->n_inputs <= max_batch - group_inputs &&
+ r->total_tokens <= max_batch_tokens - group_tokens;
+}
+
+void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
+ if (n_reqs <= 0)
+ return;
+ loaded_model *m = reqs[0]->model;
+ int dim = m->info->dim;
+ int total_inputs = 0;
+ for (int i = 0; i < n_reqs; i++)
+ total_inputs += reqs[i]->n_inputs;
+
+ embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items));
+ float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float));
+
+ int pos = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ for (int k = 0; k < reqs[i]->n_inputs; k++) {
+ items[pos].input = reqs[i]->inputs[k];
+ items[pos].output_index = pos;
+ pos++;
+ }
+ }
+ qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp);
+
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs;
+ if (max_batch > total_inputs)
+ max_batch = total_inputs;
+ int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
+ embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
+ float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float));
+
+ int failed = 0;
+ uint64_t infer_ns = 0;
+ for (int start = 0; start < total_inputs;) {
+ int cur = 0;
+ int tokens = 0;
+ while (start + cur < total_inputs && cur < max_batch) {
+ int n_tokens = items[start + cur].input.n_tokens;
+ if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens))
+ break;
+ inputs[cur] = items[start + cur].input;
+ tokens += n_tokens;
+ cur++;
+ }
+ uint64_t t0 = nstime();
+ int embed_rc = model_embed_batch(m, inputs, cur, batch_embs);
+ infer_ns += nstime() - t0;
+ if (embed_rc != 0) {
+ failed = 1;
+ break;
+ }
+ for (int i = 0; i < cur; i++)
+ memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim,
+ (size_t)dim * sizeof(float));
+ start += cur;
+ }
+
+ pos = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ reqs[i]->j->infer_ns += infer_ns;
+ if (failed) {
+ job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
+ } else {
+ uint64_t t0 = nstime();
+ if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
+ job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
+ reqs[i]->j->encode_ns += nstime() - t0;
+ }
+ pos += reqs[i]->n_inputs;
+ }
+
+ free(batch_embs);
+ free(inputs);
+ free(embs);
+ free(items);
+}
+
+int contextual_request_compatible(const contextual_request *a, const contextual_request *b) {
+ return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
+ !strcmp(a->encoding, b->encoding);
+}
+
+int contextual_request_fits_group(const contextual_request *r,
+ int group_docs,
+ int group_tokens,
+ int max_batch,
+ int max_batch_tokens) {
+ if (group_docs == 0)
+ return 1;
+ return r->n_docs <= max_batch - group_docs &&
+ r->total_tokens <= max_batch_tokens - group_tokens;
+}
+
+typedef struct {
+ embed_context_input_t input;
+ int output_span_index;
+ int order;
+} contextual_batch_item;
+
+static int contextual_batch_item_cmp(const void *a, const void *b) {
+ const contextual_batch_item *ia = a;
+ const contextual_batch_item *ib = b;
+ if (ia->input.input.n_tokens < ib->input.input.n_tokens)
+ return -1;
+ if (ia->input.input.n_tokens > ib->input.input.n_tokens)
+ return 1;
+ return ia->order - ib->order;
+}
+
+void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
+ if (n_reqs <= 0)
+ return;
+ loaded_model *m = reqs[0]->model;
+ int dim = m->info->dim;
+ int total_docs = 0;
+ int total_chunks = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ if (total_docs > INT_MAX - reqs[i]->n_docs ||
+ total_chunks > INT_MAX - reqs[i]->total_chunks) {
+ for (int k = 0; k < n_reqs; k++)
+ job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error");
+ return;
+ }
+ total_docs += reqs[i]->n_docs;
+ total_chunks += reqs[i]->total_chunks;
+ }
+ if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) {
+ for (int i = 0; i < n_reqs; i++)
+ job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error");
+ return;
+ }
+
+ contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items));
+ float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs));
+ int pos = 0;
+ int span_offset = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ for (int d = 0; d < reqs[i]->n_docs; d++) {
+ contextual_doc *doc = &reqs[i]->docs[d];
+ items[pos].input.input.ids = doc->ids;
+ items[pos].input.input.n_tokens = doc->n_tokens;
+ items[pos].input.spans = doc->spans;
+ items[pos].input.n_spans = doc->n_spans;
+ items[pos].output_span_index = span_offset;
+ items[pos].order = pos;
+ span_offset += doc->n_spans;
+ pos++;
+ }
+ }
+ qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp);
+
+ int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs;
+ if (max_batch > total_docs)
+ max_batch = total_docs;
+ int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
+ embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
+ float *batch_embs = xmalloc(sizeof(*batch_embs));
+ size_t batch_emb_cap = 0;
+
+ int failed = 0;
+ uint64_t infer_ns = 0;
+ for (int start = 0; start < total_docs;) {
+ int cur = 0;
+ int tokens = 0;
+ int chunks = 0;
+ while (start + cur < total_docs && cur < max_batch) {
+ const embed_context_input_t *input = &items[start + cur].input;
+ if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens,
+ max_batch_tokens))
+ break;
+ inputs[cur] = *input;
+ tokens += input->input.n_tokens;
+ chunks += input->n_spans;
+ cur++;
+ }
+ size_t needed = (size_t)chunks * dim;
+ if (needed > batch_emb_cap) {
+ batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs));
+ batch_emb_cap = needed;
+ }
+ uint64_t t0 = nstime();
+ int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs);
+ infer_ns += nstime() - t0;
+ if (embed_rc != 0) {
+ failed = 1;
+ break;
+ }
+ int batch_span = 0;
+ for (int i = 0; i < cur; i++) {
+ int n_spans = inputs[i].n_spans;
+ memcpy(embs + (size_t)items[start + i].output_span_index * dim,
+ batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs));
+ batch_span += n_spans;
+ }
+ start += cur;
+ }
+
+ span_offset = 0;
+ for (int i = 0; i < n_reqs; i++) {
+ reqs[i]->j->infer_ns += infer_ns;
+ if (failed) {
+ job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error");
+ } else {
+ uint64_t t0 = nstime();
+ render_contextual_response(reqs[i], embs + (size_t)span_offset * dim);
+ reqs[i]->j->encode_ns += nstime() - t0;
+ }
+ span_offset += reqs[i]->total_chunks;
+ }
+
+ free(batch_embs);
+ free(inputs);
+ free(embs);
+ free(items);
+}
+
+static int validate_common(cJSON *root,
+ cJSON *detail,
+ model_kind expected_kind,
+ loaded_model **out_model,
+ int *out_dims,
+ const char **out_encoding,
+ http_server *s) {
+ cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
+ const char *model_id = NULL;
+ if (!model_item) {
+ ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
+ } else if (!cJSON_IsString(model_item) || !model_item->valuestring) {
+ ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
+ } else {
+ model_id = model_item->valuestring;
+ model_slot slot = model_slot_for_id(model_id);
+ if (slot == MODEL_UNKNOWN || k_models[slot].kind != expected_kind) {
+ ve_add(detail, "[\"body\",\"model\"]",
+ "value is not a valid enum member for this endpoint", "enum");
+ } else {
+ loaded_model *m = &s->models[slot];
+ if (!m->info) {
+ *out_model = NULL;
+ } else {
+ *out_model = m;
+ }
+ }
+ }
+ model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN;
+ embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY;
+ const char *enc = encoding_from_root(root, detail, api);
+ int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128;
+ int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560;
+ int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc);
+ *out_encoding = enc;
+ *out_dims = dims;
+ return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN &&
+ !s->models[model_slot_for_id(model_id)].info
+ ? 503
+ : 0;
+}
+
+void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
+
+ cJSON *detail = cJSON_CreateArray();
+ loaded_model *m = NULL;
+ int dims = 0;
+ const char *encoding = NULL;
+ int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s);
+ out->model = m;
+ out->dims = dims;
+ out->encoding = encoding;
+
+ /* Resolve request behavior from the model registry independent of load
+ * state. The query instruction, when requested, applies to every input. */
+ cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
+ model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring)
+ ? model_slot_for_id(model_item->valuestring)
+ : MODEL_UNKNOWN;
+ const char *query_instruct =
+ api_slot != MODEL_UNKNOWN &&
+ text_type_is_query(root, detail, k_models[api_slot].query_instruct)
+ ? k_models[api_slot].query_instruct
+ : NULL;
+
+ cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
+ int n_inputs = 0;
+ if (!input) {
+ ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
+ } else if (cJSON_IsString(input)) {
+ if (!input->valuestring || input->valuestring[0] == '\0')
+ ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty");
+ n_inputs = 1;
+ } else if (cJSON_IsArray(input)) {
+ n_inputs = cJSON_GetArraySize(input);
+ if (n_inputs < 1)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item",
+ "value_error.list.min_items");
+ else if (n_inputs > EMBED_API_MAX_STANDARD_INPUTS)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items",
+ "value_error.list.max_items");
+ cJSON *item;
+ int i = 0;
+ cJSON_ArrayForEach(item, input) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i);
+ if (!cJSON_IsString(item)) {
+ ve_add(detail, loc, "input item must be a string", "type_error.string");
+ } else if (!item->valuestring || item->valuestring[0] == '\0') {
+ ve_add(detail, loc, "input item must not be empty", "value_error.empty");
+ }
+ i++;
+ }
+ } else {
+ ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings",
+ "type_error");
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && unloaded) {
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
+ cJSON_Delete(detail);
+ return;
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && m && input) {
+ out->n_inputs = n_inputs;
+ out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens));
+ out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs));
+
+ int idx = 0;
+ if (input && cJSON_IsString(input) && input->valuestring) {
+ if (tokenize_input(m, j, input->valuestring, query_instruct, &out->tokens[0]) != 0) {
+ ve_add(detail, "[\"body\",\"input\"]", "tokenization failed",
+ "value_error.tokenization");
+ } else {
+ out->inputs[0].ids = out->tokens[0].ids;
+ out->inputs[0].n_tokens = out->tokens[0].n_tokens;
+ out->total_tokens = out->tokens[0].n_tokens;
+ if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
+ ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+ } else {
+ cJSON *item;
+ cJSON_ArrayForEach(item, input) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx);
+ if (!cJSON_IsString(item) || !item->valuestring) {
+ ve_add(detail, loc, "input item must be a string", "type_error.string");
+ idx++;
+ continue;
+ }
+ if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) !=
+ 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
+ } else {
+ out->inputs[idx].ids = out->tokens[idx].ids;
+ out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
+ out->total_tokens += out->tokens[idx].n_tokens;
+ if (out->tokens[idx].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
+ ve_add(detail, loc, "input exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+ idx++;
+ }
+ }
+
+ if (out->total_tokens > EMBED_API_MAX_TOTAL_TOKENS)
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ "value_error.context_length");
+ }
+
+ if (cJSON_GetArraySize(detail) > 0) {
+ job_set_422(j, detail);
+ embedding_request_free(out);
+ out->j = j;
+ } else {
+ out->ready = 1;
+ }
+ cJSON_Delete(detail);
+}
+
+void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
+
+ cJSON *detail = cJSON_CreateArray();
+ loaded_model *m = NULL;
+ int dims = 0;
+ const char *encoding = NULL;
+ int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s);
+ out->model = m;
+ out->dims = dims;
+ out->encoding = encoding;
+
+ cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
+ int n_docs = 0;
+ int total_chunks = 0;
+ if (!input) {
+ ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
+ } else if (!cJSON_IsArray(input)) {
+ ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays",
+ "type_error.array");
+ } else {
+ n_docs = cJSON_GetArraySize(input);
+ if (n_docs < 1)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document",
+ "value_error.list.min_items");
+ else if (n_docs > EMBED_API_MAX_CONTEXT_DOCS)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents",
+ "value_error.list.max_items");
+ cJSON *doc_arr;
+ int di = 0;
+ cJSON_ArrayForEach(doc_arr, input) {
+ if (!cJSON_IsArray(doc_arr)) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
+ ve_add(detail, loc, "document must be an array of strings", "type_error.array");
+ di++;
+ continue;
+ }
+ int doc_chunks = cJSON_GetArraySize(doc_arr);
+ if (doc_chunks > EMBED_API_MAX_CONTEXT_CHUNKS - total_chunks)
+ total_chunks = EMBED_API_MAX_CONTEXT_CHUNKS + 1;
+ else
+ total_chunks += doc_chunks;
+ if (doc_chunks < 1) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
+ ve_add(detail, loc, "document must contain at least 1 chunk",
+ "value_error.list.min_items");
+ }
+ cJSON *chunk;
+ int ci = 0;
+ cJSON_ArrayForEach(chunk, doc_arr) {
+ char loc[80];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
+ if (!cJSON_IsString(chunk)) {
+ ve_add(detail, loc, "chunk must be a string", "type_error.string");
+ } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') {
+ ve_add(detail, loc, "chunk must not be empty", "value_error.empty");
+ }
+ ci++;
+ }
+ di++;
+ }
+ if (total_chunks > EMBED_API_MAX_CONTEXT_CHUNKS)
+ ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks",
+ "value_error.list.max_items");
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && unloaded) {
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
+ cJSON_Delete(detail);
+ return;
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && m) {
+ out->n_docs = n_docs;
+ out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs));
+ int64_t request_tokens = 0;
+ cJSON *doc_arr;
+ int di = 0;
+ cJSON_ArrayForEach(doc_arr, input) {
+ contextual_doc *doc = &out->docs[di];
+ int n_chunks = cJSON_GetArraySize(doc_arr);
+ token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
+ doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans));
+ doc->n_spans = n_chunks;
+ int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0;
+ int doc_valid = 1;
+
+ cJSON *chunk;
+ int ci = 0;
+ cJSON_ArrayForEach(chunk, doc_arr) {
+ char loc[80];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
+ if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
+ doc_valid = 0;
+ } else {
+ doc_tokens += chunk_tokens[ci].n_tokens;
+ }
+ ci++;
+ }
+
+ if (doc_tokens > EMBED_API_MAX_ITEM_TOKENS) {
+ char loc[64];
+ snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
+ ve_add(detail, loc, "document exceeds 32768 token limit",
+ "value_error.context_length");
+ doc_valid = 0;
+ }
+ request_tokens += doc_tokens;
+
+ if (doc_valid) {
+ doc->n_tokens = (int)doc_tokens;
+ doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids));
+ int pos = 0;
+ for (ci = 0; ci < n_chunks; ci++) {
+ doc->spans[ci].start = pos;
+ doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens;
+ /* n_tokens > 0 implies ids != NULL (tokenize_one sets both
+ * together), and the guard avoids a memcpy(dst, NULL, 0) for
+ * an empty chunk that the static analyzer reports as UB. */
+ if (chunk_tokens[ci].n_tokens > 0)
+ memcpy(doc->ids + pos, chunk_tokens[ci].ids,
+ (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids));
+ pos += chunk_tokens[ci].n_tokens;
+ if (ci + 1 < n_chunks) {
+ doc->ids[pos++] = m->context_separator_id;
+ }
+ }
+ }
+ free_token_bufs(chunk_tokens, n_chunks);
+ free(chunk_tokens);
+ di++;
+ }
+ if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) {
+ ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
+ "value_error.context_length");
+ } else {
+ out->total_tokens = (int)request_tokens;
+ out->total_chunks = total_chunks;
+ }
+ }
+
+ if (cJSON_GetArraySize(detail) > 0) {
+ job_set_422(j, detail);
+ contextual_request_free(out);
+ out->j = j;
+ } else {
+ out->ready = 1;
+ }
+ cJSON_Delete(detail);
+}
+
+static int
+validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) {
+ cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model");
+ if (!item) {
+ ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
+ return 0;
+ }
+ if (!cJSON_IsString(item) || !item->valuestring) {
+ ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
+ return 0;
+ }
+ model_slot slot = model_slot_for_id(item->valuestring);
+ if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) {
+ ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint",
+ "enum");
+ return 0;
+ }
+ *out_model = s->models[slot].info ? &s->models[slot] : NULL;
+ return *out_model ? 0 : 503;
+}
+
+void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) {
+ memset(out, 0, sizeof(*out));
+ out->j = j;
+ out->root = root;
+
+ cJSON *detail = cJSON_CreateArray();
+ int unloaded = validate_rerank_model(root, detail, s, &out->model);
+ cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query");
+ cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents");
+ /* query_str is set only once query passes every check; the tokenize step
+ * below runs only when detail is empty, so it is non-NULL there. Hoisting it
+ * also keeps the validated value explicit for the static analyzer. */
+ const char *query_str = NULL;
+ if (!query) {
+ ve_add(detail, "[\"body\",\"query\"]", "field required", "missing");
+ } else if (!cJSON_IsString(query)) {
+ ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string");
+ } else if (!query->valuestring || !query->valuestring[0]) {
+ ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty");
+ } else {
+ query_str = query->valuestring;
+ }
+
+ int n_documents = 0;
+ if (!documents) {
+ ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing");
+ } else if (!cJSON_IsArray(documents)) {
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings",
+ "type_error.array");
+ } else {
+ n_documents = cJSON_GetArraySize(documents);
+ if (n_documents < 1)
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item",
+ "value_error.list.min_items");
+ else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS)
+ ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items",
+ "value_error.list.max_items");
+ cJSON *document;
+ int i = 0;
+ cJSON_ArrayForEach(document, documents) {
+ char loc[72];
+ snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++);
+ if (!cJSON_IsString(document))
+ ve_add(detail, loc, "document must be a string", "type_error.string");
+ else if (!document->valuestring || !document->valuestring[0])
+ ve_add(detail, loc, "document must not be empty", "value_error.empty");
+ }
+ }
+
+ cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n");
+ cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k");
+ if (top_n && top_k) {
+ ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one",
+ "value_error.conflict");
+ }
+ cJSON *top = top_n ? top_n : top_k;
+ out->top_n = n_documents;
+ if (top) {
+ if (!cjson_is_integer(top)) {
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
+ "top_n must be an integer", "type_error.integer");
+ } else {
+ out->top_n = top->valueint;
+ if (out->top_n < 1 || out->top_n > n_documents)
+ ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
+ "top_n must be between 1 and the number of documents", "value_error.range");
+ }
+ }
+
+ cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents");
+ if (return_documents) {
+ if (!cJSON_IsBool(return_documents))
+ ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean",
+ "type_error.bool");
+ else
+ out->return_documents = cJSON_IsTrue(return_documents);
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && unloaded) {
+ job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded");
+ cJSON_Delete(detail);
+ return;
+ }
+
+ if (cJSON_GetArraySize(detail) == 0 && out->model) {
+ out->n_documents = n_documents;
+ out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
+ if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) {
+ ve_add(detail, "[\"body\",\"query\"]", "tokenization failed",
+ "value_error.tokenization");
+ } else {
+ out->query_tokens = out->query.n_tokens;
+ if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS)
+ ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit",
+ "value_error.context_length");
+ }
+
+ cJSON *document;
+ int i = 0;
+ cJSON_ArrayForEach(document, documents) {
+ char loc[72];
+ snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i);
+ if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) !=
+ 0) {
+ ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
+ } else {
+ int n = out->documents[i].n_tokens;
+ if (n > EMBED_API_MAX_ITEM_TOKENS)
+ ve_add(detail, loc, "document exceeds 32768 token limit",
+ "value_error.context_length");
+ if (out->document_tokens <= INT_MAX - n)
+ out->document_tokens += n;
+ else
+ out->document_tokens = INT_MAX;
+ }
+ i++;
+ }
+ if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) {
+ ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit",
+ "value_error.context_length");
+ }
+ }
+
+ if (cJSON_GetArraySize(detail) > 0) {
+ job_set_422(j, detail);
+ rerank_request_free(out);
+ out->j = j;
+ } else {
+ out->ready = 1;
+ }
+ cJSON_Delete(detail);
+}
+
+typedef struct {
+ int index;
+ float score;
+} rerank_result;
+
+static int rerank_result_cmp(const void *a, const void *b) {
+ const rerank_result *ra = a;
+ const rerank_result *rb = b;
+ if (ra->score > rb->score)
+ return -1;
+ if (ra->score < rb->score)
+ return 1;
+ return ra->index - rb->index;
+}
+
+static int execute_rerank_cpu(rerank_request *r, float *scores) {
+ loaded_model *m = r->model;
+ int dim = m->info->token_dim;
+ int total_doc_vecs = 0;
+ for (int i = 0; i < r->n_documents; i++)
+ total_doc_vecs += r->documents[i].n_keep;
+
+ float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
+ float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
+ int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
+
+ /* Encode every candidate in one block-diagonal forward. The encoder packs
+ * each document's kept token vectors back-to-back and fills offsets, so the
+ * cost of N short documents collapses to a single packed forward. */
+ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
+ const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
+ int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
+ int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
+ for (int i = 0; i < r->n_documents; i++) {
+ doc_ids[i] = r->documents[i].ids;
+ keep[i] = r->documents[i].keep;
+ n_tokens[i] = r->documents[i].n_tokens;
+ n_keep[i] = r->documents[i].n_keep;
+ }
+
+ int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids,
+ r->query.n_tokens, 1, query);
+ if (rc == 0)
+ rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens,
+ keep, n_keep, r->n_documents, 1, docs, offsets);
+ if (rc == 0)
+ rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
+ scores);
+ free(n_keep);
+ free(n_tokens);
+ free(keep);
+ free(doc_ids);
+ free(offsets);
+ free(docs);
+ free(query);
+ return rc;
+}
+
+#ifdef USE_MLX
+static int execute_rerank_mlx(rerank_request *r, float *scores) {
+ loaded_model *m = r->model;
+ embed_mlx_late_vectors_t *query =
+ embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1);
+
+ /* Encode every candidate in one padded transformer pass. The encoder packs
+ * each document's kept token vectors back-to-back and fills offsets, so the
+ * cost of N short documents collapses to a single batched forward. */
+ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
+ const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
+ int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
+ int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
+ int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
+ for (int i = 0; i < r->n_documents; i++) {
+ doc_ids[i] = r->documents[i].ids;
+ keep[i] = r->documents[i].keep;
+ n_tokens[i] = r->documents[i].n_tokens;
+ n_keep[i] = r->documents[i].n_keep;
+ }
+ embed_mlx_late_vectors_t *packed =
+ query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep,
+ r->n_documents, 1, offsets)
+ : NULL;
+ int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets,
+ r->n_documents, scores)
+ : -1;
+ embed_mlx_late_vectors_free(packed);
+ embed_mlx_late_vectors_free(query);
+ free(offsets);
+ free(n_keep);
+ free(n_tokens);
+ free(keep);
+ free(doc_ids);
+ return rc;
+}
+#endif
+
+#ifdef USE_CUDA
+static int execute_rerank_cuda(rerank_request *r, float *scores) {
+ loaded_model *m = r->model;
+ int dim = m->info->token_dim;
+ int total_doc_vecs = 0;
+ for (int i = 0; i < r->n_documents; i++)
+ total_doc_vecs += r->documents[i].n_keep;
+
+ float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
+ float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
+ int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
+
+ /* GPU encodes the query and every candidate (one packed forward); MaxSim
+ * runs on the host, where the grouped-GEMM scorer beats a device graph. */
+ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
+ const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
+ int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
+ int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
+ for (int i = 0; i < r->n_documents; i++) {
+ doc_ids[i] = r->documents[i].ids;
+ keep[i] = r->documents[i].keep;
+ n_tokens[i] = r->documents[i].n_tokens;
+ n_keep[i] = r->documents[i].n_keep;
+ }
+
+ int rc =
+ embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query);
+ if (rc == 0)
+ rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep,
+ r->n_documents, 1, docs, offsets);
+ if (rc == 0)
+ rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
+ scores);
+ free(n_keep);
+ free(n_tokens);
+ free(keep);
+ free(doc_ids);
+ free(offsets);
+ free(docs);
+ free(query);
+ return rc;
+}
+#endif
+
+void execute_rerank_request(rerank_request *r) {
+ if (!r || !r->model) {
+ if (r && r->j)
+ job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error");
+ return;
+ }
+ float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
+ uint64_t t0 = nstime();
+#if defined(USE_MLX)
+ int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores);
+#elif defined(USE_CUDA)
+ int rc =
+ r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores);
+#else
+ int rc = execute_rerank_cpu(r, scores);
+#endif
+ r->j->infer_ns += nstime() - t0;
+ if (rc != 0) {
+ free(scores);
+ job_set_error(r->j, 500, "late-interaction reranking failed", "server_error");
+ return;
+ }
+
+ rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked));
+ for (int i = 0; i < r->n_documents; i++) {
+ ranked[i].index = i;
+ ranked[i].score = scores[i];
+ }
+ qsort(ranked, (size_t)r->n_documents, sizeof(*ranked), rerank_result_cmp);
+
+ t0 = nstime();
+ sbuf b = {0};
+ sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id);
+ cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents");
+ for (int i = 0; i < r->top_n; i++) {
+ if (i)
+ sbuf_putc(&b, ',');
+ sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index,
+ (double)ranked[i].score);
+ if (r->return_documents) {
+ cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index);
+ sbuf_puts(&b, ",\"document\":");
+ append_json_string(&b, document->valuestring);
+ }
+ sbuf_putc(&b, '}');
+ }
+ sbuf_printf(&b,
+ "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d,"
+ "\"total_tokens\":%d}}",
+ r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens);
+ set_response_from_buf(r->j, &b);
+ r->j->encode_ns += nstime() - t0;
+ free(ranked);
+ free(scores);
+}