diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 18:43:18 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 18:43:18 +0200 |
| commit | 9facc2228ce32d70fa656c3b28436740c6ebd559 (patch) | |
| tree | 63131a2cfb8e9cf822e31cd1d8239d243fa6deb7 /src | |
| parent | 5f7e80e26d4f957701bd61a23f5e75039e0a87cd (diff) | |
| download | ffwd-9facc2228ce32d70fa656c3b28436740c6ebd559.tar.gz ffwd-9facc2228ce32d70fa656c3b28436740c6ebd559.zip | |
server: extract endpoint handlers into server_handlers.c
Diffstat (limited to 'src')
| -rw-r--r-- | src/server.c | 913 | ||||
| -rw-r--r-- | src/server_handlers.c | 934 | ||||
| -rw-r--r-- | src/server_internal.h | 20 |
3 files changed, 954 insertions, 913 deletions
diff --git a/src/server.c b/src/server.c index f52d8d1..db54661 100644 --- a/src/server.c +++ b/src/server.c @@ -7,7 +7,6 @@ #include "server_internal.h" #include "server_util.h" #include "sbuf.h" -#include "base64.h" #include "embed.h" #include "safetensors.h" #include "kernels.h" @@ -29,7 +28,6 @@ #include <errno.h> #include <fcntl.h> #include <limits.h> -#include <math.h> #include <netinet/in.h> #include <pthread.h> #include <signal.h> @@ -358,917 +356,6 @@ static void job_set_timing_header(job *j) { j->extra_headers = xstrdup(buf); } -static int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { - return a->ready && b->ready && a->model == b->model && a->dims == b->dims && - !strcmp(a->encoding, b->encoding); -} - -typedef struct { - embed_input_t input; - int output_index; -} embedding_batch_item; - -static int embedding_batch_item_cmp(const void *a, const void *b) { - const embedding_batch_item *ia = a; - const embedding_batch_item *ib = b; - if (ia->input.n_tokens < ib->input.n_tokens) - return -1; - if (ia->input.n_tokens > ib->input.n_tokens) - return 1; - return ia->output_index - ib->output_index; -} - -static int embedding_request_fits_group(const embedding_request *r, - int group_inputs, - int group_tokens, - int max_batch, - int max_batch_tokens) { - if (group_inputs == 0) - return 1; - return r->n_inputs <= max_batch - group_inputs && - r->total_tokens <= max_batch_tokens - group_tokens; -} - -static void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { - if (n_reqs <= 0) - return; - loaded_model *m = reqs[0]->model; - int dim = m->info->dim; - int total_inputs = 0; - for (int i = 0; i < n_reqs; i++) - total_inputs += reqs[i]->n_inputs; - - embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items)); - float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float)); - - int pos = 0; - for (int i = 0; i < n_reqs; i++) { - for (int k = 0; k < reqs[i]->n_inputs; k++) { - items[pos].input = reqs[i]->inputs[k]; - items[pos].output_index = pos; - pos++; - } - } - qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); - - int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs; - if (max_batch > total_inputs) - max_batch = total_inputs; - int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); - float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); - - int failed = 0; - uint64_t infer_ns = 0; - for (int start = 0; start < total_inputs;) { - int cur = 0; - int tokens = 0; - while (start + cur < total_inputs && cur < max_batch) { - int n_tokens = items[start + cur].input.n_tokens; - if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens)) - break; - inputs[cur] = items[start + cur].input; - tokens += n_tokens; - cur++; - } - uint64_t t0 = nstime(); - int embed_rc = model_embed_batch(m, inputs, cur, batch_embs); - infer_ns += nstime() - t0; - if (embed_rc != 0) { - failed = 1; - break; - } - for (int i = 0; i < cur; i++) - memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim, - (size_t)dim * sizeof(float)); - start += cur; - } - - pos = 0; - for (int i = 0; i < n_reqs; i++) { - reqs[i]->j->infer_ns += infer_ns; - if (failed) { - job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); - } else { - uint64_t t0 = nstime(); - if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) - job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); - reqs[i]->j->encode_ns += nstime() - t0; - } - pos += reqs[i]->n_inputs; - } - - free(batch_embs); - free(inputs); - free(embs); - free(items); -} - -static int contextual_request_compatible(const contextual_request *a, const contextual_request *b) { - return a->ready && b->ready && a->model == b->model && a->dims == b->dims && - !strcmp(a->encoding, b->encoding); -} - -static int contextual_request_fits_group(const contextual_request *r, - int group_docs, - int group_tokens, - int max_batch, - int max_batch_tokens) { - if (group_docs == 0) - return 1; - return r->n_docs <= max_batch - group_docs && - r->total_tokens <= max_batch_tokens - group_tokens; -} - -typedef struct { - embed_context_input_t input; - int output_span_index; - int order; -} contextual_batch_item; - -static int contextual_batch_item_cmp(const void *a, const void *b) { - const contextual_batch_item *ia = a; - const contextual_batch_item *ib = b; - if (ia->input.input.n_tokens < ib->input.input.n_tokens) - return -1; - if (ia->input.input.n_tokens > ib->input.input.n_tokens) - return 1; - return ia->order - ib->order; -} - -static void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { - if (n_reqs <= 0) - return; - loaded_model *m = reqs[0]->model; - int dim = m->info->dim; - int total_docs = 0; - int total_chunks = 0; - for (int i = 0; i < n_reqs; i++) { - if (total_docs > INT_MAX - reqs[i]->n_docs || - total_chunks > INT_MAX - reqs[i]->total_chunks) { - for (int k = 0; k < n_reqs; k++) - job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error"); - return; - } - total_docs += reqs[i]->n_docs; - total_chunks += reqs[i]->total_chunks; - } - if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { - for (int i = 0; i < n_reqs; i++) - job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error"); - return; - } - - contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items)); - float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); - int pos = 0; - int span_offset = 0; - for (int i = 0; i < n_reqs; i++) { - for (int d = 0; d < reqs[i]->n_docs; d++) { - contextual_doc *doc = &reqs[i]->docs[d]; - items[pos].input.input.ids = doc->ids; - items[pos].input.input.n_tokens = doc->n_tokens; - items[pos].input.spans = doc->spans; - items[pos].input.n_spans = doc->n_spans; - items[pos].output_span_index = span_offset; - items[pos].order = pos; - span_offset += doc->n_spans; - pos++; - } - } - qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); - - int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs; - if (max_batch > total_docs) - max_batch = total_docs; - int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); - float *batch_embs = xmalloc(sizeof(*batch_embs)); - size_t batch_emb_cap = 0; - - int failed = 0; - uint64_t infer_ns = 0; - for (int start = 0; start < total_docs;) { - int cur = 0; - int tokens = 0; - int chunks = 0; - while (start + cur < total_docs && cur < max_batch) { - const embed_context_input_t *input = &items[start + cur].input; - if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens, - max_batch_tokens)) - break; - inputs[cur] = *input; - tokens += input->input.n_tokens; - chunks += input->n_spans; - cur++; - } - size_t needed = (size_t)chunks * dim; - if (needed > batch_emb_cap) { - batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs)); - batch_emb_cap = needed; - } - uint64_t t0 = nstime(); - int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs); - infer_ns += nstime() - t0; - if (embed_rc != 0) { - failed = 1; - break; - } - int batch_span = 0; - for (int i = 0; i < cur; i++) { - int n_spans = inputs[i].n_spans; - memcpy(embs + (size_t)items[start + i].output_span_index * dim, - batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs)); - batch_span += n_spans; - } - start += cur; - } - - span_offset = 0; - for (int i = 0; i < n_reqs; i++) { - reqs[i]->j->infer_ns += infer_ns; - if (failed) { - job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error"); - } else { - uint64_t t0 = nstime(); - render_contextual_response(reqs[i], embs + (size_t)span_offset * dim); - reqs[i]->j->encode_ns += nstime() - t0; - } - span_offset += reqs[i]->total_chunks; - } - - free(batch_embs); - free(inputs); - free(embs); - free(items); -} - -static int validate_common(cJSON *root, - cJSON *detail, - model_kind expected_kind, - loaded_model **out_model, - int *out_dims, - const char **out_encoding, - http_server *s) { - cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); - const char *model_id = NULL; - if (!model_item) { - ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); - } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { - ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); - } else { - model_id = model_item->valuestring; - model_slot slot = model_slot_for_id(model_id); - if (slot == MODEL_UNKNOWN || k_models[slot].kind != expected_kind) { - ve_add(detail, "[\"body\",\"model\"]", - "value is not a valid enum member for this endpoint", "enum"); - } else { - loaded_model *m = &s->models[slot]; - if (!m->info) { - *out_model = NULL; - } else { - *out_model = m; - } - } - } - model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN; - embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY; - const char *enc = encoding_from_root(root, detail, api); - int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128; - int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560; - int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc); - *out_encoding = enc; - *out_dims = dims; - return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && - !s->models[model_slot_for_id(model_id)].info - ? 503 - : 0; -} - -static void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) { - memset(out, 0, sizeof(*out)); - out->j = j; - out->root = root; - - cJSON *detail = cJSON_CreateArray(); - loaded_model *m = NULL; - int dims = 0; - const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); - out->model = m; - out->dims = dims; - out->encoding = encoding; - - /* Resolve request behavior from the model registry independent of load - * state. The query instruction, when requested, applies to every input. */ - cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); - model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring) - ? model_slot_for_id(model_item->valuestring) - : MODEL_UNKNOWN; - const char *query_instruct = - api_slot != MODEL_UNKNOWN && - text_type_is_query(root, detail, k_models[api_slot].query_instruct) - ? k_models[api_slot].query_instruct - : NULL; - - cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); - int n_inputs = 0; - if (!input) { - ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); - } else if (cJSON_IsString(input)) { - if (!input->valuestring || input->valuestring[0] == '\0') - ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty"); - n_inputs = 1; - } else if (cJSON_IsArray(input)) { - n_inputs = cJSON_GetArraySize(input); - if (n_inputs < 1) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", - "value_error.list.min_items"); - else if (n_inputs > EMBED_API_MAX_STANDARD_INPUTS) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", - "value_error.list.max_items"); - cJSON *item; - int i = 0; - cJSON_ArrayForEach(item, input) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i); - if (!cJSON_IsString(item)) { - ve_add(detail, loc, "input item must be a string", "type_error.string"); - } else if (!item->valuestring || item->valuestring[0] == '\0') { - ve_add(detail, loc, "input item must not be empty", "value_error.empty"); - } - i++; - } - } else { - ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings", - "type_error"); - } - - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); - cJSON_Delete(detail); - return; - } - - if (cJSON_GetArraySize(detail) == 0 && m && input) { - out->n_inputs = n_inputs; - out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens)); - out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs)); - - int idx = 0; - if (input && cJSON_IsString(input) && input->valuestring) { - if (tokenize_input(m, j, input->valuestring, query_instruct, &out->tokens[0]) != 0) { - ve_add(detail, "[\"body\",\"input\"]", "tokenization failed", - "value_error.tokenization"); - } else { - out->inputs[0].ids = out->tokens[0].ids; - out->inputs[0].n_tokens = out->tokens[0].n_tokens; - out->total_tokens = out->tokens[0].n_tokens; - if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", - "value_error.context_length"); - } - } else { - cJSON *item; - cJSON_ArrayForEach(item, input) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); - if (!cJSON_IsString(item) || !item->valuestring) { - ve_add(detail, loc, "input item must be a string", "type_error.string"); - idx++; - continue; - } - if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != - 0) { - ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); - } else { - out->inputs[idx].ids = out->tokens[idx].ids; - out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; - out->total_tokens += out->tokens[idx].n_tokens; - if (out->tokens[idx].n_tokens > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, loc, "input exceeds 32768 token limit", - "value_error.context_length"); - } - idx++; - } - } - - if (out->total_tokens > EMBED_API_MAX_TOTAL_TOKENS) - ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", - "value_error.context_length"); - } - - if (cJSON_GetArraySize(detail) > 0) { - job_set_422(j, detail); - embedding_request_free(out); - out->j = j; - } else { - out->ready = 1; - } - cJSON_Delete(detail); -} - -static void -prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) { - memset(out, 0, sizeof(*out)); - out->j = j; - out->root = root; - - cJSON *detail = cJSON_CreateArray(); - loaded_model *m = NULL; - int dims = 0; - const char *encoding = NULL; - int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); - out->model = m; - out->dims = dims; - out->encoding = encoding; - - cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); - int n_docs = 0; - int total_chunks = 0; - if (!input) { - ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); - } else if (!cJSON_IsArray(input)) { - ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays", - "type_error.array"); - } else { - n_docs = cJSON_GetArraySize(input); - if (n_docs < 1) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", - "value_error.list.min_items"); - else if (n_docs > EMBED_API_MAX_CONTEXT_DOCS) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", - "value_error.list.max_items"); - cJSON *doc_arr; - int di = 0; - cJSON_ArrayForEach(doc_arr, input) { - if (!cJSON_IsArray(doc_arr)) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document must be an array of strings", "type_error.array"); - di++; - continue; - } - int doc_chunks = cJSON_GetArraySize(doc_arr); - if (doc_chunks > EMBED_API_MAX_CONTEXT_CHUNKS - total_chunks) - total_chunks = EMBED_API_MAX_CONTEXT_CHUNKS + 1; - else - total_chunks += doc_chunks; - if (doc_chunks < 1) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document must contain at least 1 chunk", - "value_error.list.min_items"); - } - cJSON *chunk; - int ci = 0; - cJSON_ArrayForEach(chunk, doc_arr) { - char loc[80]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); - if (!cJSON_IsString(chunk)) { - ve_add(detail, loc, "chunk must be a string", "type_error.string"); - } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { - ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); - } - ci++; - } - di++; - } - if (total_chunks > EMBED_API_MAX_CONTEXT_CHUNKS) - ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", - "value_error.list.max_items"); - } - - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); - cJSON_Delete(detail); - return; - } - - if (cJSON_GetArraySize(detail) == 0 && m) { - out->n_docs = n_docs; - out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); - int64_t request_tokens = 0; - cJSON *doc_arr; - int di = 0; - cJSON_ArrayForEach(doc_arr, input) { - contextual_doc *doc = &out->docs[di]; - int n_chunks = cJSON_GetArraySize(doc_arr); - token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); - doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); - doc->n_spans = n_chunks; - int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; - int doc_valid = 1; - - cJSON *chunk; - int ci = 0; - cJSON_ArrayForEach(chunk, doc_arr) { - char loc[80]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); - if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) { - ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); - doc_valid = 0; - } else { - doc_tokens += chunk_tokens[ci].n_tokens; - } - ci++; - } - - if (doc_tokens > EMBED_API_MAX_ITEM_TOKENS) { - char loc[64]; - snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); - ve_add(detail, loc, "document exceeds 32768 token limit", - "value_error.context_length"); - doc_valid = 0; - } - request_tokens += doc_tokens; - - if (doc_valid) { - doc->n_tokens = (int)doc_tokens; - doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); - int pos = 0; - for (ci = 0; ci < n_chunks; ci++) { - doc->spans[ci].start = pos; - doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens; - /* n_tokens > 0 implies ids != NULL (tokenize_one sets both - * together), and the guard avoids a memcpy(dst, NULL, 0) for - * an empty chunk that the static analyzer reports as UB. */ - if (chunk_tokens[ci].n_tokens > 0) - memcpy(doc->ids + pos, chunk_tokens[ci].ids, - (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); - pos += chunk_tokens[ci].n_tokens; - if (ci + 1 < n_chunks) { - doc->ids[pos++] = m->context_separator_id; - } - } - } - free_token_bufs(chunk_tokens, n_chunks); - free(chunk_tokens); - di++; - } - if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) { - ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", - "value_error.context_length"); - } else { - out->total_tokens = (int)request_tokens; - out->total_chunks = total_chunks; - } - } - - if (cJSON_GetArraySize(detail) > 0) { - job_set_422(j, detail); - contextual_request_free(out); - out->j = j; - } else { - out->ready = 1; - } - cJSON_Delete(detail); -} - -static int -validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) { - cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model"); - if (!item) { - ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); - return 0; - } - if (!cJSON_IsString(item) || !item->valuestring) { - ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); - return 0; - } - model_slot slot = model_slot_for_id(item->valuestring); - if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) { - ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", - "enum"); - return 0; - } - *out_model = s->models[slot].info ? &s->models[slot] : NULL; - return *out_model ? 0 : 503; -} - -static void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) { - memset(out, 0, sizeof(*out)); - out->j = j; - out->root = root; - - cJSON *detail = cJSON_CreateArray(); - int unloaded = validate_rerank_model(root, detail, s, &out->model); - cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query"); - cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents"); - if (!query) { - ve_add(detail, "[\"body\",\"query\"]", "field required", "missing"); - } else if (!cJSON_IsString(query)) { - ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string"); - } else if (!query->valuestring || !query->valuestring[0]) { - ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty"); - } - - int n_documents = 0; - if (!documents) { - ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing"); - } else if (!cJSON_IsArray(documents)) { - ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings", - "type_error.array"); - } else { - n_documents = cJSON_GetArraySize(documents); - if (n_documents < 1) - ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item", - "value_error.list.min_items"); - else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS) - ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items", - "value_error.list.max_items"); - cJSON *document; - int i = 0; - cJSON_ArrayForEach(document, documents) { - char loc[72]; - snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++); - if (!cJSON_IsString(document)) - ve_add(detail, loc, "document must be a string", "type_error.string"); - else if (!document->valuestring || !document->valuestring[0]) - ve_add(detail, loc, "document must not be empty", "value_error.empty"); - } - } - - cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n"); - cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k"); - if (top_n && top_k) { - ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one", - "value_error.conflict"); - } - cJSON *top = top_n ? top_n : top_k; - out->top_n = n_documents; - if (top) { - if (!cjson_is_integer(top)) { - ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", - "top_n must be an integer", "type_error.integer"); - } else { - out->top_n = top->valueint; - if (out->top_n < 1 || out->top_n > n_documents) - ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", - "top_n must be between 1 and the number of documents", "value_error.range"); - } - } - - cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents"); - if (return_documents) { - if (!cJSON_IsBool(return_documents)) - ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean", - "type_error.bool"); - else - out->return_documents = cJSON_IsTrue(return_documents); - } - - if (cJSON_GetArraySize(detail) == 0 && unloaded) { - job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); - cJSON_Delete(detail); - return; - } - - if (cJSON_GetArraySize(detail) == 0 && out->model) { - out->n_documents = n_documents; - out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); - if (tokenize_late_text(out->model, j, query->valuestring, 1, &out->query) != 0) { - ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", - "value_error.tokenization"); - } else { - out->query_tokens = out->query.n_tokens; - if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", - "value_error.context_length"); - } - - cJSON *document; - int i = 0; - cJSON_ArrayForEach(document, documents) { - char loc[72]; - snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i); - if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) != - 0) { - ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); - } else { - int n = out->documents[i].n_tokens; - if (n > EMBED_API_MAX_ITEM_TOKENS) - ve_add(detail, loc, "document exceeds 32768 token limit", - "value_error.context_length"); - if (out->document_tokens <= INT_MAX - n) - out->document_tokens += n; - else - out->document_tokens = INT_MAX; - } - i++; - } - if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) { - ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", - "value_error.context_length"); - } - } - - if (cJSON_GetArraySize(detail) > 0) { - job_set_422(j, detail); - rerank_request_free(out); - out->j = j; - } else { - out->ready = 1; - } - cJSON_Delete(detail); -} - -typedef struct { - int index; - float score; -} rerank_result; - -static int rerank_result_cmp(const void *a, const void *b) { - const rerank_result *ra = a; - const rerank_result *rb = b; - if (ra->score > rb->score) - return -1; - if (ra->score < rb->score) - return 1; - return ra->index - rb->index; -} - -static int execute_rerank_cpu(rerank_request *r, float *scores) { - loaded_model *m = r->model; - int dim = m->info->token_dim; - int total_doc_vecs = 0; - for (int i = 0; i < r->n_documents; i++) - total_doc_vecs += r->documents[i].n_keep; - - float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); - float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); - int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - - /* Encode every candidate in one block-diagonal forward. The encoder packs - * each document's kept token vectors back-to-back and fills offsets, so the - * cost of N short documents collapses to a single packed forward. */ - const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); - const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); - int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); - int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); - for (int i = 0; i < r->n_documents; i++) { - doc_ids[i] = r->documents[i].ids; - keep[i] = r->documents[i].keep; - n_tokens[i] = r->documents[i].n_tokens; - n_keep[i] = r->documents[i].n_keep; - } - - int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, - r->query.n_tokens, 1, query); - if (rc == 0) - rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens, - keep, n_keep, r->n_documents, 1, docs, offsets); - if (rc == 0) - rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, - scores); - free(n_keep); - free(n_tokens); - free(keep); - free(doc_ids); - free(offsets); - free(docs); - free(query); - return rc; -} - -#ifdef USE_MLX -static int execute_rerank_mlx(rerank_request *r, float *scores) { - loaded_model *m = r->model; - embed_mlx_late_vectors_t *query = - embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); - - /* Encode every candidate in one padded transformer pass. The encoder packs - * each document's kept token vectors back-to-back and fills offsets, so the - * cost of N short documents collapses to a single batched forward. */ - const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); - const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); - int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); - int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); - int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - for (int i = 0; i < r->n_documents; i++) { - doc_ids[i] = r->documents[i].ids; - keep[i] = r->documents[i].keep; - n_tokens[i] = r->documents[i].n_tokens; - n_keep[i] = r->documents[i].n_keep; - } - embed_mlx_late_vectors_t *packed = - query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep, - r->n_documents, 1, offsets) - : NULL; - int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, - r->n_documents, scores) - : -1; - embed_mlx_late_vectors_free(packed); - embed_mlx_late_vectors_free(query); - free(offsets); - free(n_keep); - free(n_tokens); - free(keep); - free(doc_ids); - return rc; -} -#endif - -#ifdef USE_CUDA -static int execute_rerank_cuda(rerank_request *r, float *scores) { - loaded_model *m = r->model; - int dim = m->info->token_dim; - int total_doc_vecs = 0; - for (int i = 0; i < r->n_documents; i++) - total_doc_vecs += r->documents[i].n_keep; - - float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); - float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); - int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - - /* GPU encodes the query and every candidate (one packed forward); MaxSim - * runs on the host, where the grouped-GEMM scorer beats a device graph. */ - const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); - const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); - int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); - int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); - for (int i = 0; i < r->n_documents; i++) { - doc_ids[i] = r->documents[i].ids; - keep[i] = r->documents[i].keep; - n_tokens[i] = r->documents[i].n_tokens; - n_keep[i] = r->documents[i].n_keep; - } - - int rc = - embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query); - if (rc == 0) - rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep, - r->n_documents, 1, docs, offsets); - if (rc == 0) - rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, - scores); - free(n_keep); - free(n_tokens); - free(keep); - free(doc_ids); - free(offsets); - free(docs); - free(query); - return rc; -} -#endif - -static void execute_rerank_request(rerank_request *r) { - if (!r || !r->model) { - if (r && r->j) - job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); - return; - } - float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); - uint64_t t0 = nstime(); -#if defined(USE_MLX) - int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores); -#elif defined(USE_CUDA) - int rc = - r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores); -#else - int rc = execute_rerank_cpu(r, scores); -#endif - r->j->infer_ns += nstime() - t0; - if (rc != 0) { - free(scores); - job_set_error(r->j, 500, "late-interaction reranking failed", "server_error"); - return; - } - - rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked)); - for (int i = 0; i < r->n_documents; i++) { - ranked[i].index = i; - ranked[i].score = scores[i]; - } - qsort(ranked, (size_t)r->n_documents, sizeof(*ranked), rerank_result_cmp); - - t0 = nstime(); - sbuf b = {0}; - sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id); - cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents"); - for (int i = 0; i < r->top_n; i++) { - if (i) - sbuf_putc(&b, ','); - sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index, - (double)ranked[i].score); - if (r->return_documents) { - cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index); - sbuf_puts(&b, ",\"document\":"); - append_json_string(&b, document->valuestring); - } - sbuf_putc(&b, '}'); - } - sbuf_printf(&b, - "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," - "\"total_tokens\":%d}}", - r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens); - set_response_from_buf(r->j, &b); - r->j->encode_ns += nstime() - t0; - free(ranked); - free(scores); -} - static int parse_job_root(job *j, cJSON **out_root) { cJSON *detail = cJSON_CreateArray(); cJSON *root = parse_json_body(j, detail); diff --git a/src/server_handlers.c b/src/server_handlers.c new file mode 100644 index 0000000..95df1de --- /dev/null +++ b/src/server_handlers.c @@ -0,0 +1,934 @@ +#include "server_internal.h" +#include "server_util.h" +#include "embed.h" + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* ======================================================================== + * Endpoint handlers: prepare, batch, execute, render + * + * One prepare_* per endpoint validates the request JSON and tokenizes its + * inputs into a *_request; the worker (server_schedule.c) groups compatible + * ready requests and calls the matching execute_*_request_list, which runs the + * batched forward pass through model_embed_* and renders each response via + * server_encode.c. Rerank runs MaxSim over late-interaction token vectors. + * ======================================================================== */ + +int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && + !strcmp(a->encoding, b->encoding); +} + +typedef struct { + embed_input_t input; + int output_index; +} embedding_batch_item; + +static int embedding_batch_item_cmp(const void *a, const void *b) { + const embedding_batch_item *ia = a; + const embedding_batch_item *ib = b; + if (ia->input.n_tokens < ib->input.n_tokens) + return -1; + if (ia->input.n_tokens > ib->input.n_tokens) + return 1; + return ia->output_index - ib->output_index; +} + +int embedding_request_fits_group(const embedding_request *r, + int group_inputs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_inputs == 0) + return 1; + return r->n_inputs <= max_batch - group_inputs && + r->total_tokens <= max_batch_tokens - group_tokens; +} + +void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { + if (n_reqs <= 0) + return; + loaded_model *m = reqs[0]->model; + int dim = m->info->dim; + int total_inputs = 0; + for (int i = 0; i < n_reqs; i++) + total_inputs += reqs[i]->n_inputs; + + embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items)); + float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float)); + + int pos = 0; + for (int i = 0; i < n_reqs; i++) { + for (int k = 0; k < reqs[i]->n_inputs; k++) { + items[pos].input = reqs[i]->inputs[k]; + items[pos].output_index = pos; + pos++; + } + } + qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp); + + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs; + if (max_batch > total_inputs) + max_batch = total_inputs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); + + int failed = 0; + uint64_t infer_ns = 0; + for (int start = 0; start < total_inputs;) { + int cur = 0; + int tokens = 0; + while (start + cur < total_inputs && cur < max_batch) { + int n_tokens = items[start + cur].input.n_tokens; + if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens)) + break; + inputs[cur] = items[start + cur].input; + tokens += n_tokens; + cur++; + } + uint64_t t0 = nstime(); + int embed_rc = model_embed_batch(m, inputs, cur, batch_embs); + infer_ns += nstime() - t0; + if (embed_rc != 0) { + failed = 1; + break; + } + for (int i = 0; i < cur; i++) + memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim, + (size_t)dim * sizeof(float)); + start += cur; + } + + pos = 0; + for (int i = 0; i < n_reqs; i++) { + reqs[i]->j->infer_ns += infer_ns; + if (failed) { + job_set_error(reqs[i]->j, 500, "embedding failed", "server_error"); + } else { + uint64_t t0 = nstime(); + if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0) + job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error"); + reqs[i]->j->encode_ns += nstime() - t0; + } + pos += reqs[i]->n_inputs; + } + + free(batch_embs); + free(inputs); + free(embs); + free(items); +} + +int contextual_request_compatible(const contextual_request *a, const contextual_request *b) { + return a->ready && b->ready && a->model == b->model && a->dims == b->dims && + !strcmp(a->encoding, b->encoding); +} + +int contextual_request_fits_group(const contextual_request *r, + int group_docs, + int group_tokens, + int max_batch, + int max_batch_tokens) { + if (group_docs == 0) + return 1; + return r->n_docs <= max_batch - group_docs && + r->total_tokens <= max_batch_tokens - group_tokens; +} + +typedef struct { + embed_context_input_t input; + int output_span_index; + int order; +} contextual_batch_item; + +static int contextual_batch_item_cmp(const void *a, const void *b) { + const contextual_batch_item *ia = a; + const contextual_batch_item *ib = b; + if (ia->input.input.n_tokens < ib->input.input.n_tokens) + return -1; + if (ia->input.input.n_tokens > ib->input.input.n_tokens) + return 1; + return ia->order - ib->order; +} + +void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { + if (n_reqs <= 0) + return; + loaded_model *m = reqs[0]->model; + int dim = m->info->dim; + int total_docs = 0; + int total_chunks = 0; + for (int i = 0; i < n_reqs; i++) { + if (total_docs > INT_MAX - reqs[i]->n_docs || + total_chunks > INT_MAX - reqs[i]->total_chunks) { + for (int k = 0; k < n_reqs; k++) + job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error"); + return; + } + total_docs += reqs[i]->n_docs; + total_chunks += reqs[i]->total_chunks; + } + if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) { + for (int i = 0; i < n_reqs; i++) + job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error"); + return; + } + + contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items)); + float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs)); + int pos = 0; + int span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + for (int d = 0; d < reqs[i]->n_docs; d++) { + contextual_doc *doc = &reqs[i]->docs[d]; + items[pos].input.input.ids = doc->ids; + items[pos].input.input.n_tokens = doc->n_tokens; + items[pos].input.spans = doc->spans; + items[pos].input.n_spans = doc->n_spans; + items[pos].output_span_index = span_offset; + items[pos].order = pos; + span_offset += doc->n_spans; + pos++; + } + } + qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp); + + int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs; + if (max_batch > total_docs) + max_batch = total_docs; + int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; + embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + float *batch_embs = xmalloc(sizeof(*batch_embs)); + size_t batch_emb_cap = 0; + + int failed = 0; + uint64_t infer_ns = 0; + for (int start = 0; start < total_docs;) { + int cur = 0; + int tokens = 0; + int chunks = 0; + while (start + cur < total_docs && cur < max_batch) { + const embed_context_input_t *input = &items[start + cur].input; + if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens, + max_batch_tokens)) + break; + inputs[cur] = *input; + tokens += input->input.n_tokens; + chunks += input->n_spans; + cur++; + } + size_t needed = (size_t)chunks * dim; + if (needed > batch_emb_cap) { + batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs)); + batch_emb_cap = needed; + } + uint64_t t0 = nstime(); + int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs); + infer_ns += nstime() - t0; + if (embed_rc != 0) { + failed = 1; + break; + } + int batch_span = 0; + for (int i = 0; i < cur; i++) { + int n_spans = inputs[i].n_spans; + memcpy(embs + (size_t)items[start + i].output_span_index * dim, + batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs)); + batch_span += n_spans; + } + start += cur; + } + + span_offset = 0; + for (int i = 0; i < n_reqs; i++) { + reqs[i]->j->infer_ns += infer_ns; + if (failed) { + job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error"); + } else { + uint64_t t0 = nstime(); + render_contextual_response(reqs[i], embs + (size_t)span_offset * dim); + reqs[i]->j->encode_ns += nstime() - t0; + } + span_offset += reqs[i]->total_chunks; + } + + free(batch_embs); + free(inputs); + free(embs); + free(items); +} + +static int validate_common(cJSON *root, + cJSON *detail, + model_kind expected_kind, + loaded_model **out_model, + int *out_dims, + const char **out_encoding, + http_server *s) { + cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); + const char *model_id = NULL; + if (!model_item) { + ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); + } else if (!cJSON_IsString(model_item) || !model_item->valuestring) { + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); + } else { + model_id = model_item->valuestring; + model_slot slot = model_slot_for_id(model_id); + if (slot == MODEL_UNKNOWN || k_models[slot].kind != expected_kind) { + ve_add(detail, "[\"body\",\"model\"]", + "value is not a valid enum member for this endpoint", "enum"); + } else { + loaded_model *m = &s->models[slot]; + if (!m->info) { + *out_model = NULL; + } else { + *out_model = m; + } + } + } + model_slot slot = model_id ? model_slot_for_id(model_id) : MODEL_UNKNOWN; + embedding_api_t api = slot != MODEL_UNKNOWN ? k_models[slot].api : EMBED_API_PERPLEXITY; + const char *enc = encoding_from_root(root, detail, api); + int min_dim = slot != MODEL_UNKNOWN ? k_models[slot].min_dim : 128; + int max_dim = slot != MODEL_UNKNOWN ? k_models[slot].dim : 2560; + int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc); + *out_encoding = enc; + *out_dims = dims; + return model_id && model_slot_for_id(model_id) != MODEL_UNKNOWN && + !s->models[model_slot_for_id(model_id)].info + ? 503 + : 0; +} + +void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + int unloaded = validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s); + out->model = m; + out->dims = dims; + out->encoding = encoding; + + /* Resolve request behavior from the model registry independent of load + * state. The query instruction, when requested, applies to every input. */ + cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model"); + model_slot api_slot = (model_item && cJSON_IsString(model_item) && model_item->valuestring) + ? model_slot_for_id(model_item->valuestring) + : MODEL_UNKNOWN; + const char *query_instruct = + api_slot != MODEL_UNKNOWN && + text_type_is_query(root, detail, k_models[api_slot].query_instruct) + ? k_models[api_slot].query_instruct + : NULL; + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_inputs = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (cJSON_IsString(input)) { + if (!input->valuestring || input->valuestring[0] == '\0') + ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty"); + n_inputs = 1; + } else if (cJSON_IsArray(input)) { + n_inputs = cJSON_GetArraySize(input); + if (n_inputs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", + "value_error.list.min_items"); + else if (n_inputs > EMBED_API_MAX_STANDARD_INPUTS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", + "value_error.list.max_items"); + cJSON *item; + int i = 0; + cJSON_ArrayForEach(item, input) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i); + if (!cJSON_IsString(item)) { + ve_add(detail, loc, "input item must be a string", "type_error.string"); + } else if (!item->valuestring || item->valuestring[0] == '\0') { + ve_add(detail, loc, "input item must not be empty", "value_error.empty"); + } + i++; + } + } else { + ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings", + "type_error"); + } + + if (cJSON_GetArraySize(detail) == 0 && unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); + cJSON_Delete(detail); + return; + } + + if (cJSON_GetArraySize(detail) == 0 && m && input) { + out->n_inputs = n_inputs; + out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens)); + out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs)); + + int idx = 0; + if (input && cJSON_IsString(input) && input->valuestring) { + if (tokenize_input(m, j, input->valuestring, query_instruct, &out->tokens[0]) != 0) { + ve_add(detail, "[\"body\",\"input\"]", "tokenization failed", + "value_error.tokenization"); + } else { + out->inputs[0].ids = out->tokens[0].ids; + out->inputs[0].n_tokens = out->tokens[0].n_tokens; + out->total_tokens = out->tokens[0].n_tokens; + if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", + "value_error.context_length"); + } + } else { + cJSON *item; + cJSON_ArrayForEach(item, input) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx); + if (!cJSON_IsString(item) || !item->valuestring) { + ve_add(detail, loc, "input item must be a string", "type_error.string"); + idx++; + continue; + } + if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != + 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + } else { + out->inputs[idx].ids = out->tokens[idx].ids; + out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; + out->total_tokens += out->tokens[idx].n_tokens; + if (out->tokens[idx].n_tokens > EMBED_API_MAX_ITEM_TOKENS) + ve_add(detail, loc, "input exceeds 32768 token limit", + "value_error.context_length"); + } + idx++; + } + } + + if (out->total_tokens > EMBED_API_MAX_TOTAL_TOKENS) + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + embedding_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + loaded_model *m = NULL; + int dims = 0; + const char *encoding = NULL; + int unloaded = validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s); + out->model = m; + out->dims = dims; + out->encoding = encoding; + + cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input"); + int n_docs = 0; + int total_chunks = 0; + if (!input) { + ve_add(detail, "[\"body\",\"input\"]", "field required", "missing"); + } else if (!cJSON_IsArray(input)) { + ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays", + "type_error.array"); + } else { + n_docs = cJSON_GetArraySize(input); + if (n_docs < 1) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", + "value_error.list.min_items"); + else if (n_docs > EMBED_API_MAX_CONTEXT_DOCS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", + "value_error.list.max_items"); + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + if (!cJSON_IsArray(doc_arr)) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must be an array of strings", "type_error.array"); + di++; + continue; + } + int doc_chunks = cJSON_GetArraySize(doc_arr); + if (doc_chunks > EMBED_API_MAX_CONTEXT_CHUNKS - total_chunks) + total_chunks = EMBED_API_MAX_CONTEXT_CHUNKS + 1; + else + total_chunks += doc_chunks; + if (doc_chunks < 1) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document must contain at least 1 chunk", + "value_error.list.min_items"); + } + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (!cJSON_IsString(chunk)) { + ve_add(detail, loc, "chunk must be a string", "type_error.string"); + } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') { + ve_add(detail, loc, "chunk must not be empty", "value_error.empty"); + } + ci++; + } + di++; + } + if (total_chunks > EMBED_API_MAX_CONTEXT_CHUNKS) + ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", + "value_error.list.max_items"); + } + + if (cJSON_GetArraySize(detail) == 0 && unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); + cJSON_Delete(detail); + return; + } + + if (cJSON_GetArraySize(detail) == 0 && m) { + out->n_docs = n_docs; + out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs)); + int64_t request_tokens = 0; + cJSON *doc_arr; + int di = 0; + cJSON_ArrayForEach(doc_arr, input) { + contextual_doc *doc = &out->docs[di]; + int n_chunks = cJSON_GetArraySize(doc_arr); + token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens)); + doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans)); + doc->n_spans = n_chunks; + int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0; + int doc_valid = 1; + + cJSON *chunk; + int ci = 0; + cJSON_ArrayForEach(chunk, doc_arr) { + char loc[80]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci); + if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + doc_valid = 0; + } else { + doc_tokens += chunk_tokens[ci].n_tokens; + } + ci++; + } + + if (doc_tokens > EMBED_API_MAX_ITEM_TOKENS) { + char loc[64]; + snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); + ve_add(detail, loc, "document exceeds 32768 token limit", + "value_error.context_length"); + doc_valid = 0; + } + request_tokens += doc_tokens; + + if (doc_valid) { + doc->n_tokens = (int)doc_tokens; + doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); + int pos = 0; + for (ci = 0; ci < n_chunks; ci++) { + doc->spans[ci].start = pos; + doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens; + /* n_tokens > 0 implies ids != NULL (tokenize_one sets both + * together), and the guard avoids a memcpy(dst, NULL, 0) for + * an empty chunk that the static analyzer reports as UB. */ + if (chunk_tokens[ci].n_tokens > 0) + memcpy(doc->ids + pos, chunk_tokens[ci].ids, + (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); + pos += chunk_tokens[ci].n_tokens; + if (ci + 1 < n_chunks) { + doc->ids[pos++] = m->context_separator_id; + } + } + } + free_token_bufs(chunk_tokens, n_chunks); + free(chunk_tokens); + di++; + } + if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) { + ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } else { + out->total_tokens = (int)request_tokens; + out->total_chunks = total_chunks; + } + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + contextual_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +static int +validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) { + cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model"); + if (!item) { + ve_add(detail, "[\"body\",\"model\"]", "field required", "missing"); + return 0; + } + if (!cJSON_IsString(item) || !item->valuestring) { + ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string"); + return 0; + } + model_slot slot = model_slot_for_id(item->valuestring); + if (slot == MODEL_UNKNOWN || k_models[slot].kind != MODEL_KIND_LATE) { + ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", + "enum"); + return 0; + } + *out_model = s->models[slot].info ? &s->models[slot] : NULL; + return *out_model ? 0 : 503; +} + +void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) { + memset(out, 0, sizeof(*out)); + out->j = j; + out->root = root; + + cJSON *detail = cJSON_CreateArray(); + int unloaded = validate_rerank_model(root, detail, s, &out->model); + cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query"); + cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents"); + /* query_str is set only once query passes every check; the tokenize step + * below runs only when detail is empty, so it is non-NULL there. Hoisting it + * also keeps the validated value explicit for the static analyzer. */ + const char *query_str = NULL; + if (!query) { + ve_add(detail, "[\"body\",\"query\"]", "field required", "missing"); + } else if (!cJSON_IsString(query)) { + ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string"); + } else if (!query->valuestring || !query->valuestring[0]) { + ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty"); + } else { + query_str = query->valuestring; + } + + int n_documents = 0; + if (!documents) { + ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing"); + } else if (!cJSON_IsArray(documents)) { + ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings", + "type_error.array"); + } else { + n_documents = cJSON_GetArraySize(documents); + if (n_documents < 1) + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item", + "value_error.list.min_items"); + else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS) + ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items", + "value_error.list.max_items"); + cJSON *document; + int i = 0; + cJSON_ArrayForEach(document, documents) { + char loc[72]; + snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++); + if (!cJSON_IsString(document)) + ve_add(detail, loc, "document must be a string", "type_error.string"); + else if (!document->valuestring || !document->valuestring[0]) + ve_add(detail, loc, "document must not be empty", "value_error.empty"); + } + } + + cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n"); + cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k"); + if (top_n && top_k) { + ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one", + "value_error.conflict"); + } + cJSON *top = top_n ? top_n : top_k; + out->top_n = n_documents; + if (top) { + if (!cjson_is_integer(top)) { + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", + "top_n must be an integer", "type_error.integer"); + } else { + out->top_n = top->valueint; + if (out->top_n < 1 || out->top_n > n_documents) + ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]", + "top_n must be between 1 and the number of documents", "value_error.range"); + } + } + + cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents"); + if (return_documents) { + if (!cJSON_IsBool(return_documents)) + ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean", + "type_error.bool"); + else + out->return_documents = cJSON_IsTrue(return_documents); + } + + if (cJSON_GetArraySize(detail) == 0 && unloaded) { + job_set_error(j, 503, "requested model is valid but not loaded", "model_not_loaded"); + cJSON_Delete(detail); + return; + } + + if (cJSON_GetArraySize(detail) == 0 && out->model) { + out->n_documents = n_documents; + out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); + if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) { + ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", + "value_error.tokenization"); + } else { + out->query_tokens = out->query.n_tokens; + if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS) + ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", + "value_error.context_length"); + } + + cJSON *document; + int i = 0; + cJSON_ArrayForEach(document, documents) { + char loc[72]; + snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i); + if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) != + 0) { + ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); + } else { + int n = out->documents[i].n_tokens; + if (n > EMBED_API_MAX_ITEM_TOKENS) + ve_add(detail, loc, "document exceeds 32768 token limit", + "value_error.context_length"); + if (out->document_tokens <= INT_MAX - n) + out->document_tokens += n; + else + out->document_tokens = INT_MAX; + } + i++; + } + if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) { + ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", + "value_error.context_length"); + } + } + + if (cJSON_GetArraySize(detail) > 0) { + job_set_422(j, detail); + rerank_request_free(out); + out->j = j; + } else { + out->ready = 1; + } + cJSON_Delete(detail); +} + +typedef struct { + int index; + float score; +} rerank_result; + +static int rerank_result_cmp(const void *a, const void *b) { + const rerank_result *ra = a; + const rerank_result *rb = b; + if (ra->score > rb->score) + return -1; + if (ra->score < rb->score) + return 1; + return ra->index - rb->index; +} + +static int execute_rerank_cpu(rerank_request *r, float *scores) { + loaded_model *m = r->model; + int dim = m->info->token_dim; + int total_doc_vecs = 0; + for (int i = 0; i < r->n_documents; i++) + total_doc_vecs += r->documents[i].n_keep; + + float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); + float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); + int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); + + /* Encode every candidate in one block-diagonal forward. The encoder packs + * each document's kept token vectors back-to-back and fills offsets, so the + * cost of N short documents collapses to a single packed forward. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; + } + + int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, + r->query.n_tokens, 1, query); + if (rc == 0) + rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens, + keep, n_keep, r->n_documents, 1, docs, offsets); + if (rc == 0) + rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, + scores); + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); + free(offsets); + free(docs); + free(query); + return rc; +} + +#ifdef USE_MLX +static int execute_rerank_mlx(rerank_request *r, float *scores) { + loaded_model *m = r->model; + embed_mlx_late_vectors_t *query = + embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); + + /* Encode every candidate in one padded transformer pass. The encoder packs + * each document's kept token vectors back-to-back and fills offsets, so the + * cost of N short documents collapses to a single batched forward. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); + int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; + } + embed_mlx_late_vectors_t *packed = + query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep, + r->n_documents, 1, offsets) + : NULL; + int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, + r->n_documents, scores) + : -1; + embed_mlx_late_vectors_free(packed); + embed_mlx_late_vectors_free(query); + free(offsets); + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); + return rc; +} +#endif + +#ifdef USE_CUDA +static int execute_rerank_cuda(rerank_request *r, float *scores) { + loaded_model *m = r->model; + int dim = m->info->token_dim; + int total_doc_vecs = 0; + for (int i = 0; i < r->n_documents; i++) + total_doc_vecs += r->documents[i].n_keep; + + float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); + float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); + int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); + + /* GPU encodes the query and every candidate (one packed forward); MaxSim + * runs on the host, where the grouped-GEMM scorer beats a device graph. */ + const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); + const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); + int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); + int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); + for (int i = 0; i < r->n_documents; i++) { + doc_ids[i] = r->documents[i].ids; + keep[i] = r->documents[i].keep; + n_tokens[i] = r->documents[i].n_tokens; + n_keep[i] = r->documents[i].n_keep; + } + + int rc = + embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query); + if (rc == 0) + rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep, + r->n_documents, 1, docs, offsets); + if (rc == 0) + rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, + scores); + free(n_keep); + free(n_tokens); + free(keep); + free(doc_ids); + free(offsets); + free(docs); + free(query); + return rc; +} +#endif + +void execute_rerank_request(rerank_request *r) { + if (!r || !r->model) { + if (r && r->j) + job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); + return; + } + float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); + uint64_t t0 = nstime(); +#if defined(USE_MLX) + int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores); +#elif defined(USE_CUDA) + int rc = + r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores); +#else + int rc = execute_rerank_cpu(r, scores); +#endif + r->j->infer_ns += nstime() - t0; + if (rc != 0) { + free(scores); + job_set_error(r->j, 500, "late-interaction reranking failed", "server_error"); + return; + } + + rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked)); + for (int i = 0; i < r->n_documents; i++) { + ranked[i].index = i; + ranked[i].score = scores[i]; + } + qsort(ranked, (size_t)r->n_documents, sizeof(*ranked), rerank_result_cmp); + + t0 = nstime(); + sbuf b = {0}; + sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id); + cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents"); + for (int i = 0; i < r->top_n; i++) { + if (i) + sbuf_putc(&b, ','); + sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index, + (double)ranked[i].score); + if (r->return_documents) { + cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index); + sbuf_puts(&b, ",\"document\":"); + append_json_string(&b, document->valuestring); + } + sbuf_putc(&b, '}'); + } + sbuf_printf(&b, + "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d," + "\"total_tokens\":%d}}", + r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens); + set_response_from_buf(r->j, &b); + r->j->encode_ns += nstime() - t0; + free(ranked); + free(scores); +} diff --git a/src/server_internal.h b/src/server_internal.h index 88bd5d4..31b6c11 100644 --- a/src/server_internal.h +++ b/src/server_internal.h @@ -361,4 +361,24 @@ int configure_loaded_model(loaded_model *m, const embed_config_t *config, int to int load_one_model(http_server *s, model_slot slot, const char *path); void free_models(http_server *s); +/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ +void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out); +int embedding_request_compatible(const embedding_request *a, const embedding_request *b); +int embedding_request_fits_group(const embedding_request *r, + int group_inputs, + int group_tokens, + int max_batch, + int max_batch_tokens); +void execute_embedding_request_list(embedding_request **reqs, int n_reqs); +void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out); +int contextual_request_compatible(const contextual_request *a, const contextual_request *b); +int contextual_request_fits_group(const contextual_request *r, + int group_docs, + int group_tokens, + int max_batch, + int max_batch_tokens); +void execute_contextual_request_list(contextual_request **reqs, int n_reqs); +void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out); +void execute_rerank_request(rerank_request *r); + #endif /* EMBED_SERVER_INTERNAL_H */ |
