aboutsummaryrefslogtreecommitdiff
path: root/tools/server/models.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865 /tools/server/models.c
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
Diffstat (limited to 'tools/server/models.c')
-rw-r--r--tools/server/models.c342
1 files changed, 342 insertions, 0 deletions
diff --git a/tools/server/models.c b/tools/server/models.c
new file mode 100644
index 0000000..8fdc590
--- /dev/null
+++ b/tools/server/models.c
@@ -0,0 +1,342 @@
+#include "server_internal.h"
+#include "util.h"
+
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* ========================================================================
+ * Model load/lifecycle, and tokenize/inference dispatch
+ *
+ * Each loaded model owns resolved serving metadata for the operator-chosen
+ * label. Transformer behavior comes from the model files through ffwd_config;
+ * server-only facts absent from those files, such as output API family and
+ * Matryoshka floor, come from the load spec. The tokenize_* and model_ffwd_*
+ * helpers are the single dispatch point the request handlers call, hiding the
+ * compiled backend choice behind one signature.
+ * ======================================================================== */
+
+loaded_model *loaded_model_for_label(http_server *s, const char *label) {
+ if (!label)
+ return NULL;
+ for (int i = 0; i < s->n_models; i++) {
+ if (s->models[i].info && !strcmp(label, s->models[i].info->id))
+ return &s->models[i];
+ }
+ return NULL;
+}
+
+static model_kind model_kind_from_public(ffwd_server_model_kind_t kind) {
+ switch (kind) {
+ case FFWD_SERVER_MODEL_CONTEXTUAL:
+ return MODEL_KIND_CONTEXTUAL;
+ case FFWD_SERVER_MODEL_LATE:
+ return MODEL_KIND_LATE;
+ case FFWD_SERVER_MODEL_STANDARD:
+ default:
+ return MODEL_KIND_STANDARD;
+ }
+}
+
+static embedding_api_t embedding_api_from_public(ffwd_server_embedding_api_t api) {
+ return api == FFWD_SERVER_API_PERPLEXITY ? FFWD_API_PERPLEXITY : FFWD_API_OPENAI;
+}
+
+static int read_optional_text_file(const char *model_dir, const char *relative, char **out) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ server_log("ffwd-server: model path too long");
+ return -1;
+ }
+ FILE *f = fopen(path, "rb");
+ if (!f && errno == ENOENT)
+ return 0;
+ if (!f) {
+ server_log("ffwd-server: failed to open %s", path);
+ return -1;
+ }
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf) {
+ fclose(f);
+ return -1;
+ }
+ if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ free(buf);
+ fclose(f);
+ return -1;
+ }
+ fclose(f);
+ buf[sz] = '\0';
+ *out = buf;
+ return 1;
+}
+
+static int read_query_instruct_from_files(const char *model_dir, char **out) {
+ char *buf = NULL;
+ int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf);
+ if (rc <= 0)
+ return rc;
+ cJSON *root = cJSON_Parse(buf);
+ free(buf);
+ if (!root) {
+ server_log("ffwd-server: invalid config_sentence_transformers.json: %s", model_dir);
+ return -1;
+ }
+ cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts");
+ cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL;
+ if (query && !cJSON_IsString(query)) {
+ cJSON_Delete(root);
+ server_log("ffwd-server: prompts.query must be a string: %s", model_dir);
+ return -1;
+ }
+ if (query && query->valuestring && query->valuestring[0]) {
+ *out = xstrdup(query->valuestring);
+ cJSON_Delete(root);
+ return *out ? 1 : -1;
+ }
+ cJSON_Delete(root);
+ return 0;
+}
+
+static int init_model_info(model_info **out, const ffwd_server_model_spec_t *spec) {
+ model_info *info = xcalloc(1, sizeof(*info));
+ info->id = xstrdup(spec->id);
+ info->kind = model_kind_from_public(spec->kind);
+ info->api = embedding_api_from_public(spec->api);
+ info->min_dim = spec->min_dim;
+ if (spec->query_instruct) {
+ info->query_instruct = xstrdup(spec->query_instruct);
+ } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) {
+ free(info->id);
+ free(info);
+ return -1;
+ }
+ if (!info->id || (spec->query_instruct && !info->query_instruct)) {
+ free(info->query_instruct);
+ free(info->id);
+ free(info);
+ return -1;
+ }
+ *out = info;
+ return 0;
+}
+
+static void free_model_info(model_info *info) {
+ if (!info)
+ return;
+ free(info->id);
+ free(info->query_instruct);
+ free(info);
+}
+
+void free_token_bufs(token_buf *t, int n) {
+ if (!t)
+ return;
+ for (int i = 0; i < n; i++)
+ free(t[i].ids);
+}
+
+void embedding_request_free(embedding_request *r) {
+ if (!r)
+ return;
+ free_token_bufs(r->tokens, r->n_inputs);
+ free(r->tokens);
+ free(r->inputs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void contextual_request_free(contextual_request *r) {
+ if (!r)
+ return;
+ for (int i = 0; i < r->n_docs; i++) {
+ free(r->docs[i].ids);
+ free(r->docs[i].spans);
+ }
+ free(r->docs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void late_tokens_free(late_tokens *t) {
+ if (!t)
+ return;
+ free(t->ids);
+ free(t->keep);
+ memset(t, 0, sizeof(*t));
+}
+
+void rerank_request_free(rerank_request *r) {
+ if (!r)
+ return;
+ late_tokens_free(&r->query);
+ for (int i = 0; i < r->n_documents; i++)
+ late_tokens_free(&r->documents[i]);
+ free(r->documents);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+/* Tokenization itself lives in libffwd (ffwd_tok). These thin wrappers only
+ * add the server's per-stage timing onto the job; the request handlers call
+ * them unchanged. */
+int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
+ uint64_t t0 = nstime();
+ out->ids = ffwd_tokenize(m->tok, text, query_instruct, &out->n_tokens);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ if (!out->ids) {
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ return 0;
+}
+
+int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
+ return tokenize_input(m, j, text, NULL, out);
+}
+
+int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
+ uint64_t t0 = nstime();
+ int rc = ffwd_tokenize_late(m->tok, text, is_query, out);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ return rc;
+}
+
+int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out) {
+ return ffwd_encode_batch(m->backend, inputs, batch, out);
+}
+
+int model_ffwd_spans_batch(loaded_model *m,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ return ffwd_encode_spans_batch(m->backend, inputs, batch, out);
+}
+
+static int model_uses_dense_batches(const loaded_model *m) {
+ return ffwd_uses_dense_batches(m->backend);
+}
+
+/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
+ * pads each dense row to the longest input in the chunk. */
+int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
+ if (batch == 0)
+ return 1;
+ if (model_uses_dense_batches(m))
+ return next_tokens <= max_batch_tokens / (batch + 1);
+ return next_tokens <= max_batch_tokens - packed_tokens;
+}
+
+int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim) {
+ if (!m || !m->info || !config || config->hidden_size <= 0) {
+ if (m && m->info)
+ server_log("ffwd-server: model %s has invalid config", m->info->id);
+ return -1;
+ }
+ if (m->info->kind == MODEL_KIND_LATE) {
+ if (token_dim <= 0) {
+ server_log("ffwd-server: late model %s is missing token projection metadata",
+ m->info->id);
+ return -1;
+ }
+ } else if (token_dim != 0) {
+ server_log("ffwd-server: non-late model %s unexpectedly has token projection metadata",
+ m->info->id);
+ return -1;
+ }
+
+ int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size;
+ if (min_dim <= 0 || min_dim > config->hidden_size) {
+ server_log("ffwd-server: model %s has invalid min_dim=%d for dim=%d", m->info->id, min_dim,
+ config->hidden_size);
+ return -1;
+ }
+ m->info->dim = config->hidden_size;
+ m->info->min_dim = min_dim;
+ m->info->token_dim = token_dim;
+ m->info->attention_mode = config->attention_mode;
+ m->info->pooling_mode = config->pooling_mode;
+ m->info->normalize_embeddings = config->normalize_embeddings;
+ m->renormalize_truncated =
+ config->normalize_embeddings && config->pooling_mode == FFWD_POOL_LAST_TOKEN;
+ return 0;
+}
+
+/* Apply the loaded model's config to its serving metadata and validate it. CPU
+ * runs this in load_one_model; a GPU build runs it on the inference thread once
+ * ffwd_activate has created the context. */
+int finalize_loaded_model(loaded_model *m) {
+ const ffwd_config_t *config = ffwd_config(m->backend);
+ int token_dim = ffwd_token_dim(m->backend);
+ ffwd_tok_set_append_terminal(m->tok, config->append_terminal_token);
+ return configure_loaded_model(m, config, token_dim);
+}
+
+int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec) {
+ /* Grow the model list by one entry. */
+ loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models));
+ if (!nm)
+ return -1;
+ s->models = nm;
+ loaded_model *m = &s->models[s->n_models];
+ memset(m, 0, sizeof(*m));
+ s->n_models++;
+
+ if (init_model_info(&m->info, spec) != 0)
+ return -1;
+ const char *path = spec->path;
+ m->path = xstrdup(path);
+ char err[256];
+ /* The tokenizer (file probe, special-token resolution, [CLS]/[SEP] vs Qwen
+ * terminal-token layout) is owned by libffwd now. */
+ m->tok = ffwd_tok_open(path, m->info->kind == MODEL_KIND_LATE, err, sizeof(err));
+ if (!m->tok) {
+ server_log("ffwd-server: %s: %s", path, err);
+ return -1;
+ }
+
+ m->backend =
+ ffwd_open(path, m->info->kind == MODEL_KIND_LATE, &s->backend_opts, err, sizeof(err));
+ if (!m->backend) {
+ server_log("ffwd-server: %s: %s", path, err);
+ return -1;
+ }
+ /* CPU loads the model in open, so its config is ready to finalize now. A GPU
+ * build defers the load - and this finalize - to the inference thread. */
+ if (ffwd_config(m->backend))
+ return finalize_loaded_model(m);
+ return 0;
+}
+
+void free_models(http_server *s) {
+ for (int i = 0; i < s->n_models; i++) {
+ loaded_model *m = &s->models[i];
+ free_model_info(m->info);
+ free(m->path);
+ ffwd_tok_free(m->tok);
+ ffwd_free(m->backend);
+ }
+ free(s->models);
+ s->models = NULL;
+ s->n_models = 0;
+}