diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 17:16:14 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 17:16:14 +0200 |
| commit | 3904a33e4416c35a48e8dd7c242a137a2cc17871 (patch) | |
| tree | 677c7d0a8d272fdc4a089d3bbd2004712938f0ee /src/late.c | |
| parent | cca81e59a9afeebe1237f713e919b5002df3d749 (diff) | |
| download | ffwd-3904a33e4416c35a48e8dd7c242a137a2cc17871.tar.gz ffwd-3904a33e4416c35a48e8dd7c242a137a2cc17871.zip | |
cpu: extract late-interaction model into late.c
Diffstat (limited to 'src/late.c')
| -rw-r--r-- | src/late.c | 377 |
1 files changed, 377 insertions, 0 deletions
diff --git a/src/late.c b/src/late.c new file mode 100644 index 0000000..c3d9d3f --- /dev/null +++ b/src/late.c @@ -0,0 +1,377 @@ +#include "embed.h" +#include "engine.h" +#include "kernels.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <limits.h> +#include <float.h> + +/* ======================================================================== + * Late-interaction model + * + * A late-interaction model is a pooled base model plus a 1_Dense linear + * projection from hidden states to token vectors. Instead of pooling to one + * vector, it keeps one projected vector per kept token and scores query/doc + * pairs with MaxSim. It reuses the base model's packed forward pass and + * workspace; only the projection, the token-vector buffer, and the scoring + * live here. + * ======================================================================== */ + +struct embed_late_model { + embed_model_t *base; + safetensors_file_t *dense_sf; + embed_weight_ref_t projection; /* [token_dim, hidden] */ + int token_dim; +}; + +struct embed_late_workspace { + const embed_late_model_t *model; + embed_workspace_t *base_ws; + float *states; + int states_seq_cap; +}; + +static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) { + if (!ws || !ws->model || n_tokens <= 0) + return -1; + if (n_tokens <= ws->states_seq_cap) + return 0; + int hidden = ws->model->base->config.hidden_size; + if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0) + return -1; + ws->states_seq_cap = n_tokens; + return 0; +} + +static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) { + if (!sf || !name) + return NULL; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) + return &sf->tensors[i]; + } + return NULL; +} + +embed_late_model_t *embed_late_model_load(const char *model_dir) { + if (!model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n"); + return NULL; + } + + embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late)); + if (!late) + return NULL; + + late->base = model_load_range_ex(model_dir, 0, -1, 1); + if (!late->base) + goto fail; + + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "embed_late_model_load: model path too long\n"); + goto fail; + } + + late->dense_sf = safetensors_open(path); + if (!late->dense_sf) { + fprintf(stderr, "embed_late_model_load: cannot open %s\n", path); + goto fail; + } + + const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight"); + if (!t) { + fprintf(stderr, "embed_late_model_load: missing linear.weight\n"); + goto fail; + } + if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) { + fprintf(stderr, "embed_late_model_load: bad projection shape\n"); + goto fail; + } + + int64_t shape[2] = {t->shape[0], late->base->config.hidden_size}; + if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1)) + goto fail; + + late->projection.dtype = t->dtype; + late->projection.data = safetensors_data(late->dense_sf, t); + late->token_dim = (int)t->shape[0]; + return late; + +fail: + embed_late_model_free(late); + return NULL; +} + +void embed_late_model_free(embed_late_model_t *model) { + if (!model) + return; + embed_model_free(model->base); + safetensors_close(model->dense_sf); + free(model); +} + +embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) { + if (!model || !model->base) + return NULL; + embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws)); + if (!ws) + return NULL; + ws->model = model; + ws->base_ws = embed_workspace_new(model->base); + if (!ws->base_ws) { + free(ws); + return NULL; + } + return ws; +} + +void embed_late_workspace_free(embed_late_workspace_t *ws) { + if (!ws) + return; + embed_workspace_free(ws->base_ws); + free(ws->states); + free(ws); +} + +const embed_config_t *embed_late_model_config(const embed_late_model_t *model) { + return model && model->base ? &model->base->config : NULL; +} + +int embed_late_model_token_dim(const embed_late_model_t *model) { + return model ? model->token_dim : 0; +} + +/* Internal (embed_internal.h): expose the base model and projection so GPU + * backends can upload them to device without seeing struct embed_late_model. */ +embed_model_t *embed_late_model_base(const embed_late_model_t *model) { + return model ? model->base : NULL; +} + +const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) { + return model ? &model->projection : NULL; +} + +int embed_late_model_encode_tokens(const embed_late_model_t *model, + embed_late_workspace_t *ws, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { + if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors || + model->token_dim <= 0) + return -1; + + int hidden = model->base->config.hidden_size; + int dim = model->token_dim; + if (ensure_late_state_buffer(ws, n_tokens) != 0) + return -1; + if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0) + return -1; + + linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden, + dim); + if (normalize) { + for (int i = 0; i < n_tokens; i++) { + if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + } + return 0; +} + +float embed_late_maxsim(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + int doc_tokens, + int dim) { + if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0) + return 0.0f; + + float score = 0.0f; + for (int qi = 0; qi < query_tokens; qi++) { + const float *q = query_vectors + (size_t)qi * dim; + float best = -FLT_MAX; + for (int di = 0; di < doc_tokens; di++) { + const float *d = doc_vectors + (size_t)di * dim; + float dot = embed_dot_f32(q, d, dim); + if (dot > best) + best = dot; + } + score += best; + } + return score; +} + +int embed_late_maxsim_batch(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores) { + if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || + docs <= 0 || dim <= 0 || doc_offsets[0] != 0) + return -1; + + for (int i = 0; i < docs; i++) { + if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i]) + return -1; + } + +#ifdef USE_BLAS + /* One similarity GEMM per group of whole candidates: + * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document + * row-max plus sum. The group token budget bounds the scratch matrix; + * one oversized document still runs alone in a group of its own. */ + enum { LATE_GROUP_TOKEN_BUDGET = 4096 }; + int total = doc_offsets[docs]; + int budget = LATE_GROUP_TOKEN_BUDGET; + size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget); + int largest = 0; + for (int i = 0; i < docs; i++) { + int len = doc_offsets[i + 1] - doc_offsets[i]; + if (len > largest) + largest = len; + } + if ((size_t)query_tokens * (size_t)largest > cap) + cap = (size_t)query_tokens * (size_t)largest; + float *sim = (float *)malloc(cap * sizeof(float)); + if (sim) { + int doc = 0; + while (doc < docs) { + int first = doc; + int start = doc_offsets[first]; + doc++; + while (doc < docs && doc_offsets[doc + 1] - start <= budget && + doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens)) + doc++; + int group_tokens = doc_offsets[doc] - start; + + embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, + group_tokens); + + for (int i = first; i < doc; i++) { + int s0 = doc_offsets[i] - start; + int s1 = doc_offsets[i + 1] - start; + float score = 0.0f; + for (int qi = 0; qi < query_tokens; qi++) { + const float *row = sim + (size_t)qi * group_tokens; + float best = row[s0]; + for (int t = s0 + 1; t < s1; t++) + if (row[t] > best) + best = row[t]; + score += best; + } + scores[i] = score; + } + } + free(sim); + return 0; + } + /* Allocation failure: fall through to the scalar path. */ +#endif + + for (int i = 0; i < docs; i++) { + int start = doc_offsets[i]; + int end = doc_offsets[i + 1]; + scores[i] = embed_late_maxsim(query_vectors, query_tokens, + doc_vectors + (size_t)start * dim, end - start, dim); + } + return 0; +} + +/* Late-interaction batch encoder. Lives here, beside the per-document encoder, + * and reuses the base model's packed/ragged batch forward pass. */ +int embed_late_model_encode_docs(const embed_late_model_t *model, + embed_late_workspace_t *ws, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets) { + if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep || + n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0) + return -1; + + int hidden = model->base->config.hidden_size; + int dim = model->token_dim; + + /* Pack all candidates for one block-diagonal forward. inputs/offsets index + * the full token stream; out_offsets is the prefix sum of kept tokens. */ + embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); + int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets)); + if (!inputs || !offsets) { + free(inputs); + free(offsets); + return -1; + } + + int total_keep = 0; + out_offsets[0] = 0; + for (int d = 0; d < n_docs; d++) { + if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 || + n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) { + free(inputs); + free(offsets); + return -1; + } + for (int k = 0; k < n_keep[d]; k++) { + if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) { + free(inputs); + free(offsets); + return -1; + } + } + inputs[d].ids = doc_ids[d]; + inputs[d].n_tokens = n_tokens[d]; + total_keep += n_keep[d]; + out_offsets[d + 1] = total_keep; + } + + int total_seq = 0, max_seq = 0; + int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq); + if (rc == 0) + rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq, + max_seq, 1); + free(inputs); + if (rc != 0) { + free(offsets); + return -1; + } + + /* Gather each document's kept hidden rows into ws->states (its natural + * [seq, hidden] shape), then project the packed [total_keep, hidden] block + * to [total_keep, dim] in one GEMM, matching the per-document path. */ + if (ensure_late_state_buffer(ws, total_keep) != 0) { + free(offsets); + return -1; + } + const float *states = ws->base_ws->x; + int pos = 0; + for (int d = 0; d < n_docs; d++) { + int start = offsets[d]; + for (int k = 0; k < n_keep[d]; k++) { + memcpy(ws->states + (size_t)pos * hidden, + states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float)); + pos++; + } + } + free(offsets); + + linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep, + hidden, dim); + if (normalize) { + for (int i = 0; i < total_keep; i++) { + if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + } + return 0; +} |
