diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 /libffwd/late.c | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
Diffstat (limited to 'libffwd/late.c')
| -rw-r--r-- | libffwd/late.c | 218 |
1 files changed, 218 insertions, 0 deletions
diff --git a/libffwd/late.c b/libffwd/late.c new file mode 100644 index 0000000..05d0341 --- /dev/null +++ b/libffwd/late.c @@ -0,0 +1,218 @@ +#include "ffwd.h" +#include "model_internal.h" +#include "kernels.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <limits.h> +#include <float.h> + +/* ======================================================================== + * Late-interaction model + * + * A late-interaction model is a pooled base model plus a 1_Dense linear + * projection from hidden states to token vectors. Instead of pooling to one + * vector, it keeps one projected vector per kept token and scores query/doc + * pairs with MaxSim. It reuses the base model's packed forward pass and + * workspace; only the projection, the token-vector buffer, and the scoring + * live here. + * ======================================================================== */ + +struct ffwd_late_model { + ffwd_model_t *base; + safetensors_file_t *dense_sf; + ffwd_weight_ref_t projection; /* [token_dim, hidden] */ + int token_dim; +}; + +static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) { + if (!sf || !name) + return NULL; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) + return &sf->tensors[i]; + } + return NULL; +} + +ffwd_late_model_t *ffwd_late_model_load(const char *model_dir) { + if (!model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "ffwd_late_model_load: missing 1_Dense projection\n"); + return NULL; + } + + ffwd_late_model_t *late = (ffwd_late_model_t *)calloc(1, sizeof(*late)); + if (!late) + return NULL; + + late->base = model_load_range_ex(model_dir, 0, -1, 1); + if (!late->base) + goto fail; + + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "ffwd_late_model_load: model path too long\n"); + goto fail; + } + + late->dense_sf = safetensors_open(path); + if (!late->dense_sf) { + fprintf(stderr, "ffwd_late_model_load: cannot open %s\n", path); + goto fail; + } + + const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight"); + if (!t) { + fprintf(stderr, "ffwd_late_model_load: missing linear.weight\n"); + goto fail; + } + if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) { + fprintf(stderr, "ffwd_late_model_load: bad projection shape\n"); + goto fail; + } + + int64_t shape[2] = {t->shape[0], late->base->config.hidden_size}; + if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1)) + goto fail; + + /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */ + late->projection = weight_ref_from_tensor(late->dense_sf, t); + if (!late->projection.data) + goto fail; + late->token_dim = (int)t->shape[0]; + return late; + +fail: + ffwd_late_model_free(late); + return NULL; +} + +void ffwd_late_model_free(ffwd_late_model_t *model) { + if (!model) + return; + ffwd_model_free(model->base); + /* Free the projection before the mmap: only an F16-widened ref owns its + * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */ + if (model->projection.owned) + free((void *)model->projection.data); + safetensors_close(model->dense_sf); + free(model); +} + +const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model) { + return model && model->base ? &model->base->config : NULL; +} + +int ffwd_late_model_token_dim(const ffwd_late_model_t *model) { + return model ? model->token_dim : 0; +} + +/* Internal (internal.h): expose the base model and projection so GPU + * backends can upload them to device without seeing struct ffwd_late_model. */ +ffwd_model_t *ffwd_late_model_base(const ffwd_late_model_t *model) { + return model ? model->base : NULL; +} + +const ffwd_weight_ref_t *ffwd_late_model_projection(const ffwd_late_model_t *model) { + return model ? &model->projection : NULL; +} + +float ffwd_late_maxsim( + const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim) { + if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0) + return 0.0f; + + float score = 0.0f; + for (int qi = 0; qi < query_tokens; qi++) { + const float *q = query_vectors + (size_t)qi * dim; + float best = -FLT_MAX; + for (int di = 0; di < doc_tokens; di++) { + const float *d = doc_vectors + (size_t)di * dim; + float dot = dot_f32(q, d, dim); + if (dot > best) + best = dot; + } + score += best; + } + return score; +} + +int ffwd_late_maxsim_batch(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores) { + if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || docs <= 0 || + dim <= 0 || doc_offsets[0] != 0) + return -1; + + for (int i = 0; i < docs; i++) { + if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i]) + return -1; + } + +#ifdef USE_BLAS + /* One similarity GEMM per group of whole candidates: + * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document + * row-max plus sum. The group token budget bounds the scratch matrix; + * one oversized document still runs alone in a group of its own. */ + enum { LATE_GROUP_TOKEN_BUDGET = 4096 }; + int total = doc_offsets[docs]; + int budget = LATE_GROUP_TOKEN_BUDGET; + size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget); + int largest = 0; + for (int i = 0; i < docs; i++) { + int len = doc_offsets[i + 1] - doc_offsets[i]; + if (len > largest) + largest = len; + } + if ((size_t)query_tokens * (size_t)largest > cap) + cap = (size_t)query_tokens * (size_t)largest; + float *sim = (float *)malloc(cap * sizeof(float)); + if (sim) { + int doc = 0; + while (doc < docs) { + int first = doc; + int start = doc_offsets[first]; + doc++; + while (doc < docs && doc_offsets[doc + 1] - start <= budget && + doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens)) + doc++; + int group_tokens = doc_offsets[doc] - start; + + matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, + group_tokens); + + for (int i = first; i < doc; i++) { + int s0 = doc_offsets[i] - start; + int s1 = doc_offsets[i + 1] - start; + float score = 0.0f; + for (int qi = 0; qi < query_tokens; qi++) { + const float *row = sim + (size_t)qi * group_tokens; + float best = row[s0]; + for (int t = s0 + 1; t < s1; t++) + if (row[t] > best) + best = row[t]; + score += best; + } + scores[i] = score; + } + } + free(sim); + return 0; + } + /* Allocation failure: fall through to the scalar path. */ +#endif + + for (int i = 0; i < docs; i++) { + int start = doc_offsets[i]; + int end = doc_offsets[i + 1]; + scores[i] = ffwd_late_maxsim(query_vectors, query_tokens, + doc_vectors + (size_t)start * dim, end - start, dim); + } + return 0; +} |
