1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
/*
* Run the transformer on Metal using mlx-c.
*/
#ifndef FFWD_MLX_H
#define FFWD_MLX_H
/* The libraries are built with -fvisibility=hidden, so only declarations carrying
* FFWD_API are exported from the shared library; everything else is internal. */
#ifndef FFWD_API
# if defined(__GNUC__)
# define FFWD_API __attribute__((visibility("default")))
# else
# define FFWD_API
# endif
#endif
#include <stddef.h>
#include <stdint.h>
#include "ffwd.h"
/* Opaque MLX model context */
typedef struct ffwd_mlx_ctx ffwd_mlx_ctx_t;
typedef struct ffwd_mlx_late_ctx ffwd_mlx_late_ctx_t;
typedef struct ffwd_mlx_late_vectors ffwd_mlx_late_vectors_t;
typedef struct {
int quantize_bits; /* 0 disables; currently 8 */
int quantize_group_size; /* default 64 when quantize_bits is set */
} ffwd_mlx_options_t;
/*
* Load model into MLX arrays from safetensors + config.json.
* Returns NULL on error.
*/
FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir);
FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir,
const ffwd_mlx_options_t *opts);
FFWD_API void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx);
/*
* Late-interaction MLX path. This is separate from pooled embeddings because
* late snapshots produce one projected token vector per input token for MaxSim
* scoring, not one pooled document vector.
*/
FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load(const char *model_dir);
FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir,
const ffwd_mlx_options_t *opts);
FFWD_API void ffwd_mlx_late_free(ffwd_mlx_late_ctx_t *ctx);
FFWD_API const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx);
FFWD_API int ffwd_mlx_late_token_dim(const ffwd_mlx_late_ctx_t *ctx);
/*
* Encode token_ids into out_vectors[n_tokens, token_dim].
*
* If normalize is non-zero, every token vector is L2-normalized on device
* before copying back to CPU memory.
*/
FFWD_API int ffwd_mlx_late_encode_tokens(
ffwd_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, float *out_vectors);
/*
* Device-resident late-interaction token vectors.
*
* The model context owns the MLX stream; the returned vector handle must not
* outlive ctx. Copying is optional: MaxSim can consume these handles directly
* and copy back only final candidate scores.
*/
FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx,
const int *token_ids,
int n_tokens,
int normalize);
FFWD_API void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs);
FFWD_API int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs);
FFWD_API int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs);
FFWD_API int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors);
/* Encode n_docs documents in one padded transformer pass.
*
* Packs each document's kept token vectors back-to-back into [total_keep, token_dim]
* in document order.
*
* out_offsets[n_docs + 1] gets the prefix sum of kept counts, so the result can
* feed ffwd_mlx_late_maxsim_batch_device directly.
*
* Returns NULL on error. The handle must not outlive ctx. */
FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx,
const int *const *doc_ids,
const int *n_tokens,
const int *const *keep,
const int *n_keep,
int n_docs,
int normalize,
int *out_offsets);
/* Device-resident select/concat primitives.
* Kept for the late-interaction verification harness (tests/check_late_interaction.py).
* The server rerank path uses ffwd_mlx_late_encode_docs_device. */
FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(
ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count);
FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx,
const ffwd_mlx_late_vectors_t *vecs,
const int *token_indices,
int count);
FFWD_API int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx,
const ffwd_mlx_late_vectors_t *query,
const ffwd_mlx_late_vectors_t *docs,
const int *doc_offsets,
int docs_count,
float *scores);
/*
* Compute embedding for token_ids[0..n_tokens-1].
* Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/
FFWD_API float *ffwd_mlx_encode(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens);
/*
* Compute one embedding into caller-provided out[hidden_size].
* Returns 0 on success, -1 on error.
*/
FFWD_API int
ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding);
/*
* Compute a true padded dense batch.
*
* out_embeddings is caller-provided [batch, hidden_size].
* Pooling and attention masking follow the loaded model configuration.
* Returns 0 on success, -1 on error.
*/
FFWD_API int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx,
const ffwd_input_t *inputs,
int batch,
float *out_embeddings);
/*
* Run one contextual sequence and pool selected token spans.
* Returns 0 on success, -1 on error.
*/
FFWD_API int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx,
const int *token_ids,
int n_tokens,
const ffwd_span_t *spans,
int n_spans,
float *out_embeddings);
/*
* Run a padded dense contextual document batch and pool every selected span.
* out_embeddings contains spans in document order.
*/
FFWD_API int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx,
const ffwd_context_input_t *inputs,
int batch,
float *out_embeddings);
/* Get the config (so ffwd_cli.c can read hidden_size etc.) */
FFWD_API const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx);
#endif /* FFWD_MLX_H */
|