aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-25 17:58:46 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-25 17:58:46 +0200
commitf7ef995ff54da671f9d7665b3d92ff25934c1e8b (patch)
tree1401d04c0f464b32bffd63f787a63a82c6959e97
parent417f861f3661c2253258b5683a627d961e81fc68 (diff)
downloadffwd-f7ef995ff54da671f9d7665b3d92ff25934c1e8b.tar.gz
ffwd-f7ef995ff54da671f9d7665b3d92ff25934c1e8b.zip
chore: run clang-format
-rw-r--r--bench/bench_tokenizer.c28
-rw-r--r--libffwd/backend_blas.c23
-rw-r--r--libffwd/backend_cuda.c21
-rw-r--r--libffwd/backend_mlx.c18
-rw-r--r--libffwd/cuda.h41
-rw-r--r--libffwd/forward.c81
-rw-r--r--libffwd/internal.h114
-rw-r--r--libffwd/kernels.h134
-rw-r--r--libffwd/late.c20
-rw-r--r--libffwd/late_blas.c28
-rw-r--r--libffwd/mlx.c125
-rw-r--r--libffwd/mlx.h65
-rw-r--r--libffwd/model_types.h42
-rw-r--r--libffwd/tokenize.c9
-rw-r--r--libffwd/wordpiece.h16
-rw-r--r--tests/test_cls.c4
-rw-r--r--tests/test_late.c28
-rw-r--r--tests/test_qwen3.c5
-rw-r--r--tests/test_workspace.c4
-rw-r--r--tools/server/models.c6
20 files changed, 380 insertions, 432 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index 9bb8822..a5a16ee 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -39,9 +39,7 @@ typedef struct {
void *ws;
} bench_tok_t;
-static int path_exists(const char *path) {
- return access(path, R_OK) == 0;
-}
+static int path_exists(const char *path) { return access(path, R_OK) == 0; }
static int load_tokenizer(bench_tok_t *bt, const char *path) {
char p[1024];
@@ -88,28 +86,28 @@ static int *tok_encode(bench_tok_t *bt, const char *text, int *n) {
static int *tok_encode_ws(bench_tok_t *bt, const char *text, int *n) {
switch (bt->kind) {
case TOK_WORDPIECE:
- return tok_wp_encode_with_workspace((tok_wp_t *)bt->tok,
- (tok_wp_workspace_t *)bt->ws, text, n);
+ return tok_wp_encode_with_workspace((tok_wp_t *)bt->tok, (tok_wp_workspace_t *)bt->ws, text,
+ n);
case TOK_SENTENCEPIECE:
- return tok_spm_encode_with_workspace(
- (tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws, text, n);
+ return tok_spm_encode_with_workspace((tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws,
+ text, n);
default:
- return tok_bpe_encode_with_workspace((tok_bpe_t *)bt->tok,
- (tok_bpe_workspace_t *)bt->ws, text, n);
+ return tok_bpe_encode_with_workspace((tok_bpe_t *)bt->tok, (tok_bpe_workspace_t *)bt->ws,
+ text, n);
}
}
static int tok_encode_into(bench_tok_t *bt, const char *text, int *out, int cap, int *n) {
switch (bt->kind) {
case TOK_WORDPIECE:
- return tok_wp_encode_into((tok_wp_t *)bt->tok,
- (tok_wp_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_wp_encode_into((tok_wp_t *)bt->tok, (tok_wp_workspace_t *)bt->ws, text, out, cap,
+ n);
case TOK_SENTENCEPIECE:
- return tok_spm_encode_into((tok_spm_t *)bt->tok,
- (tok_spm_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_spm_encode_into((tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws, text, out,
+ cap, n);
default:
- return tok_bpe_encode_into((tok_bpe_t *)bt->tok,
- (tok_bpe_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_bpe_encode_into((tok_bpe_t *)bt->tok, (tok_bpe_workspace_t *)bt->ws, text, out,
+ cap, n);
}
}
diff --git a/libffwd/backend_blas.c b/libffwd/backend_blas.c
index edf54bb..9e3d376 100644
--- a/libffwd/backend_blas.c
+++ b/libffwd/backend_blas.c
@@ -38,8 +38,8 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
return 0;
}
-ffwd_t *ffwd_open(
- const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
+ffwd_t *
+ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
(void)opts;
ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b));
if (!b) {
@@ -102,9 +102,7 @@ const ffwd_config_t *ffwd_config(const ffwd_t *b) {
return b->is_late ? ffwd_late_model_config(b->late) : ffwd_model_config(b->model);
}
-int ffwd_token_dim(const ffwd_t *b) {
- return b->is_late ? ffwd_late_model_token_dim(b->late) : 0;
-}
+int ffwd_token_dim(const ffwd_t *b) { return b->is_late ? ffwd_late_model_token_dim(b->late) : 0; }
int ffwd_uses_dense_batches(const ffwd_t *b) {
(void)b;
@@ -115,10 +113,7 @@ int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *o
return ffwd_model_encode_batch(b->model, b->ws, inputs, batch, out);
}
-int ffwd_encode_spans_batch(ffwd_t *b,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out) {
+int ffwd_encode_spans_batch(ffwd_t *b, const ffwd_context_input_t *inputs, int batch, float *out) {
return ffwd_model_encode_spans_batch(b->model, b->ws, inputs, batch, out);
}
@@ -136,15 +131,13 @@ int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) {
/* Encode every candidate in one block-diagonal forward: the encoder packs
* each document's kept token vectors back-to-back and fills offsets. */
if (rc == 0)
- rc = ffwd_late_model_encode_tokens(b->late, b->late_ws, in->query_ids, in->query_n_tokens,
- 1, query);
+ rc = ffwd_late_model_encode_tokens(b->late, b->late_ws, in->query_ids, in->query_n_tokens, 1,
+ query);
if (rc == 0)
rc = ffwd_late_model_encode_docs(b->late, b->late_ws, in->doc_ids, in->doc_n_tokens,
- in->doc_keep, in->doc_n_keep, in->n_docs, 1, docs,
- offsets);
+ in->doc_keep, in->doc_n_keep, in->n_docs, 1, docs, offsets);
if (rc == 0)
- rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim,
- scores);
+ rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, scores);
free(offsets);
free(docs);
free(query);
diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c
index dd05c16..a925404 100644
--- a/libffwd/backend_cuda.c
+++ b/libffwd/backend_cuda.c
@@ -58,8 +58,8 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
return 0;
}
-ffwd_t *ffwd_open(
- const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
+ffwd_t *
+ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
(void)opts;
ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b));
if (!b || !(b->dir = strdup(model_dir))) {
@@ -108,9 +108,7 @@ const ffwd_config_t *ffwd_config(const ffwd_t *b) {
return b->is_late ? ffwd_cuda_late_config(b->late_ctx) : ffwd_cuda_config(b->ctx);
}
-int ffwd_token_dim(const ffwd_t *b) {
- return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0;
-}
+int ffwd_token_dim(const ffwd_t *b) { return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0; }
int ffwd_uses_dense_batches(const ffwd_t *b) {
(void)b;
@@ -121,10 +119,7 @@ int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *o
return ffwd_cuda_encode_batch(b->ctx, inputs, batch, out);
}
-int ffwd_encode_spans_batch(ffwd_t *b,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out) {
+int ffwd_encode_spans_batch(ffwd_t *b, const ffwd_context_input_t *inputs, int batch, float *out) {
return ffwd_cuda_encode_spans_batch(b->ctx, inputs, batch, out);
}
@@ -142,14 +137,12 @@ int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) {
/* GPU encodes the query and every candidate in one packed forward; MaxSim
* runs on the host, where the grouped-GEMM scorer beats a device graph. */
if (rc == 0)
- rc =
- ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query);
+ rc = ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query);
if (rc == 0)
rc = ffwd_cuda_late_encode_docs(b->late_ctx, in->doc_ids, in->doc_n_tokens, in->doc_keep,
- in->doc_n_keep, in->n_docs, 1, docs, offsets);
+ in->doc_n_keep, in->n_docs, 1, docs, offsets);
if (rc == 0)
- rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim,
- scores);
+ rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, scores);
free(offsets);
free(docs);
free(query);
diff --git a/libffwd/backend_mlx.c b/libffwd/backend_mlx.c
index 80b893c..932e550 100644
--- a/libffwd/backend_mlx.c
+++ b/libffwd/backend_mlx.c
@@ -62,8 +62,8 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
return 0;
}
-ffwd_t *ffwd_open(
- const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
+ffwd_t *
+ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b));
if (!b || !(b->dir = strdup(model_dir))) {
snprintf(err, errlen, "out of memory");
@@ -120,9 +120,7 @@ const ffwd_config_t *ffwd_config(const ffwd_t *b) {
return b->is_late ? ffwd_mlx_late_config(b->late_ctx) : ffwd_mlx_config(b->ctx);
}
-int ffwd_token_dim(const ffwd_t *b) {
- return b->is_late ? ffwd_mlx_late_token_dim(b->late_ctx) : 0;
-}
+int ffwd_token_dim(const ffwd_t *b) { return b->is_late ? ffwd_mlx_late_token_dim(b->late_ctx) : 0; }
int ffwd_uses_dense_batches(const ffwd_t *b) { return b->ctx != NULL; }
@@ -130,10 +128,7 @@ int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *o
return ffwd_mlx_encode_batch(b->ctx, inputs, batch, out);
}
-int ffwd_encode_spans_batch(ffwd_t *b,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out) {
+int ffwd_encode_spans_batch(ffwd_t *b, const ffwd_context_input_t *inputs, int batch, float *out) {
return ffwd_mlx_encode_spans_batch(b->ctx, inputs, batch, out);
}
@@ -144,11 +139,10 @@ int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) {
ffwd_mlx_late_vectors_t *packed =
(query && offsets)
? ffwd_mlx_late_encode_docs_device(b->late_ctx, in->doc_ids, in->doc_n_tokens,
- in->doc_keep, in->doc_n_keep, in->n_docs, 1,
- offsets)
+ in->doc_keep, in->doc_n_keep, in->n_docs, 1, offsets)
: NULL;
int rc = packed ? ffwd_mlx_late_maxsim_batch_device(b->late_ctx, query, packed, offsets,
- in->n_docs, scores)
+ in->n_docs, scores)
: -1;
ffwd_mlx_late_vectors_free(packed);
ffwd_mlx_late_vectors_free(query);
diff --git a/libffwd/cuda.h b/libffwd/cuda.h
index 7fc3515..851e412 100644
--- a/libffwd/cuda.h
+++ b/libffwd/cuda.h
@@ -40,11 +40,8 @@ FFWD_API const ffwd_config_t *ffwd_cuda_late_config(const ffwd_cuda_late_ctx_t *
FFWD_API int ffwd_cuda_late_token_dim(const ffwd_cuda_late_ctx_t *ctx);
/* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */
-FFWD_API int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors);
+FFWD_API int ffwd_cuda_late_encode_tokens(
+ ffwd_cuda_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, float *out_vectors);
/*
* Encode n_docs documents in one packed forward and pack each document's kept
@@ -53,29 +50,27 @@ FFWD_API int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx,
* (the layout ffwd_late_maxsim_batch consumes). Returns 0 on success.
*/
FFWD_API int ffwd_cuda_late_encode_docs(ffwd_cuda_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets);
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets);
FFWD_API const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx);
-FFWD_API int ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
+FFWD_API int
+ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding);
FFWD_API float *ffwd_cuda_encode(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens);
FFWD_API int ffwd_cuda_encode_batch(ffwd_cuda_ctx_t *ctx,
- const ffwd_input_t *inputs,
- int batch,
- float *out_embeddings);
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings);
FFWD_API int ffwd_cuda_encode_spans_batch(ffwd_cuda_ctx_t *ctx,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out_embeddings);
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings);
#endif /* FFWD_CUDA_H */
diff --git a/libffwd/forward.c b/libffwd/forward.c
index e8a5cdf..d8e45f9 100644
--- a/libffwd/forward.c
+++ b/libffwd/forward.c
@@ -225,8 +225,7 @@ static int forward_packed_slice_inplace(const ffwd_model_t *model,
for (int i = 0; i < inputs[b].n_tokens; i++) {
int id = inputs[b].ids[i];
if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b,
- i);
+ fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, i);
return -1;
}
copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id,
@@ -466,10 +465,10 @@ static int pool_embeddings(const ffwd_model_t *model,
* ======================================================================== */
int ffwd_model_forward_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_states) {
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_states) {
if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states)
return -1;
@@ -488,11 +487,11 @@ int ffwd_model_forward_into(const ffwd_model_t *model,
}
int ffwd_pool_spans(const ffwd_config_t *cfg,
- const float *states,
- int n_tokens,
- const ffwd_span_t *spans,
- int n_spans,
- float *out_embeddings) {
+ const float *states,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings)
return -1;
@@ -524,10 +523,10 @@ int ffwd_pool_spans(const ffwd_config_t *cfg,
}
int ffwd_model_encode_spans_batch(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
+ ffwd_workspace_t *ws,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
return -1;
@@ -579,8 +578,8 @@ int ffwd_model_encode_spans_batch(const ffwd_model_t *model,
for (int b = 0; b < batch; b++) {
const ffwd_context_input_t *input = &inputs[b];
if (ffwd_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden,
- input->input.n_tokens, input->spans, input->n_spans,
- out_embeddings + (size_t)span_offset * hidden) != 0)
+ input->input.n_tokens, input->spans, input->n_spans,
+ out_embeddings + (size_t)span_offset * hidden) != 0)
return -1;
span_offset += input->n_spans;
}
@@ -588,12 +587,12 @@ int ffwd_model_encode_spans_batch(const ffwd_model_t *model,
}
int ffwd_model_encode_spans(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- const ffwd_span_t *spans,
- int n_spans,
- float *out_embeddings) {
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
ffwd_context_input_t input = {
.input = {token_ids, n_tokens},
.spans = spans,
@@ -603,9 +602,9 @@ int ffwd_model_encode_spans(const ffwd_model_t *model,
}
float *ffwd_model_forward(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens) {
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
if (!model || !ws || !token_ids || n_tokens <= 0)
return NULL;
@@ -624,10 +623,10 @@ float *ffwd_model_forward(const ffwd_model_t *model,
}
int ffwd_model_encode_batch(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const ffwd_input_t *inputs,
- int batch,
- float *out_embeddings) {
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
return -1;
@@ -650,10 +649,10 @@ int ffwd_model_encode_batch(const ffwd_model_t *model,
}
int ffwd_pool_batch(const ffwd_config_t *cfg,
- const float *states,
- const int *seq_lengths,
- int batch,
- float *out_embeddings) {
+ const float *states,
+ const int *seq_lengths,
+ int batch,
+ float *out_embeddings) {
if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0)
return -1;
@@ -689,18 +688,18 @@ int ffwd_pool_batch(const ffwd_config_t *cfg,
}
int ffwd_model_encode_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
ffwd_input_t input = {token_ids, n_tokens};
return ffwd_model_encode_batch(model, ws, &input, 1, out_embedding);
}
float *ffwd_model_encode(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens) {
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
if (!model || !ws || !token_ids || n_tokens <= 0)
return NULL;
diff --git a/libffwd/internal.h b/libffwd/internal.h
index 0338bc2..a2b933a 100644
--- a/libffwd/internal.h
+++ b/libffwd/internal.h
@@ -35,59 +35,59 @@ const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model);
size_t ffwd_workspace_nbytes(const ffwd_workspace_t *ws);
float *ffwd_model_encode(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens);
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens);
int ffwd_model_encode_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
int ffwd_model_encode_batch(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const ffwd_input_t *inputs,
- int batch,
- float *out_embeddings);
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings);
int ffwd_pool_batch(const ffwd_config_t *cfg,
- const float *states,
- const int *seq_lengths,
- int batch,
- float *out_embeddings);
+ const float *states,
+ const int *seq_lengths,
+ int batch,
+ float *out_embeddings);
float *ffwd_model_forward(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens);
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens);
int ffwd_model_forward_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_states);
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_states);
int ffwd_pool_spans(const ffwd_config_t *cfg,
- const float *states,
- int n_tokens,
- const ffwd_span_t *spans,
- int n_spans,
- float *out_embeddings);
+ const float *states,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
int ffwd_model_encode_spans(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- const ffwd_span_t *spans,
- int n_spans,
- float *out_embeddings);
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
int ffwd_model_encode_spans_batch(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out_embeddings);
+ ffwd_workspace_t *ws,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings);
/*
* Late-interaction CPU model API. The public engine path exposes reranking via
@@ -103,32 +103,32 @@ const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model);
int ffwd_late_model_token_dim(const ffwd_late_model_t *model);
int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model,
- ffwd_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors);
+ ffwd_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors);
int ffwd_late_model_encode_docs(const ffwd_late_model_t *model,
- ffwd_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets);
+ ffwd_late_workspace_t *ws,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets);
float ffwd_late_maxsim(
const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim);
int ffwd_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores);
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores);
#endif /* FFWD_INTERNAL_H */
diff --git a/libffwd/kernels.h b/libffwd/kernels.h
index f36d0d3..2a917e0 100644
--- a/libffwd/kernels.h
+++ b/libffwd/kernels.h
@@ -27,8 +27,7 @@ void matmul_t(float *C, const float *A, const float *B, int M, int K, int N);
void linear(
float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim);
-void linear_nobias(
- float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim);
+void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim);
/* bf16 weight variants */
@@ -38,27 +37,27 @@ void linear_nobias_bf16(
/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */
void linear_nobias_bf16_pair(float *a,
- float *b,
- const float *x,
- const uint16_t *Wa_bf16,
- const uint16_t *Wb_bf16,
- int seq_len,
- int in_dim,
- int a_dim,
- int b_dim);
+ float *b,
+ const float *x,
+ const uint16_t *Wa_bf16,
+ const uint16_t *Wb_bf16,
+ int seq_len,
+ int in_dim,
+ int a_dim,
+ int b_dim);
/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */
void linear_nobias_bf16_qkv(float *q,
- float *k,
- float *v,
- const float *x,
- const uint16_t *Wq_bf16,
- const uint16_t *Wk_bf16,
- const uint16_t *Wv_bf16,
- int seq_len,
- int in_dim,
- int q_dim,
- int kv_dim);
+ float *k,
+ float *v,
+ const float *x,
+ const uint16_t *Wq_bf16,
+ const uint16_t *Wk_bf16,
+ const uint16_t *Wv_bf16,
+ int seq_len,
+ int in_dim,
+ int q_dim,
+ int kv_dim);
/* Dot product using the best available local SIMD implementation. */
float dot_f32(const float *a, const float *b, int n);
@@ -68,8 +67,7 @@ float dot_f32(const float *a, const float *b, int n);
* ======================================================================== */
/* RMS Normalization: out = x / rms(x) * weight */
-void rms_norm(
- float *out, const float *x, const float *weight, int seq_len, int hidden, float eps);
+void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps);
/* Per-head RMS Normalization for Q/K norms in decoder
* x: [seq, n_heads, head_dim], weight: [head_dim]
@@ -81,12 +79,12 @@ void rms_norm_per_head(
* + beta, with mean and biased (population) variance over the hidden axis per
* row. Unlike RMSNorm this subtracts the mean and adds a bias. */
void layer_norm(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int seq_len,
- int hidden,
- float eps);
+ const float *x,
+ const float *gamma,
+ const float *beta,
+ int seq_len,
+ int hidden,
+ float eps);
/* ========================================================================
* Activation Functions
@@ -119,57 +117,57 @@ void gelu_tanh_inplace(float *x, int n);
* offsets[b]..offsets[b+1] span.
*/
void bidirectional_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale);
/* Optional reusable scratch for the BLAS-tiled packed attention path. */
size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
void bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
/*
* Causal GQA attention over a packed/ragged batch. Query position i attends
* only to positions at or before i within its sequence.
*/
void causal_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale);
void causal_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
/* ========================================================================
* Position Embeddings
diff --git a/libffwd/late.c b/libffwd/late.c
index 05d0341..e086ad3 100644
--- a/libffwd/late.c
+++ b/libffwd/late.c
@@ -105,9 +105,7 @@ const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model) {
return model && model->base ? &model->base->config : NULL;
}
-int ffwd_late_model_token_dim(const ffwd_late_model_t *model) {
- return model ? model->token_dim : 0;
-}
+int ffwd_late_model_token_dim(const ffwd_late_model_t *model) { return model ? model->token_dim : 0; }
/* Internal (internal.h): expose the base model and projection so GPU
* backends can upload them to device without seeing struct ffwd_late_model. */
@@ -140,12 +138,12 @@ float ffwd_late_maxsim(
}
int ffwd_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores) {
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores) {
if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || docs <= 0 ||
dim <= 0 || doc_offsets[0] != 0)
return -1;
@@ -211,8 +209,8 @@ int ffwd_late_maxsim_batch(const float *query_vectors,
for (int i = 0; i < docs; i++) {
int start = doc_offsets[i];
int end = doc_offsets[i + 1];
- scores[i] = ffwd_late_maxsim(query_vectors, query_tokens,
- doc_vectors + (size_t)start * dim, end - start, dim);
+ scores[i] = ffwd_late_maxsim(query_vectors, query_tokens, doc_vectors + (size_t)start * dim,
+ end - start, dim);
}
return 0;
}
diff --git a/libffwd/late_blas.c b/libffwd/late_blas.c
index cc57596..dabeb58 100644
--- a/libffwd/late_blas.c
+++ b/libffwd/late_blas.c
@@ -55,11 +55,11 @@ void ffwd_late_workspace_free(ffwd_late_workspace_t *ws) {
}
int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model,
- ffwd_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
+ ffwd_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
int dim = ffwd_late_model_token_dim(model);
ffwd_model_t *base = ffwd_late_model_base(model);
const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model);
@@ -86,15 +86,15 @@ int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model,
/* Late-interaction batch encoder. Reuses the base model's packed/ragged batch
* forward pass, then gathers kept rows and projects them in one GEMM. */
int ffwd_late_model_encode_docs(const ffwd_late_model_t *model,
- ffwd_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets) {
+ ffwd_late_workspace_t *ws,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets) {
int dim = ffwd_late_model_token_dim(model);
ffwd_model_t *base = ffwd_late_model_base(model);
const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model);
diff --git a/libffwd/mlx.c b/libffwd/mlx.c
index 10299c1..0b6d8b7 100644
--- a/libffwd/mlx.c
+++ b/libffwd/mlx.c
@@ -327,9 +327,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) {
return res;
}
-const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx) {
- return ctx ? &ctx->config : NULL;
-}
+const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx) { return ctx ? &ctx->config : NULL; }
/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the
* tiny RMSNorm vectors so non-quantized BF16 models can run norms and the
@@ -432,8 +430,7 @@ static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer
snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p);
ctx->ffwd_ln_b = load_tensor(ctx->ms, name, ln_shape, 1);
if (!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->position_embeddings) ||
- !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->ffwd_ln_w) ||
- !arr_ok(ctx->ffwd_ln_b))
+ !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->ffwd_ln_w) || !arr_ok(ctx->ffwd_ln_b))
return -1;
}
@@ -483,10 +480,10 @@ static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer
}
static ffwd_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
- int layer_start,
- int layer_end,
- const ffwd_mlx_options_t *opts,
- int allow_late) {
+ int layer_start,
+ int layer_end,
+ const ffwd_mlx_options_t *opts,
+ int allow_late) {
ffwd_mlx_options_t options = normalize_mlx_options(opts);
if (options.quantize_bits < 0)
return NULL;
@@ -703,8 +700,7 @@ ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir) {
return mlx_load_range_ex(model_dir, 0, -1, NULL, 0);
}
-ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir,
- const ffwd_mlx_options_t *opts) {
+ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, const ffwd_mlx_options_t *opts) {
return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
}
@@ -756,7 +752,7 @@ void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx) {
}
ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir,
- const ffwd_mlx_options_t *opts) {
+ const ffwd_mlx_options_t *opts) {
if (!mlx_model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "mlx late: missing 1_Dense projection\n");
return NULL;
@@ -990,7 +986,7 @@ static mlx_array mlx_forward_bert(ffwd_mlx_ctx_t *ctx,
mlx_add(&mid_b, mid, l->ffn_inter_bias, S);
mlx_array_free(mid);
mlx_array g = ctx->config.ffn_act == FFWD_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S)
- : mlx_gelu_erf(mid_b, S);
+ : mlx_gelu_erf(mid_b, S);
mlx_array_free(mid_b);
mlx_array out = linear(g, &l->down_proj, S);
mlx_array_free(g);
@@ -1171,9 +1167,9 @@ static mlx_array mlx_forward_layers(ffwd_mlx_ctx_t *ctx,
}
ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize) {
+ const int *token_ids,
+ int n_tokens,
+ int normalize) {
if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 ||
!linear_ok(ctx->projection))
return NULL;
@@ -1302,9 +1298,7 @@ int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs) {
return vecs ? vecs->tokens : 0;
}
-int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs) {
- return vecs ? vecs->dim : 0;
-}
+int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs) { return vecs ? vecs->dim : 0; }
int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors) {
if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0)
@@ -1351,13 +1345,13 @@ int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_v
* within MLX's batched-GEMM rounding.
*/
ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- int *out_offsets) {
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ int *out_offsets) {
if (!ctx || !ctx->base || !doc_ids || !n_tokens || !keep || !n_keep || !out_offsets ||
n_docs <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection))
return NULL;
@@ -1577,8 +1571,9 @@ cleanup:
* candidates through ffwd_mlx_late_encode_docs_device, but the late-interaction
* verification harness (tests/integration/check_late_interaction.py) drives these directly
* to validate per-document gather and concat against the CPU MaxSim reference. */
-ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(
- ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count) {
+ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(ffwd_mlx_late_ctx_t *ctx,
+ const ffwd_mlx_late_vectors_t *const *items,
+ int count) {
if (!ctx || !ctx->base || !items || count <= 0)
return NULL;
int dim = 0;
@@ -1627,9 +1622,9 @@ ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(
}
ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx,
- const ffwd_mlx_late_vectors_t *vecs,
- const int *token_indices,
- int count) {
+ const ffwd_mlx_late_vectors_t *vecs,
+ const int *token_indices,
+ int count) {
if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) ||
!token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0)
return NULL;
@@ -1675,11 +1670,11 @@ ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx,
}
int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx,
- const ffwd_mlx_late_vectors_t *query,
- const ffwd_mlx_late_vectors_t *docs,
- const int *doc_offsets,
- int docs_count,
- float *scores) {
+ const ffwd_mlx_late_vectors_t *query,
+ const ffwd_mlx_late_vectors_t *docs,
+ const int *doc_offsets,
+ int docs_count,
+ float *scores) {
if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores ||
query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim ||
query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) ||
@@ -1882,11 +1877,8 @@ cleanup:
return rc;
}
-int ffwd_mlx_late_encode_tokens(ffwd_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
+int ffwd_mlx_late_encode_tokens(
+ ffwd_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, float *out_vectors) {
ffwd_mlx_late_vectors_t *vecs =
ffwd_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize);
if (!vecs)
@@ -1897,9 +1889,9 @@ int ffwd_mlx_late_encode_tokens(ffwd_mlx_late_ctx_t *ctx,
}
static int ffwd_mlx_encode_batch_dense(ffwd_mlx_ctx_t *ctx,
- const ffwd_input_t *inputs,
- int batch,
- float *out_embeddings) {
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
ctx->layer_end != ctx->config.n_layers)
return -1;
@@ -2208,10 +2200,8 @@ static int mlx_batch_should_split(const ffwd_input_t *inputs,
return mlx_padding_exceeds_limit(padded, (size_t)total_tokens);
}
-static int mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx,
- int group_count,
- int group_max,
- int next_tokens) {
+static int
+mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx, int group_count, int group_max, int next_tokens) {
if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0)
return 1;
@@ -2231,9 +2221,9 @@ static int mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx,
}
int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx,
- const ffwd_input_t *inputs,
- int batch,
- float *out_embeddings) {
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
@@ -2306,9 +2296,9 @@ int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx,
}
int ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
ffwd_input_t input = {token_ids, n_tokens};
return ffwd_mlx_encode_batch(ctx, &input, 1, out_embedding);
}
@@ -2333,9 +2323,8 @@ static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) {
return padded * 4 > tokens * 5;
}
-static int mlx_context_batch_should_split(const ffwd_context_input_t *inputs,
- int batch,
- int *out_total_spans) {
+static int
+mlx_context_batch_should_split(const ffwd_context_input_t *inputs, int batch, int *out_total_spans) {
int max_seq = 0;
int total_tokens = 0;
int total_spans = 0;
@@ -2362,9 +2351,9 @@ static int mlx_context_batch_should_split(const ffwd_context_input_t *inputs,
}
static int ffwd_mlx_encode_spans_batch_dense(ffwd_mlx_ctx_t *ctx,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
return -1;
@@ -2663,9 +2652,9 @@ cleanup:
}
int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
@@ -2748,11 +2737,11 @@ int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx,
}
int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- const ffwd_span_t *spans,
- int n_spans,
- float *out_embeddings) {
+ const int *token_ids,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
ffwd_context_input_t input = {
.input = {token_ids, n_tokens},
.spans = spans,
diff --git a/libffwd/mlx.h b/libffwd/mlx.h
index d8a915f..e4e2a8e 100644
--- a/libffwd/mlx.h
+++ b/libffwd/mlx.h
@@ -38,7 +38,7 @@ typedef struct {
*/
FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir);
FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir,
- const ffwd_mlx_options_t *opts);
+ const ffwd_mlx_options_t *opts);
FFWD_API void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx);
@@ -49,7 +49,7 @@ FFWD_API void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx);
*/
FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load(const char *model_dir);
FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir,
- const ffwd_mlx_options_t *opts);
+ const ffwd_mlx_options_t *opts);
FFWD_API void ffwd_mlx_late_free(ffwd_mlx_late_ctx_t *ctx);
FFWD_API const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx);
@@ -72,9 +72,9 @@ FFWD_API int ffwd_mlx_late_encode_tokens(
* and copy back only final candidate scores.
*/
FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize);
+ const int *token_ids,
+ int n_tokens,
+ int normalize);
FFWD_API void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs);
FFWD_API int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs);
FFWD_API int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs);
@@ -87,29 +87,28 @@ FFWD_API int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, flo
* Returns NULL on error; the handle must not outlive ctx.
*/
FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- int *out_offsets);
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ int *out_offsets);
/* Device-resident select/concat primitives, retained for the late-interaction
* verification harness (tests/integration/check_late_interaction.py); the server rerank
* path uses ffwd_mlx_late_encode_docs_device. */
FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(
ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count);
-FFWD_API ffwd_mlx_late_vectors_t *
-ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx,
- const ffwd_mlx_late_vectors_t *vecs,
- const int *token_indices,
- int count);
+FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx,
+ const ffwd_mlx_late_vectors_t *vecs,
+ const int *token_indices,
+ int count);
FFWD_API int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx,
- const ffwd_mlx_late_vectors_t *query,
- const ffwd_mlx_late_vectors_t *docs,
- const int *doc_offsets,
- int docs_count,
- float *scores);
+ const ffwd_mlx_late_vectors_t *query,
+ const ffwd_mlx_late_vectors_t *docs,
+ const int *doc_offsets,
+ int docs_count,
+ float *scores);
/*
* Compute embedding for token_ids[0..n_tokens-1].
@@ -132,29 +131,29 @@ ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, fl
* Returns 0 on success, -1 on error.
*/
FFWD_API int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx,
- const ffwd_input_t *inputs,
- int batch,
- float *out_embeddings);
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings);
/*
* Run one contextual sequence and pool selected token spans.
* Returns 0 on success, -1 on error.
*/
FFWD_API int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- const ffwd_span_t *spans,
- int n_spans,
- float *out_embeddings);
+ const int *token_ids,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
/*
* Run a padded dense contextual document batch and pool every selected span.
* out_embeddings contains spans in document order.
*/
FFWD_API int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out_embeddings);
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings);
/* Get the config (so ffwd_cli.c can read hidden_size etc.) */
FFWD_API const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx);
diff --git a/libffwd/model_types.h b/libffwd/model_types.h
index 2719a5b..345e0cd 100644
--- a/libffwd/model_types.h
+++ b/libffwd/model_types.h
@@ -26,13 +26,13 @@ typedef struct {
ffwd_weight_ref_t wk; /* [kv_dim, hidden] */
ffwd_weight_ref_t wv; /* [kv_dim, hidden] */
ffwd_weight_ref_t wo; /* [hidden, q_dim] */
- const float *q_bias; /* [q_dim], optional */
- const float *k_bias; /* [kv_dim], optional */
- const float *v_bias; /* [kv_dim], optional */
- const float *q_norm; /* [head_dim] */
- const float *k_norm; /* [head_dim] */
- const float *input_norm; /* [hidden] */
- const float *post_attn_norm; /* [hidden] */
+ const float *q_bias; /* [q_dim], optional */
+ const float *k_bias; /* [kv_dim], optional */
+ const float *v_bias; /* [kv_dim], optional */
+ const float *q_norm; /* [head_dim] */
+ const float *k_norm; /* [head_dim] */
+ const float *input_norm; /* [hidden] */
+ const float *post_attn_norm; /* [hidden] */
ffwd_weight_ref_t gate_proj; /* [intermediate, hidden] */
ffwd_weight_ref_t up_proj; /* [intermediate, hidden] */
ffwd_weight_ref_t down_proj; /* [hidden, intermediate] */
@@ -51,27 +51,27 @@ typedef struct {
typedef struct {
ffwd_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
ffwd_layer_t *layers; /* [n_layers] heap-allocated */
- const float *norm; /* [hidden]; NULL for BERT (no final norm) */
+ const float *norm; /* [hidden]; NULL for BERT (no final norm) */
/* BERT family (NULL for Qwen3): learned absolute position embeddings,
* token-type (segment) embeddings, and the embedding LayerNorm. */
const float *position_embeddings; /* [max_position_embeddings, hidden] */
const float *token_type_embeddings; /* [type_vocab_size, hidden] */
- const float *ffwd_ln_w; /* embedding LayerNorm weight [hidden] */
- const float *ffwd_ln_b; /* embedding LayerNorm bias [hidden] */
+ const float *ffwd_ln_w; /* embedding LayerNorm weight [hidden] */
+ const float *ffwd_ln_b; /* embedding LayerNorm bias [hidden] */
} ffwd_weights_t;
typedef void (*ffwd_attention_fn)(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
struct ffwd_model {
ffwd_config_t config;
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c
index f7a50f7..df5861e 100644
--- a/libffwd/tokenize.c
+++ b/libffwd/tokenize.c
@@ -189,9 +189,7 @@ void ffwd_tok_free(ffwd_tok_t *t) {
free(t);
}
-int ffwd_tok_context_separator_id(const ffwd_tok_t *t) {
- return t ? t->context_separator_id : -1;
-}
+int ffwd_tok_context_separator_id(const ffwd_tok_t *t) { return t ? t->context_separator_id : -1; }
void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on) {
if (t)
@@ -289,10 +287,7 @@ static int late_id_is_skipped(const ffwd_tok_t *t, int id) {
return 0;
}
-int ffwd_tokenize_late(ffwd_tok_t *t,
- const char *text,
- int is_query,
- ffwd_late_tokens_t *out) {
+int ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out) {
memset(out, 0, sizeof(*out));
if (!t || !text)
return -1;
diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h
index 66c62a0..c93140a 100644
--- a/libffwd/wordpiece.h
+++ b/libffwd/wordpiece.h
@@ -52,18 +52,18 @@ void tok_wp_workspace_free(tok_wp_workspace_t *ws);
* on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to
* the number of tokens needed/emitted in all cases. */
int tok_wp_encode_into(const tok_wp_t *tok,
- tok_wp_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
+ tok_wp_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
/* Same tokenization as tok_wp_encode(), reusing ws internally and
* performing only the final returned int[] allocation. */
int *tok_wp_encode_with_workspace(const tok_wp_t *tok,
- tok_wp_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
+ tok_wp_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
void tok_wp_free(tok_wp_t *tok);
diff --git a/tests/test_cls.c b/tests/test_cls.c
index 79db064..b6e2dfb 100644
--- a/tests/test_cls.c
+++ b/tests/test_cls.c
@@ -37,8 +37,8 @@ int main(void) {
fprintf(stderr, "model setup failed\n");
return 1;
}
- if (cfg->pooling_mode != FFWD_POOL_CLS ||
- cfg->attention_mode != FFWD_ATTENTION_BIDIRECTIONAL || !cfg->normalize_embeddings) {
+ if (cfg->pooling_mode != FFWD_POOL_CLS || cfg->attention_mode != FFWD_ATTENTION_BIDIRECTIONAL ||
+ !cfg->normalize_embeddings) {
fprintf(stderr, "CLS pooling metadata was not detected\n");
return 1;
}
diff --git a/tests/test_late.c b/tests/test_late.c
index fccd040..37a1bf1 100644
--- a/tests/test_late.c
+++ b/tests/test_late.c
@@ -48,7 +48,7 @@ static int check_maxsim_batch_parity(const float *query,
}
for (int i = 0; i < n_docs; i++) {
float one = ffwd_late_maxsim(query, query_tokens, docs + (size_t)offsets[i] * dim,
- offsets[i + 1] - offsets[i], dim);
+ offsets[i + 1] - offsets[i], dim);
if (fabsf(scores[i] - one) > 0.0001f) {
fprintf(stderr, "maxsim_batch (%s) doc %d: %g vs %g\n", what, i, scores[i], one);
return 1;
@@ -257,7 +257,7 @@ int main(void) {
}
for (int i = 0; i < 3; i++) {
if (ffwd_late_model_encode_tokens(late, ws, doc_ids[i], lens[i], 1,
- doc_vecs + (size_t)offsets[i] * TOKEN_DIM) != 0) {
+ doc_vecs + (size_t)offsets[i] * TOKEN_DIM) != 0) {
fprintf(stderr, "doc encode failed\n");
goto fail;
}
@@ -269,10 +269,8 @@ int main(void) {
float scores[3];
int bad_first[4] = {1, 2, 3, 4};
int empty_doc[3] = {0, 2, 2};
- if (ffwd_late_maxsim_batch(vecs, n_tokens, doc_vecs, bad_first, 3, TOKEN_DIM, scores) ==
- 0 ||
- ffwd_late_maxsim_batch(vecs, n_tokens, doc_vecs, empty_doc, 2, TOKEN_DIM, scores) ==
- 0 ||
+ if (ffwd_late_maxsim_batch(vecs, n_tokens, doc_vecs, bad_first, 3, TOKEN_DIM, scores) == 0 ||
+ ffwd_late_maxsim_batch(vecs, n_tokens, doc_vecs, empty_doc, 2, TOKEN_DIM, scores) == 0 ||
ffwd_late_maxsim_batch(vecs, n_tokens, doc_vecs, offsets, 3, TOKEN_DIM, NULL) == 0) {
fprintf(stderr, "maxsim_batch accepted invalid arguments\n");
goto fail;
@@ -297,8 +295,8 @@ int main(void) {
keep_arrs[i][t] = t;
}
if (ffwd_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens,
- (const int *const *)keep_arrs, lens, 3, 1, batched,
- batch_offsets) != 0) {
+ (const int *const *)keep_arrs, lens, 3, 1, batched,
+ batch_offsets) != 0) {
fprintf(stderr, "encode_docs failed\n");
goto fail;
}
@@ -333,7 +331,7 @@ int main(void) {
if (!filtered)
goto fail;
if (ffwd_late_model_encode_docs(late, ws, one_ids, one_len, one_keep, one_nkeep, 1, 1,
- filtered, one_off) != 0 ||
+ filtered, one_off) != 0 ||
one_off[1] != nk0 ||
max_abs_diff(filtered, doc_vecs + TOKEN_DIM, (size_t)nk0 * TOKEN_DIM) > 0.0001f) {
fprintf(stderr, "encode_docs keep-filter mismatch\n");
@@ -345,14 +343,14 @@ int main(void) {
/* Invalid arguments must be rejected. */
if (ffwd_late_model_encode_docs(NULL, ws, (const int *const *)doc_ids, lens,
- (const int *const *)keep_arrs, lens, 3, 1, batched,
- batch_offsets) == 0 ||
+ (const int *const *)keep_arrs, lens, 3, 1, batched,
+ batch_offsets) == 0 ||
ffwd_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens,
- (const int *const *)keep_arrs, lens, 0, 1, batched,
- batch_offsets) == 0 ||
+ (const int *const *)keep_arrs, lens, 0, 1, batched,
+ batch_offsets) == 0 ||
ffwd_late_model_encode_docs(late, ws, (const int *const *)doc_ids, lens,
- (const int *const *)keep_arrs, lens, 3, 1, NULL,
- batch_offsets) == 0) {
+ (const int *const *)keep_arrs, lens, 3, 1, NULL,
+ batch_offsets) == 0) {
fprintf(stderr, "encode_docs accepted invalid arguments\n");
goto fail;
}
diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c
index 5d1387a..50f7e02 100644
--- a/tests/test_qwen3.c
+++ b/tests/test_qwen3.c
@@ -36,9 +36,8 @@ int main(void) {
fprintf(stderr, "model setup failed\n");
return 1;
}
- if (cfg->attention_mode != FFWD_ATTENTION_CAUSAL ||
- cfg->pooling_mode != FFWD_POOL_LAST_TOKEN || !cfg->normalize_embeddings ||
- !cfg->append_terminal_token) {
+ if (cfg->attention_mode != FFWD_ATTENTION_CAUSAL || cfg->pooling_mode != FFWD_POOL_LAST_TOKEN ||
+ !cfg->normalize_embeddings || !cfg->append_terminal_token) {
fprintf(stderr, "Qwen3 config semantics were not detected\n");
return 1;
}
diff --git a/tests/test_workspace.c b/tests/test_workspace.c
index 1921465..ca5b136 100644
--- a/tests/test_workspace.c
+++ b/tests/test_workspace.c
@@ -165,8 +165,8 @@ static int check_spans_batch_parity(const ffwd_model_t *model,
ffwd_span_t spans0[2] = {{0, ntok[0]}, {ntok[0] + 1, ntok[1]}};
ffwd_span_t span1 = {0, ntok[2]};
if (ffwd_model_encode_spans(model, ws, doc0, n0, spans0, 2, expected) != 0 ||
- ffwd_model_encode_spans(model, ws, ids[2], ntok[2], &span1, 1,
- expected + (size_t)2 * dim) != 0) {
+ ffwd_model_encode_spans(model, ws, ids[2], ntok[2], &span1, 1, expected + (size_t)2 * dim) !=
+ 0) {
fprintf(stderr, "singleton contextual embedding failed\n");
goto done;
}
diff --git a/tools/server/models.c b/tools/server/models.c
index 479ff97..91aee18 100644
--- a/tools/server/models.c
+++ b/tools/server/models.c
@@ -228,9 +228,9 @@ int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, flo
}
int model_ffwd_spans_batch(loaded_model *m,
- const ffwd_context_input_t *inputs,
- int batch,
- float *out) {
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
return ffwd_encode_spans_batch(m->backend, inputs, batch, out);
}