From f7ef995ff54da671f9d7665b3d92ff25934c1e8b Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Thu, 25 Jun 2026 17:58:46 +0200 Subject: chore: run clang-format --- libffwd/backend_blas.c | 23 +++------ libffwd/backend_cuda.c | 21 +++----- libffwd/backend_mlx.c | 18 +++---- libffwd/cuda.h | 41 +++++++-------- libffwd/forward.c | 81 +++++++++++++++--------------- libffwd/internal.h | 114 ++++++++++++++++++++--------------------- libffwd/kernels.h | 134 ++++++++++++++++++++++++------------------------- libffwd/late.c | 20 ++++---- libffwd/late_blas.c | 28 +++++------ libffwd/mlx.c | 125 +++++++++++++++++++++------------------------ libffwd/mlx.h | 65 ++++++++++++------------ libffwd/model_types.h | 42 ++++++++-------- libffwd/tokenize.c | 9 +--- libffwd/wordpiece.h | 16 +++--- 14 files changed, 345 insertions(+), 392 deletions(-) (limited to 'libffwd') diff --git a/libffwd/backend_blas.c b/libffwd/backend_blas.c index edf54bb..9e3d376 100644 --- a/libffwd/backend_blas.c +++ b/libffwd/backend_blas.c @@ -38,8 +38,8 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { return 0; } -ffwd_t *ffwd_open( - const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { +ffwd_t * +ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { (void)opts; ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b)); if (!b) { @@ -102,9 +102,7 @@ const ffwd_config_t *ffwd_config(const ffwd_t *b) { return b->is_late ? ffwd_late_model_config(b->late) : ffwd_model_config(b->model); } -int ffwd_token_dim(const ffwd_t *b) { - return b->is_late ? ffwd_late_model_token_dim(b->late) : 0; -} +int ffwd_token_dim(const ffwd_t *b) { return b->is_late ? ffwd_late_model_token_dim(b->late) : 0; } int ffwd_uses_dense_batches(const ffwd_t *b) { (void)b; @@ -115,10 +113,7 @@ int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *o return ffwd_model_encode_batch(b->model, b->ws, inputs, batch, out); } -int ffwd_encode_spans_batch(ffwd_t *b, - const ffwd_context_input_t *inputs, - int batch, - float *out) { +int ffwd_encode_spans_batch(ffwd_t *b, const ffwd_context_input_t *inputs, int batch, float *out) { return ffwd_model_encode_spans_batch(b->model, b->ws, inputs, batch, out); } @@ -136,15 +131,13 @@ int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) { /* Encode every candidate in one block-diagonal forward: the encoder packs * each document's kept token vectors back-to-back and fills offsets. */ if (rc == 0) - rc = ffwd_late_model_encode_tokens(b->late, b->late_ws, in->query_ids, in->query_n_tokens, - 1, query); + rc = ffwd_late_model_encode_tokens(b->late, b->late_ws, in->query_ids, in->query_n_tokens, 1, + query); if (rc == 0) rc = ffwd_late_model_encode_docs(b->late, b->late_ws, in->doc_ids, in->doc_n_tokens, - in->doc_keep, in->doc_n_keep, in->n_docs, 1, docs, - offsets); + in->doc_keep, in->doc_n_keep, in->n_docs, 1, docs, offsets); if (rc == 0) - rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, - scores); + rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, scores); free(offsets); free(docs); free(query); diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c index dd05c16..a925404 100644 --- a/libffwd/backend_cuda.c +++ b/libffwd/backend_cuda.c @@ -58,8 +58,8 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { return 0; } -ffwd_t *ffwd_open( - const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { +ffwd_t * +ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { (void)opts; ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b)); if (!b || !(b->dir = strdup(model_dir))) { @@ -108,9 +108,7 @@ const ffwd_config_t *ffwd_config(const ffwd_t *b) { return b->is_late ? ffwd_cuda_late_config(b->late_ctx) : ffwd_cuda_config(b->ctx); } -int ffwd_token_dim(const ffwd_t *b) { - return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0; -} +int ffwd_token_dim(const ffwd_t *b) { return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0; } int ffwd_uses_dense_batches(const ffwd_t *b) { (void)b; @@ -121,10 +119,7 @@ int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *o return ffwd_cuda_encode_batch(b->ctx, inputs, batch, out); } -int ffwd_encode_spans_batch(ffwd_t *b, - const ffwd_context_input_t *inputs, - int batch, - float *out) { +int ffwd_encode_spans_batch(ffwd_t *b, const ffwd_context_input_t *inputs, int batch, float *out) { return ffwd_cuda_encode_spans_batch(b->ctx, inputs, batch, out); } @@ -142,14 +137,12 @@ int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) { /* GPU encodes the query and every candidate in one packed forward; MaxSim * runs on the host, where the grouped-GEMM scorer beats a device graph. */ if (rc == 0) - rc = - ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query); + rc = ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query); if (rc == 0) rc = ffwd_cuda_late_encode_docs(b->late_ctx, in->doc_ids, in->doc_n_tokens, in->doc_keep, - in->doc_n_keep, in->n_docs, 1, docs, offsets); + in->doc_n_keep, in->n_docs, 1, docs, offsets); if (rc == 0) - rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, - scores); + rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, scores); free(offsets); free(docs); free(query); diff --git a/libffwd/backend_mlx.c b/libffwd/backend_mlx.c index 80b893c..932e550 100644 --- a/libffwd/backend_mlx.c +++ b/libffwd/backend_mlx.c @@ -62,8 +62,8 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { return 0; } -ffwd_t *ffwd_open( - const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { +ffwd_t * +ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b)); if (!b || !(b->dir = strdup(model_dir))) { snprintf(err, errlen, "out of memory"); @@ -120,9 +120,7 @@ const ffwd_config_t *ffwd_config(const ffwd_t *b) { return b->is_late ? ffwd_mlx_late_config(b->late_ctx) : ffwd_mlx_config(b->ctx); } -int ffwd_token_dim(const ffwd_t *b) { - return b->is_late ? ffwd_mlx_late_token_dim(b->late_ctx) : 0; -} +int ffwd_token_dim(const ffwd_t *b) { return b->is_late ? ffwd_mlx_late_token_dim(b->late_ctx) : 0; } int ffwd_uses_dense_batches(const ffwd_t *b) { return b->ctx != NULL; } @@ -130,10 +128,7 @@ int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *o return ffwd_mlx_encode_batch(b->ctx, inputs, batch, out); } -int ffwd_encode_spans_batch(ffwd_t *b, - const ffwd_context_input_t *inputs, - int batch, - float *out) { +int ffwd_encode_spans_batch(ffwd_t *b, const ffwd_context_input_t *inputs, int batch, float *out) { return ffwd_mlx_encode_spans_batch(b->ctx, inputs, batch, out); } @@ -144,11 +139,10 @@ int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) { ffwd_mlx_late_vectors_t *packed = (query && offsets) ? ffwd_mlx_late_encode_docs_device(b->late_ctx, in->doc_ids, in->doc_n_tokens, - in->doc_keep, in->doc_n_keep, in->n_docs, 1, - offsets) + in->doc_keep, in->doc_n_keep, in->n_docs, 1, offsets) : NULL; int rc = packed ? ffwd_mlx_late_maxsim_batch_device(b->late_ctx, query, packed, offsets, - in->n_docs, scores) + in->n_docs, scores) : -1; ffwd_mlx_late_vectors_free(packed); ffwd_mlx_late_vectors_free(query); diff --git a/libffwd/cuda.h b/libffwd/cuda.h index 7fc3515..851e412 100644 --- a/libffwd/cuda.h +++ b/libffwd/cuda.h @@ -40,11 +40,8 @@ FFWD_API const ffwd_config_t *ffwd_cuda_late_config(const ffwd_cuda_late_ctx_t * FFWD_API int ffwd_cuda_late_token_dim(const ffwd_cuda_late_ctx_t *ctx); /* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */ -FFWD_API int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors); +FFWD_API int ffwd_cuda_late_encode_tokens( + ffwd_cuda_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, float *out_vectors); /* * Encode n_docs documents in one packed forward and pack each document's kept @@ -53,29 +50,27 @@ FFWD_API int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx, * (the layout ffwd_late_maxsim_batch consumes). Returns 0 on success. */ FFWD_API int ffwd_cuda_late_encode_docs(ffwd_cuda_late_ctx_t *ctx, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - float *out_vectors, - int *out_offsets); + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets); FFWD_API const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx); -FFWD_API int ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx, - const int *token_ids, - int n_tokens, - float *out_embedding); +FFWD_API int +ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding); FFWD_API float *ffwd_cuda_encode(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens); FFWD_API int ffwd_cuda_encode_batch(ffwd_cuda_ctx_t *ctx, - const ffwd_input_t *inputs, - int batch, - float *out_embeddings); + const ffwd_input_t *inputs, + int batch, + float *out_embeddings); FFWD_API int ffwd_cuda_encode_spans_batch(ffwd_cuda_ctx_t *ctx, - const ffwd_context_input_t *inputs, - int batch, - float *out_embeddings); + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings); #endif /* FFWD_CUDA_H */ diff --git a/libffwd/forward.c b/libffwd/forward.c index e8a5cdf..d8e45f9 100644 --- a/libffwd/forward.c +++ b/libffwd/forward.c @@ -225,8 +225,7 @@ static int forward_packed_slice_inplace(const ffwd_model_t *model, for (int i = 0; i < inputs[b].n_tokens; i++) { int id = inputs[b].ids[i]; if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, - i); + fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, i); return -1; } copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id, @@ -466,10 +465,10 @@ static int pool_embeddings(const ffwd_model_t *model, * ======================================================================== */ int ffwd_model_forward_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_states) { + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_states) { if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) return -1; @@ -488,11 +487,11 @@ int ffwd_model_forward_into(const ffwd_model_t *model, } int ffwd_pool_spans(const ffwd_config_t *cfg, - const float *states, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings) { + const float *states, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings) { if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings) return -1; @@ -524,10 +523,10 @@ int ffwd_pool_spans(const ffwd_config_t *cfg, } int ffwd_model_encode_spans_batch(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_context_input_t *inputs, - int batch, - float *out_embeddings) { + ffwd_workspace_t *ws, + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; @@ -579,8 +578,8 @@ int ffwd_model_encode_spans_batch(const ffwd_model_t *model, for (int b = 0; b < batch; b++) { const ffwd_context_input_t *input = &inputs[b]; if (ffwd_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden, - input->input.n_tokens, input->spans, input->n_spans, - out_embeddings + (size_t)span_offset * hidden) != 0) + input->input.n_tokens, input->spans, input->n_spans, + out_embeddings + (size_t)span_offset * hidden) != 0) return -1; span_offset += input->n_spans; } @@ -588,12 +587,12 @@ int ffwd_model_encode_spans_batch(const ffwd_model_t *model, } int ffwd_model_encode_spans(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings) { + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings) { ffwd_context_input_t input = { .input = {token_ids, n_tokens}, .spans = spans, @@ -603,9 +602,9 @@ int ffwd_model_encode_spans(const ffwd_model_t *model, } float *ffwd_model_forward(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens) { + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens) { if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; @@ -624,10 +623,10 @@ float *ffwd_model_forward(const ffwd_model_t *model, } int ffwd_model_encode_batch(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_input_t *inputs, - int batch, - float *out_embeddings) { + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, + int batch, + float *out_embeddings) { if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; @@ -650,10 +649,10 @@ int ffwd_model_encode_batch(const ffwd_model_t *model, } int ffwd_pool_batch(const ffwd_config_t *cfg, - const float *states, - const int *seq_lengths, - int batch, - float *out_embeddings) { + const float *states, + const int *seq_lengths, + int batch, + float *out_embeddings) { if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0) return -1; @@ -689,18 +688,18 @@ int ffwd_pool_batch(const ffwd_config_t *cfg, } int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding) { + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding) { ffwd_input_t input = {token_ids, n_tokens}; return ffwd_model_encode_batch(model, ws, &input, 1, out_embedding); } float *ffwd_model_encode(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens) { + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens) { if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; diff --git a/libffwd/internal.h b/libffwd/internal.h index 0338bc2..a2b933a 100644 --- a/libffwd/internal.h +++ b/libffwd/internal.h @@ -35,59 +35,59 @@ const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model); size_t ffwd_workspace_nbytes(const ffwd_workspace_t *ws); float *ffwd_model_encode(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens); + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens); int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding); + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding); int ffwd_model_encode_batch(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_input_t *inputs, - int batch, - float *out_embeddings); + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, + int batch, + float *out_embeddings); int ffwd_pool_batch(const ffwd_config_t *cfg, - const float *states, - const int *seq_lengths, - int batch, - float *out_embeddings); + const float *states, + const int *seq_lengths, + int batch, + float *out_embeddings); float *ffwd_model_forward(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens); + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens); int ffwd_model_forward_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_states); + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_states); int ffwd_pool_spans(const ffwd_config_t *cfg, - const float *states, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings); + const float *states, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings); int ffwd_model_encode_spans(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const int *token_ids, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings); + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings); int ffwd_model_encode_spans_batch(const ffwd_model_t *model, - ffwd_workspace_t *ws, - const ffwd_context_input_t *inputs, - int batch, - float *out_embeddings); + ffwd_workspace_t *ws, + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings); /* * Late-interaction CPU model API. The public engine path exposes reranking via @@ -103,32 +103,32 @@ const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model); int ffwd_late_model_token_dim(const ffwd_late_model_t *model); int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model, - ffwd_late_workspace_t *ws, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors); + ffwd_late_workspace_t *ws, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors); int ffwd_late_model_encode_docs(const ffwd_late_model_t *model, - ffwd_late_workspace_t *ws, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - float *out_vectors, - int *out_offsets); + ffwd_late_workspace_t *ws, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets); float ffwd_late_maxsim( const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim); int ffwd_late_maxsim_batch(const float *query_vectors, - int query_tokens, - const float *doc_vectors, - const int *doc_offsets, - int docs, - int dim, - float *scores); + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores); #endif /* FFWD_INTERNAL_H */ diff --git a/libffwd/kernels.h b/libffwd/kernels.h index f36d0d3..2a917e0 100644 --- a/libffwd/kernels.h +++ b/libffwd/kernels.h @@ -27,8 +27,7 @@ void matmul_t(float *C, const float *A, const float *B, int M, int K, int N); void linear( float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); -void linear_nobias( - float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); +void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); /* bf16 weight variants */ @@ -38,27 +37,27 @@ void linear_nobias_bf16( /* Small-sequence BF16 path: compute two projections with one threaded dispatch. */ void linear_nobias_bf16_pair(float *a, - float *b, - const float *x, - const uint16_t *Wa_bf16, - const uint16_t *Wb_bf16, - int seq_len, - int in_dim, - int a_dim, - int b_dim); + float *b, + const float *x, + const uint16_t *Wa_bf16, + const uint16_t *Wb_bf16, + int seq_len, + int in_dim, + int a_dim, + int b_dim); /* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ void linear_nobias_bf16_qkv(float *q, - float *k, - float *v, - const float *x, - const uint16_t *Wq_bf16, - const uint16_t *Wk_bf16, - const uint16_t *Wv_bf16, - int seq_len, - int in_dim, - int q_dim, - int kv_dim); + float *k, + float *v, + const float *x, + const uint16_t *Wq_bf16, + const uint16_t *Wk_bf16, + const uint16_t *Wv_bf16, + int seq_len, + int in_dim, + int q_dim, + int kv_dim); /* Dot product using the best available local SIMD implementation. */ float dot_f32(const float *a, const float *b, int n); @@ -68,8 +67,7 @@ float dot_f32(const float *a, const float *b, int n); * ======================================================================== */ /* RMS Normalization: out = x / rms(x) * weight */ -void rms_norm( - float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); +void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); /* Per-head RMS Normalization for Q/K norms in decoder * x: [seq, n_heads, head_dim], weight: [head_dim] @@ -81,12 +79,12 @@ void rms_norm_per_head( * + beta, with mean and biased (population) variance over the hidden axis per * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ void layer_norm(float *out, - const float *x, - const float *gamma, - const float *beta, - int seq_len, - int hidden, - float eps); + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps); /* ======================================================================== * Activation Functions @@ -119,57 +117,57 @@ void gelu_tanh_inplace(float *x, int n); * offsets[b]..offsets[b+1] span. */ void bidirectional_gqa_attention_packed(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale); + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale); /* Optional reusable scratch for the BLAS-tiled packed attention path. */ size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes); + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); /* * Causal GQA attention over a packed/ragged batch. Query position i attends * only to positions at or before i within its sequence. */ void causal_gqa_attention_packed(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale); + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale); void causal_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes); + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); /* ======================================================================== * Position Embeddings diff --git a/libffwd/late.c b/libffwd/late.c index 05d0341..e086ad3 100644 --- a/libffwd/late.c +++ b/libffwd/late.c @@ -105,9 +105,7 @@ const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model) { return model && model->base ? &model->base->config : NULL; } -int ffwd_late_model_token_dim(const ffwd_late_model_t *model) { - return model ? model->token_dim : 0; -} +int ffwd_late_model_token_dim(const ffwd_late_model_t *model) { return model ? model->token_dim : 0; } /* Internal (internal.h): expose the base model and projection so GPU * backends can upload them to device without seeing struct ffwd_late_model. */ @@ -140,12 +138,12 @@ float ffwd_late_maxsim( } int ffwd_late_maxsim_batch(const float *query_vectors, - int query_tokens, - const float *doc_vectors, - const int *doc_offsets, - int docs, - int dim, - float *scores) { + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores) { if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0) return -1; @@ -211,8 +209,8 @@ int ffwd_late_maxsim_batch(const float *query_vectors, for (int i = 0; i < docs; i++) { int start = doc_offsets[i]; int end = doc_offsets[i + 1]; - scores[i] = ffwd_late_maxsim(query_vectors, query_tokens, - doc_vectors + (size_t)start * dim, end - start, dim); + scores[i] = ffwd_late_maxsim(query_vectors, query_tokens, doc_vectors + (size_t)start * dim, + end - start, dim); } return 0; } diff --git a/libffwd/late_blas.c b/libffwd/late_blas.c index cc57596..dabeb58 100644 --- a/libffwd/late_blas.c +++ b/libffwd/late_blas.c @@ -55,11 +55,11 @@ void ffwd_late_workspace_free(ffwd_late_workspace_t *ws) { } int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model, - ffwd_late_workspace_t *ws, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors) { + ffwd_late_workspace_t *ws, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { int dim = ffwd_late_model_token_dim(model); ffwd_model_t *base = ffwd_late_model_base(model); const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model); @@ -86,15 +86,15 @@ int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model, /* Late-interaction batch encoder. Reuses the base model's packed/ragged batch * forward pass, then gathers kept rows and projects them in one GEMM. */ int ffwd_late_model_encode_docs(const ffwd_late_model_t *model, - ffwd_late_workspace_t *ws, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - float *out_vectors, - int *out_offsets) { + ffwd_late_workspace_t *ws, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets) { int dim = ffwd_late_model_token_dim(model); ffwd_model_t *base = ffwd_late_model_base(model); const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model); diff --git a/libffwd/mlx.c b/libffwd/mlx.c index 10299c1..0b6d8b7 100644 --- a/libffwd/mlx.c +++ b/libffwd/mlx.c @@ -327,9 +327,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) { return res; } -const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx) { - return ctx ? &ctx->config : NULL; -} +const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx) { return ctx ? &ctx->config : NULL; } /* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the @@ -432,8 +430,7 @@ static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); ctx->ffwd_ln_b = load_tensor(ctx->ms, name, ln_shape, 1); if (!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->position_embeddings) || - !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->ffwd_ln_w) || - !arr_ok(ctx->ffwd_ln_b)) + !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->ffwd_ln_w) || !arr_ok(ctx->ffwd_ln_b)) return -1; } @@ -483,10 +480,10 @@ static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer } static ffwd_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, - int layer_start, - int layer_end, - const ffwd_mlx_options_t *opts, - int allow_late) { + int layer_start, + int layer_end, + const ffwd_mlx_options_t *opts, + int allow_late) { ffwd_mlx_options_t options = normalize_mlx_options(opts); if (options.quantize_bits < 0) return NULL; @@ -703,8 +700,7 @@ ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir) { return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); } -ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, - const ffwd_mlx_options_t *opts) { +ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, const ffwd_mlx_options_t *opts) { return mlx_load_range_ex(model_dir, 0, -1, opts, 0); } @@ -756,7 +752,7 @@ void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx) { } ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir, - const ffwd_mlx_options_t *opts) { + const ffwd_mlx_options_t *opts) { if (!mlx_model_dir_has_late_projection(model_dir)) { fprintf(stderr, "mlx late: missing 1_Dense projection\n"); return NULL; @@ -990,7 +986,7 @@ static mlx_array mlx_forward_bert(ffwd_mlx_ctx_t *ctx, mlx_add(&mid_b, mid, l->ffn_inter_bias, S); mlx_array_free(mid); mlx_array g = ctx->config.ffn_act == FFWD_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S) - : mlx_gelu_erf(mid_b, S); + : mlx_gelu_erf(mid_b, S); mlx_array_free(mid_b); mlx_array out = linear(g, &l->down_proj, S); mlx_array_free(g); @@ -1171,9 +1167,9 @@ static mlx_array mlx_forward_layers(ffwd_mlx_ctx_t *ctx, } ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize) { + const int *token_ids, + int n_tokens, + int normalize) { if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection)) return NULL; @@ -1302,9 +1298,7 @@ int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs) { return vecs ? vecs->tokens : 0; } -int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs) { - return vecs ? vecs->dim : 0; -} +int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs) { return vecs ? vecs->dim : 0; } int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors) { if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0) @@ -1351,13 +1345,13 @@ int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_v * within MLX's batched-GEMM rounding. */ ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - int *out_offsets) { + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + int *out_offsets) { if (!ctx || !ctx->base || !doc_ids || !n_tokens || !keep || !n_keep || !out_offsets || n_docs <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection)) return NULL; @@ -1577,8 +1571,9 @@ cleanup: * candidates through ffwd_mlx_late_encode_docs_device, but the late-interaction * verification harness (tests/integration/check_late_interaction.py) drives these directly * to validate per-document gather and concat against the CPU MaxSim reference. */ -ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat( - ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count) { +ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(ffwd_mlx_late_ctx_t *ctx, + const ffwd_mlx_late_vectors_t *const *items, + int count) { if (!ctx || !ctx->base || !items || count <= 0) return NULL; int dim = 0; @@ -1627,9 +1622,9 @@ ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat( } ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, - const ffwd_mlx_late_vectors_t *vecs, - const int *token_indices, - int count) { + const ffwd_mlx_late_vectors_t *vecs, + const int *token_indices, + int count) { if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) || !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0) return NULL; @@ -1675,11 +1670,11 @@ ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, } int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx, - const ffwd_mlx_late_vectors_t *query, - const ffwd_mlx_late_vectors_t *docs, - const int *doc_offsets, - int docs_count, - float *scores) { + const ffwd_mlx_late_vectors_t *query, + const ffwd_mlx_late_vectors_t *docs, + const int *doc_offsets, + int docs_count, + float *scores) { if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores || query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim || query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) || @@ -1882,11 +1877,8 @@ cleanup: return rc; } -int ffwd_mlx_late_encode_tokens(ffwd_mlx_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors) { +int ffwd_mlx_late_encode_tokens( + ffwd_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, float *out_vectors) { ffwd_mlx_late_vectors_t *vecs = ffwd_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize); if (!vecs) @@ -1897,9 +1889,9 @@ int ffwd_mlx_late_encode_tokens(ffwd_mlx_late_ctx_t *ctx, } static int ffwd_mlx_encode_batch_dense(ffwd_mlx_ctx_t *ctx, - const ffwd_input_t *inputs, - int batch, - float *out_embeddings) { + const ffwd_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) return -1; @@ -2208,10 +2200,8 @@ static int mlx_batch_should_split(const ffwd_input_t *inputs, return mlx_padding_exceeds_limit(padded, (size_t)total_tokens); } -static int mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx, - int group_count, - int group_max, - int next_tokens) { +static int +mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx, int group_count, int group_max, int next_tokens) { if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0) return 1; @@ -2231,9 +2221,9 @@ static int mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx, } int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx, - const ffwd_input_t *inputs, - int batch, - float *out_embeddings) { + const ffwd_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; @@ -2306,9 +2296,9 @@ int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx, } int ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, - const int *token_ids, - int n_tokens, - float *out_embedding) { + const int *token_ids, + int n_tokens, + float *out_embedding) { ffwd_input_t input = {token_ids, n_tokens}; return ffwd_mlx_encode_batch(ctx, &input, 1, out_embedding); } @@ -2333,9 +2323,8 @@ static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) { return padded * 4 > tokens * 5; } -static int mlx_context_batch_should_split(const ffwd_context_input_t *inputs, - int batch, - int *out_total_spans) { +static int +mlx_context_batch_should_split(const ffwd_context_input_t *inputs, int batch, int *out_total_spans) { int max_seq = 0; int total_tokens = 0; int total_spans = 0; @@ -2362,9 +2351,9 @@ static int mlx_context_batch_should_split(const ffwd_context_input_t *inputs, } static int ffwd_mlx_encode_spans_batch_dense(ffwd_mlx_ctx_t *ctx, - const ffwd_context_input_t *inputs, - int batch, - float *out_embeddings) { + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) return -1; @@ -2663,9 +2652,9 @@ cleanup: } int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx, - const ffwd_context_input_t *inputs, - int batch, - float *out_embeddings) { + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; @@ -2748,11 +2737,11 @@ int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx, } int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx, - const int *token_ids, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings) { + const int *token_ids, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings) { ffwd_context_input_t input = { .input = {token_ids, n_tokens}, .spans = spans, diff --git a/libffwd/mlx.h b/libffwd/mlx.h index d8a915f..e4e2a8e 100644 --- a/libffwd/mlx.h +++ b/libffwd/mlx.h @@ -38,7 +38,7 @@ typedef struct { */ FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir); FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, - const ffwd_mlx_options_t *opts); + const ffwd_mlx_options_t *opts); FFWD_API void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx); @@ -49,7 +49,7 @@ FFWD_API void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx); */ FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load(const char *model_dir); FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir, - const ffwd_mlx_options_t *opts); + const ffwd_mlx_options_t *opts); FFWD_API void ffwd_mlx_late_free(ffwd_mlx_late_ctx_t *ctx); FFWD_API const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx); @@ -72,9 +72,9 @@ FFWD_API int ffwd_mlx_late_encode_tokens( * and copy back only final candidate scores. */ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize); + const int *token_ids, + int n_tokens, + int normalize); FFWD_API void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs); @@ -87,29 +87,28 @@ FFWD_API int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, flo * Returns NULL on error; the handle must not outlive ctx. */ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - int *out_offsets); + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + int *out_offsets); /* Device-resident select/concat primitives, retained for the late-interaction * verification harness (tests/integration/check_late_interaction.py); the server rerank * path uses ffwd_mlx_late_encode_docs_device. */ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat( ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count); -FFWD_API ffwd_mlx_late_vectors_t * -ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, - const ffwd_mlx_late_vectors_t *vecs, - const int *token_indices, - int count); +FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, + const ffwd_mlx_late_vectors_t *vecs, + const int *token_indices, + int count); FFWD_API int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx, - const ffwd_mlx_late_vectors_t *query, - const ffwd_mlx_late_vectors_t *docs, - const int *doc_offsets, - int docs_count, - float *scores); + const ffwd_mlx_late_vectors_t *query, + const ffwd_mlx_late_vectors_t *docs, + const int *doc_offsets, + int docs_count, + float *scores); /* * Compute embedding for token_ids[0..n_tokens-1]. @@ -132,29 +131,29 @@ ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, fl * Returns 0 on success, -1 on error. */ FFWD_API int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx, - const ffwd_input_t *inputs, - int batch, - float *out_embeddings); + const ffwd_input_t *inputs, + int batch, + float *out_embeddings); /* * Run one contextual sequence and pool selected token spans. * Returns 0 on success, -1 on error. */ FFWD_API int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx, - const int *token_ids, - int n_tokens, - const ffwd_span_t *spans, - int n_spans, - float *out_embeddings); + const int *token_ids, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings); /* * Run a padded dense contextual document batch and pool every selected span. * out_embeddings contains spans in document order. */ FFWD_API int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx, - const ffwd_context_input_t *inputs, - int batch, - float *out_embeddings); + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings); /* Get the config (so ffwd_cli.c can read hidden_size etc.) */ FFWD_API const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx); diff --git a/libffwd/model_types.h b/libffwd/model_types.h index 2719a5b..345e0cd 100644 --- a/libffwd/model_types.h +++ b/libffwd/model_types.h @@ -26,13 +26,13 @@ typedef struct { ffwd_weight_ref_t wk; /* [kv_dim, hidden] */ ffwd_weight_ref_t wv; /* [kv_dim, hidden] */ ffwd_weight_ref_t wo; /* [hidden, q_dim] */ - const float *q_bias; /* [q_dim], optional */ - const float *k_bias; /* [kv_dim], optional */ - const float *v_bias; /* [kv_dim], optional */ - const float *q_norm; /* [head_dim] */ - const float *k_norm; /* [head_dim] */ - const float *input_norm; /* [hidden] */ - const float *post_attn_norm; /* [hidden] */ + const float *q_bias; /* [q_dim], optional */ + const float *k_bias; /* [kv_dim], optional */ + const float *v_bias; /* [kv_dim], optional */ + const float *q_norm; /* [head_dim] */ + const float *k_norm; /* [head_dim] */ + const float *input_norm; /* [hidden] */ + const float *post_attn_norm; /* [hidden] */ ffwd_weight_ref_t gate_proj; /* [intermediate, hidden] */ ffwd_weight_ref_t up_proj; /* [intermediate, hidden] */ ffwd_weight_ref_t down_proj; /* [hidden, intermediate] */ @@ -51,27 +51,27 @@ typedef struct { typedef struct { ffwd_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ ffwd_layer_t *layers; /* [n_layers] heap-allocated */ - const float *norm; /* [hidden]; NULL for BERT (no final norm) */ + const float *norm; /* [hidden]; NULL for BERT (no final norm) */ /* BERT family (NULL for Qwen3): learned absolute position embeddings, * token-type (segment) embeddings, and the embedding LayerNorm. */ const float *position_embeddings; /* [max_position_embeddings, hidden] */ const float *token_type_embeddings; /* [type_vocab_size, hidden] */ - const float *ffwd_ln_w; /* embedding LayerNorm weight [hidden] */ - const float *ffwd_ln_b; /* embedding LayerNorm bias [hidden] */ + const float *ffwd_ln_w; /* embedding LayerNorm weight [hidden] */ + const float *ffwd_ln_b; /* embedding LayerNorm bias [hidden] */ } ffwd_weights_t; typedef void (*ffwd_attention_fn)(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes); + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); struct ffwd_model { ffwd_config_t config; diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c index f7a50f7..df5861e 100644 --- a/libffwd/tokenize.c +++ b/libffwd/tokenize.c @@ -189,9 +189,7 @@ void ffwd_tok_free(ffwd_tok_t *t) { free(t); } -int ffwd_tok_context_separator_id(const ffwd_tok_t *t) { - return t ? t->context_separator_id : -1; -} +int ffwd_tok_context_separator_id(const ffwd_tok_t *t) { return t ? t->context_separator_id : -1; } void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on) { if (t) @@ -289,10 +287,7 @@ static int late_id_is_skipped(const ffwd_tok_t *t, int id) { return 0; } -int ffwd_tokenize_late(ffwd_tok_t *t, - const char *text, - int is_query, - ffwd_late_tokens_t *out) { +int ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out) { memset(out, 0, sizeof(*out)); if (!t || !text) return -1; diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h index 66c62a0..c93140a 100644 --- a/libffwd/wordpiece.h +++ b/libffwd/wordpiece.h @@ -52,18 +52,18 @@ void tok_wp_workspace_free(tok_wp_workspace_t *ws); * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to * the number of tokens needed/emitted in all cases. */ int tok_wp_encode_into(const tok_wp_t *tok, - tok_wp_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens); + tok_wp_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); /* Same tokenization as tok_wp_encode(), reusing ws internally and * performing only the final returned int[] allocation. */ int *tok_wp_encode_with_workspace(const tok_wp_t *tok, - tok_wp_workspace_t *ws, - const char *text, - int *out_n_tokens); + tok_wp_workspace_t *ws, + const char *text, + int *out_n_tokens); void tok_wp_free(tok_wp_t *tok); -- cgit v1.3.1