aboutsummaryrefslogtreecommitdiff
path: root/libffwd/ffwd.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
commitcf7d8169ec12cb3adf90d4ff78b3257e74985302 (patch)
treef666589acfd68e76bbde4e74eead44ae7819d0bb /libffwd/ffwd.h
parent91871be6ece9317c5d49353f6474583e7e28168d (diff)
downloadlangdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.tar.gz
langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.zip
Remove unused model families and inference paths
Diffstat (limited to 'libffwd/ffwd.h')
-rw-r--r--libffwd/ffwd.h284
1 files changed, 26 insertions, 258 deletions
diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h
index c4708fa..527e92f 100644
--- a/libffwd/ffwd.h
+++ b/libffwd/ffwd.h
@@ -1,281 +1,49 @@
#ifndef FFWD_H
#define FFWD_H
-/* The libraries are built with -fvisibility=hidden, so only declarations carrying
- * FFWD_API are exported from the shared library; everything else stays internal. */
-#ifndef FFWD_API
-# if defined(__GNUC__)
-# define FFWD_API __attribute__((visibility("default")))
-# else
-# define FFWD_API
-# endif
-#endif
-
#include <stddef.h>
-#include <stdint.h>
-
-/* Shared constants and defaults */
-
-#define FFWD_VOCAB_SIZE 151936
-#define FFWD_HEAD_DIM 128
-#define FFWD_MAX_LAYERS 64 /* upper bound for stack arrays */
-#define FFWD_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */
-
-/* Late-interaction (ColBERT) fallback special-token ids and query pad length.
- * Used when the model tokenizer does not define these tokens itself. */
-#define FFWD_LATE_QUERY_TOKENS 32
-#define FFWD_LATE_MASK_TOKEN_ID 151642
-#define FFWD_LATE_QUERY_PREFIX_ID 151669
-#define FFWD_LATE_DOCUMENT_PREFIX_ID 151670
-
-/* Model Configuration (populated from config.json at load time) */
-
-typedef enum {
- FFWD_ATTENTION_BIDIRECTIONAL = 0,
- FFWD_ATTENTION_CAUSAL = 1,
-} ffwd_attention_mode_t;
-
-typedef enum {
- FFWD_POOL_MEAN = 0,
- FFWD_POOL_LAST_TOKEN = 1,
- FFWD_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */
-} ffwd_pooling_mode_t;
-
-typedef enum {
- FFWD_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (Qwen2/3, pplx-embed) */
- FFWD_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */
-} ffwd_family_t;
-/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf
- * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or
- * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */
-typedef enum {
- FFWD_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */
- FFWD_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */
-} ffwd_act_t;
+#define FFWD_MAX_LAYERS 64
typedef struct {
- ffwd_family_t family;
- ffwd_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */
int hidden_size;
int n_layers;
int n_heads;
- int n_kv_heads;
int head_dim;
- int q_dim; /* n_heads * head_dim */
- int kv_dim; /* n_kv_heads * head_dim */
int intermediate_size;
int vocab_size;
- int max_position_embeddings; /* BERT learned position table size */
- int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */
- int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */
- float rms_norm_eps;
- float layer_norm_eps; /* BERT LayerNorm epsilon */
- float rope_theta;
- int qk_norm;
- int qkv_bias;
- ffwd_attention_mode_t attention_mode;
- ffwd_pooling_mode_t pooling_mode;
- int normalize_embeddings;
- /* Qwen3-Embedding pools the last token: the tokenizer's <|endoftext|> suffix
- * added by its post-processor, not the model's chat eos_token_id (<|im_end|>).
- *
- * Text frontends append it and resolve its id from the tokenizer, just like the
- * contextual separator. FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model
- * fallback.
- *
- * Model forward consumes token ids exactly and never appends. */
- int append_terminal_token;
+ int max_position_embeddings;
+ int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
+ int type_vocab_size;
+ float layer_norm_eps;
} ffwd_config_t;
-typedef struct {
- const int *ids;
- int n_tokens;
-} ffwd_input_t;
-
-typedef struct {
- int start;
- int n_tokens;
-} ffwd_span_t;
-
-typedef struct {
- ffwd_input_t input;
- const ffwd_span_t *spans;
- int n_spans;
-} ffwd_context_input_t;
-
-/*
- * Backend-selected inference handle.
- *
- * This is the library's main public interface and the full supported surface of
- * the shared library. Exactly one backend is linked at build time: CPU, Apple Metal,
- * or NVIDIA CUDA. That backend implements these functions, so callers never branch
- * on backend.
- *
- * The CLI, server, and language bindings use only these functions plus the math
- * helpers below.
- *
- * Model loading has two phases so a GPU context can be created on the thread
- * that owns it.
- *
- * ffwd_open runs on the caller's thread. It loads a CPU model immediately; in a
- * GPU build, it only records what to load.
- *
- * ffwd_activate runs on the inference thread and creates the GPU context. It is
- * a no-op for CPU.
- *
- * Teardown mirrors activation: ffwd_worker_free releases the GPU context from
- * the inference thread, and ffwd_free releases the rest.
- */
-
-/* Opaque inference handle; layout is private to the linked backend. */
-typedef struct embed ffwd_t;
-
-/* Backend tuning, parsed from the command line. A backend reads the fields it
- * supports and ignores the rest, so the front-ends parse them with no backend
- * branch. */
-typedef struct {
- int n_threads; /* CPU worker threads; <= 0 auto-detects */
- int gpu_quant_bits; /* GPU load-time weight quantization (0 or 8) */
- int gpu_quant_group_size; /* GPU quantization group size (> 0) */
- const char *gpu_gemm_mode; /* GPU GEMM compute mode, or NULL */
- const char *gpu_weight_dtype; /* GPU weight storage dtype, or NULL */
- double memory_utilization; /* GPU memory budget fraction; 0 = default */
-} ffwd_options_t;
-
-/* One late-interaction rerank: a query and N candidate documents, each given as
- * token ids plus the kept-token indices. The vector dimension is the loaded
- * late model's token_dim. */
-typedef struct {
- const int *query_ids;
- int query_n_tokens;
- int query_n_keep;
- const int *const *doc_ids;
- const int *doc_n_tokens;
- const int *const *doc_keep;
- const int *doc_n_keep;
- int n_docs;
-} ffwd_rerank_input_t;
-
-/* Apply process-wide settings before any load (CPU thread count, GPU GEMM and
- * weight settings, GPU option checks) and print the startup banner. Returns -1 and
- * writes a reason into err on a rejected option value. */
-FFWD_API int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen);
-
-/* First load phase, on the caller's thread: load a CPU model now, or record what
- * a GPU build will load in ffwd_activate. is_late selects a late-interaction
- * model. NULL on failure with a reason in err. */
-FFWD_API ffwd_t *
-ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen);
-/* Second load phase, on the inference thread: create the GPU context. CPU
- * no-op. Returns -1 with a reason in err on failure. */
-FFWD_API int ffwd_activate(ffwd_t *e, char *err, size_t errlen);
-/* Release the inference-thread-owned GPU context; CPU no-op. Idempotent. */
-FFWD_API void ffwd_worker_free(ffwd_t *e);
-/* Release CPU state and the handle; call after ffwd_worker_free. */
-FFWD_API void ffwd_free(ffwd_t *e);
-
-/* Available once the model is loaded (after open for CPU, after activate for
- * GPU); config is NULL until then. */
-FFWD_API const ffwd_config_t *ffwd_config(const ffwd_t *e);
-/* Late projection dimension, or 0 for a standard model. */
-FFWD_API int ffwd_token_dim(const ffwd_t *e);
-/* Whether the backend pads batches to a common length rather than packing them. */
-FFWD_API int ffwd_uses_dense_batches(const ffwd_t *e);
-
-/* Encode a packed batch into out[batch, hidden_size]. Returns 0 on success. */
-FFWD_API int ffwd_encode_batch(ffwd_t *e, const ffwd_input_t *inputs, int batch, float *out);
-/* Encode a contextual document batch, pooling each selected span in order. */
-FFWD_API int
-ffwd_encode_spans_batch(ffwd_t *e, const ffwd_context_input_t *inputs, int batch, float *out);
-/* Score one query against its candidate documents into scores[n_docs]. */
-FFWD_API int ffwd_rerank(ffwd_t *e, const ffwd_rerank_input_t *in, float *scores);
-/* Scratch memory the model currently holds, for diagnostics; 0 on GPU. */
-FFWD_API size_t ffwd_scratch_nbytes(const ffwd_t *e);
-
-/* Accelerator label for --build-info and the startup banner: "Apple Metal",
- * "NVIDIA CUDA", or "" on a CPU build. A function, not a macro, so a shared
- * object does not bake in one backend's label and stays correct across a
- * rebuild. */
-FFWD_API const char *ffwd_capability(void);
-
-/* First-arrival micro-batch wait the backend prefers, in microseconds. */
-FFWD_API int ffwd_default_batch_wait_us(void);
-/* Reject a model set that would not fit host memory (GPU backends); 0 otherwise. On
- * rejection returns -1 with a reason in err. paths are the model directories. */
-FFWD_API int ffwd_preflight(
- const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen);
-
-/*
- * L2-normalize vec[dim] in place.
- *
- * Returns 0 on success. Returns -1 for invalid arguments or a zero-length
- * vector, which cannot be normalized.
- */
-FFWD_API int ffwd_l2_normalize(float *vec, int dim);
-
-/*
- * Cosine similarity between two vectors.
- * Returns 0 for invalid arguments or if either vector has zero length.
- */
-FFWD_API float ffwd_cosine_similarity(const float *a, const float *b, int dim);
-
-/*
- * Tokenization: text -> token ids the model expects.
- *
- * Tokenization is backend-independent because text -> ids is the same on CPU
- * and GPU, so it has its own handle, separate from ffwd_t.
- *
- * ffwd_tok_open selects the tokenizer from files in model_dir: WordPiece
- * vocab.txt, SentencePiece .model, or byte-level vocab.json. It also resolves
- * the model's special-token ids.
- *
- * ffwd_tokenize applies the model's token layout: BERT and XLM-R wrap ids with
- * [CLS]..[SEP] (<s>..</s>), while Qwen appends its terminal token.
- */
-
-/* Opaque tokenizer handle; owns the tokenizer and its resolved special tokens. */
+typedef struct ffwd_model ffwd_model_t;
+typedef struct ffwd_workspace ffwd_workspace_t;
typedef struct ffwd_tok ffwd_tok_t;
-/* Late-interaction tokens:
- * token ids plus the document positions to score.
- * For queries, keep is NULL because every position is kept, and
- * n_keep == n_tokens. */
-typedef struct {
- int *ids;
- int n_tokens;
- int *keep;
- int n_keep;
-} ffwd_late_tokens_t;
-
-/* Load the tokenizer for the model in model_dir and resolve its special-token
- * ids. is_late additionally resolves the late-interaction [Q]/[D]/[MASK] ids.
- * NULL on failure with a reason in err. */
-FFWD_API ffwd_tok_t *ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen);
-FFWD_API void ffwd_tok_free(ffwd_tok_t *t);
-
-/* Tokenize text for this model: applies the model's special-token layout and
- * optionally prepends a query instruction. Returns a malloc'd id array (caller
- * frees) and sets *n_out. NULL on error. */
-FFWD_API int *ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out);
+/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
+ffwd_model_t *ffwd_model_load(const char *model_dir);
+void ffwd_model_free(ffwd_model_t *model);
+const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model);
-/* Late-interaction tokenization: [Q]/[D] prefix, [MASK]-pad queries to a fixed
- * length, and mark the document positions to score. Returns 0 on success. */
-FFWD_API int
-ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out);
-FFWD_API void ffwd_late_tokens_free(ffwd_late_tokens_t *out);
+/* A workspace belongs to one model and may serve only one inference call at a time. */
+ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model);
+void ffwd_workspace_free(ffwd_workspace_t *ws);
-/* The contextual chunk-separator id (the tokenizer's <|endoftext|>), for
- * frontends that split a document into chunks. */
-FFWD_API int ffwd_tok_context_separator_id(const ffwd_tok_t *t);
+/* Write the first token's final hidden state to out_embedding[hidden_size].
+ * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
+int ffwd_model_encode_into(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
-/* Whether ffwd_tokenize appends the model's terminal token (<|endoftext|>) for
- * last-token pooling. Off until set; the caller sets it from the loaded model's
- * config (ffwd_config(...)->append_terminal_token), since it is model-specific
- * and only known after the backend is open. */
-FFWD_API void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on);
+ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen);
+void ffwd_tok_free(ffwd_tok_t *t);
-/* Verbose level: 0=quiet, 1=info, 2=debug */
-FFWD_API extern int ffwd_verbose;
+/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
+ * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
+int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out);
#endif /* FFWD_H */