diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:54:13 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:54:13 +0200 |
| commit | 91871be6ece9317c5d49353f6474583e7e28168d (patch) | |
| tree | 8cabf2546322a9e1b724bf8413f32de54378824f /libffwd/ffwd.h | |
| parent | 35ab9c12ba4d396e6cba99d6ea947e26becedad4 (diff) | |
| download | langdet-91871be6ece9317c5d49353f6474583e7e28168d.tar.gz langdet-91871be6ece9317c5d49353f6474583e7e28168d.zip | |
Add native CPU language detection CLI
Diffstat (limited to 'libffwd/ffwd.h')
| -rw-r--r-- | libffwd/ffwd.h | 281 |
1 files changed, 281 insertions, 0 deletions
diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h new file mode 100644 index 0000000..c4708fa --- /dev/null +++ b/libffwd/ffwd.h @@ -0,0 +1,281 @@ +#ifndef FFWD_H +#define FFWD_H + +/* The libraries are built with -fvisibility=hidden, so only declarations carrying + * FFWD_API are exported from the shared library; everything else stays internal. */ +#ifndef FFWD_API +# if defined(__GNUC__) +# define FFWD_API __attribute__((visibility("default"))) +# else +# define FFWD_API +# endif +#endif + +#include <stddef.h> +#include <stdint.h> + +/* Shared constants and defaults */ + +#define FFWD_VOCAB_SIZE 151936 +#define FFWD_HEAD_DIM 128 +#define FFWD_MAX_LAYERS 64 /* upper bound for stack arrays */ +#define FFWD_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ + +/* Late-interaction (ColBERT) fallback special-token ids and query pad length. + * Used when the model tokenizer does not define these tokens itself. */ +#define FFWD_LATE_QUERY_TOKENS 32 +#define FFWD_LATE_MASK_TOKEN_ID 151642 +#define FFWD_LATE_QUERY_PREFIX_ID 151669 +#define FFWD_LATE_DOCUMENT_PREFIX_ID 151670 + +/* Model Configuration (populated from config.json at load time) */ + +typedef enum { + FFWD_ATTENTION_BIDIRECTIONAL = 0, + FFWD_ATTENTION_CAUSAL = 1, +} ffwd_attention_mode_t; + +typedef enum { + FFWD_POOL_MEAN = 0, + FFWD_POOL_LAST_TOKEN = 1, + FFWD_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */ +} ffwd_pooling_mode_t; + +typedef enum { + FFWD_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (Qwen2/3, pplx-embed) */ + FFWD_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */ +} ffwd_family_t; + +/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf + * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or + * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */ +typedef enum { + FFWD_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */ + FFWD_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */ +} ffwd_act_t; + +typedef struct { + ffwd_family_t family; + ffwd_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */ + int hidden_size; + int n_layers; + int n_heads; + int n_kv_heads; + int head_dim; + int q_dim; /* n_heads * head_dim */ + int kv_dim; /* n_kv_heads * head_dim */ + int intermediate_size; + int vocab_size; + int max_position_embeddings; /* BERT learned position table size */ + int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */ + int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */ + float rms_norm_eps; + float layer_norm_eps; /* BERT LayerNorm epsilon */ + float rope_theta; + int qk_norm; + int qkv_bias; + ffwd_attention_mode_t attention_mode; + ffwd_pooling_mode_t pooling_mode; + int normalize_embeddings; + /* Qwen3-Embedding pools the last token: the tokenizer's <|endoftext|> suffix + * added by its post-processor, not the model's chat eos_token_id (<|im_end|>). + * + * Text frontends append it and resolve its id from the tokenizer, just like the + * contextual separator. FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model + * fallback. + * + * Model forward consumes token ids exactly and never appends. */ + int append_terminal_token; +} ffwd_config_t; + +typedef struct { + const int *ids; + int n_tokens; +} ffwd_input_t; + +typedef struct { + int start; + int n_tokens; +} ffwd_span_t; + +typedef struct { + ffwd_input_t input; + const ffwd_span_t *spans; + int n_spans; +} ffwd_context_input_t; + +/* + * Backend-selected inference handle. + * + * This is the library's main public interface and the full supported surface of + * the shared library. Exactly one backend is linked at build time: CPU, Apple Metal, + * or NVIDIA CUDA. That backend implements these functions, so callers never branch + * on backend. + * + * The CLI, server, and language bindings use only these functions plus the math + * helpers below. + * + * Model loading has two phases so a GPU context can be created on the thread + * that owns it. + * + * ffwd_open runs on the caller's thread. It loads a CPU model immediately; in a + * GPU build, it only records what to load. + * + * ffwd_activate runs on the inference thread and creates the GPU context. It is + * a no-op for CPU. + * + * Teardown mirrors activation: ffwd_worker_free releases the GPU context from + * the inference thread, and ffwd_free releases the rest. + */ + +/* Opaque inference handle; layout is private to the linked backend. */ +typedef struct embed ffwd_t; + +/* Backend tuning, parsed from the command line. A backend reads the fields it + * supports and ignores the rest, so the front-ends parse them with no backend + * branch. */ +typedef struct { + int n_threads; /* CPU worker threads; <= 0 auto-detects */ + int gpu_quant_bits; /* GPU load-time weight quantization (0 or 8) */ + int gpu_quant_group_size; /* GPU quantization group size (> 0) */ + const char *gpu_gemm_mode; /* GPU GEMM compute mode, or NULL */ + const char *gpu_weight_dtype; /* GPU weight storage dtype, or NULL */ + double memory_utilization; /* GPU memory budget fraction; 0 = default */ +} ffwd_options_t; + +/* One late-interaction rerank: a query and N candidate documents, each given as + * token ids plus the kept-token indices. The vector dimension is the loaded + * late model's token_dim. */ +typedef struct { + const int *query_ids; + int query_n_tokens; + int query_n_keep; + const int *const *doc_ids; + const int *doc_n_tokens; + const int *const *doc_keep; + const int *doc_n_keep; + int n_docs; +} ffwd_rerank_input_t; + +/* Apply process-wide settings before any load (CPU thread count, GPU GEMM and + * weight settings, GPU option checks) and print the startup banner. Returns -1 and + * writes a reason into err on a rejected option value. */ +FFWD_API int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen); + +/* First load phase, on the caller's thread: load a CPU model now, or record what + * a GPU build will load in ffwd_activate. is_late selects a late-interaction + * model. NULL on failure with a reason in err. */ +FFWD_API ffwd_t * +ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen); +/* Second load phase, on the inference thread: create the GPU context. CPU + * no-op. Returns -1 with a reason in err on failure. */ +FFWD_API int ffwd_activate(ffwd_t *e, char *err, size_t errlen); +/* Release the inference-thread-owned GPU context; CPU no-op. Idempotent. */ +FFWD_API void ffwd_worker_free(ffwd_t *e); +/* Release CPU state and the handle; call after ffwd_worker_free. */ +FFWD_API void ffwd_free(ffwd_t *e); + +/* Available once the model is loaded (after open for CPU, after activate for + * GPU); config is NULL until then. */ +FFWD_API const ffwd_config_t *ffwd_config(const ffwd_t *e); +/* Late projection dimension, or 0 for a standard model. */ +FFWD_API int ffwd_token_dim(const ffwd_t *e); +/* Whether the backend pads batches to a common length rather than packing them. */ +FFWD_API int ffwd_uses_dense_batches(const ffwd_t *e); + +/* Encode a packed batch into out[batch, hidden_size]. Returns 0 on success. */ +FFWD_API int ffwd_encode_batch(ffwd_t *e, const ffwd_input_t *inputs, int batch, float *out); +/* Encode a contextual document batch, pooling each selected span in order. */ +FFWD_API int +ffwd_encode_spans_batch(ffwd_t *e, const ffwd_context_input_t *inputs, int batch, float *out); +/* Score one query against its candidate documents into scores[n_docs]. */ +FFWD_API int ffwd_rerank(ffwd_t *e, const ffwd_rerank_input_t *in, float *scores); +/* Scratch memory the model currently holds, for diagnostics; 0 on GPU. */ +FFWD_API size_t ffwd_scratch_nbytes(const ffwd_t *e); + +/* Accelerator label for --build-info and the startup banner: "Apple Metal", + * "NVIDIA CUDA", or "" on a CPU build. A function, not a macro, so a shared + * object does not bake in one backend's label and stays correct across a + * rebuild. */ +FFWD_API const char *ffwd_capability(void); + +/* First-arrival micro-batch wait the backend prefers, in microseconds. */ +FFWD_API int ffwd_default_batch_wait_us(void); +/* Reject a model set that would not fit host memory (GPU backends); 0 otherwise. On + * rejection returns -1 with a reason in err. paths are the model directories. */ +FFWD_API int ffwd_preflight( + const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen); + +/* + * L2-normalize vec[dim] in place. + * + * Returns 0 on success. Returns -1 for invalid arguments or a zero-length + * vector, which cannot be normalized. + */ +FFWD_API int ffwd_l2_normalize(float *vec, int dim); + +/* + * Cosine similarity between two vectors. + * Returns 0 for invalid arguments or if either vector has zero length. + */ +FFWD_API float ffwd_cosine_similarity(const float *a, const float *b, int dim); + +/* + * Tokenization: text -> token ids the model expects. + * + * Tokenization is backend-independent because text -> ids is the same on CPU + * and GPU, so it has its own handle, separate from ffwd_t. + * + * ffwd_tok_open selects the tokenizer from files in model_dir: WordPiece + * vocab.txt, SentencePiece .model, or byte-level vocab.json. It also resolves + * the model's special-token ids. + * + * ffwd_tokenize applies the model's token layout: BERT and XLM-R wrap ids with + * [CLS]..[SEP] (<s>..</s>), while Qwen appends its terminal token. + */ + +/* Opaque tokenizer handle; owns the tokenizer and its resolved special tokens. */ +typedef struct ffwd_tok ffwd_tok_t; + +/* Late-interaction tokens: + * token ids plus the document positions to score. + * For queries, keep is NULL because every position is kept, and + * n_keep == n_tokens. */ +typedef struct { + int *ids; + int n_tokens; + int *keep; + int n_keep; +} ffwd_late_tokens_t; + +/* Load the tokenizer for the model in model_dir and resolve its special-token + * ids. is_late additionally resolves the late-interaction [Q]/[D]/[MASK] ids. + * NULL on failure with a reason in err. */ +FFWD_API ffwd_tok_t *ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen); +FFWD_API void ffwd_tok_free(ffwd_tok_t *t); + +/* Tokenize text for this model: applies the model's special-token layout and + * optionally prepends a query instruction. Returns a malloc'd id array (caller + * frees) and sets *n_out. NULL on error. */ +FFWD_API int *ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out); + +/* Late-interaction tokenization: [Q]/[D] prefix, [MASK]-pad queries to a fixed + * length, and mark the document positions to score. Returns 0 on success. */ +FFWD_API int +ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out); +FFWD_API void ffwd_late_tokens_free(ffwd_late_tokens_t *out); + +/* The contextual chunk-separator id (the tokenizer's <|endoftext|>), for + * frontends that split a document into chunks. */ +FFWD_API int ffwd_tok_context_separator_id(const ffwd_tok_t *t); + +/* Whether ffwd_tokenize appends the model's terminal token (<|endoftext|>) for + * last-token pooling. Off until set; the caller sets it from the loaded model's + * config (ffwd_config(...)->append_terminal_token), since it is model-specific + * and only known after the backend is open. */ +FFWD_API void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on); + +/* Verbose level: 0=quiet, 1=info, 2=debug */ +FFWD_API extern int ffwd_verbose; + +#endif /* FFWD_H */ |
