diff options
Diffstat (limited to 'libffwd')
42 files changed, 439 insertions, 452 deletions
diff --git a/libffwd/Makefile b/libffwd/Makefile index 955f93e..2372697 100644 --- a/libffwd/Makefile +++ b/libffwd/Makefile @@ -9,15 +9,15 @@ SHARED := $(OUTDIR)/lib$(LIB).$(SHARED_EXT) CFLAGS += -std=c11 -pthread $(BACKEND_CFLAGS) CPPFLAGS += -I. SHARED_FLAGS += -pthread + # Restrict the shared library's exported symbols to the public ABI. SHARED_LDFLAGS += $(EXPORT_FLAGS) -# Tokenizer, config, safetensors reader, and L2/cosine helpers: every backend. +# Tokenizer, config, safetensors reader, and vector helpers. COMMON_SRCS := config.c vec.c tokenize.c bpe.c sentencepiece.c wordpiece.c safetensors.c -# CPU math kernels and SIMD dispatch. generic.c is always built; avx.c and -# neon.c each guard their entire body by arch (__AVX2__ / __ARM_NEON), so the -# off-target file compiles to an empty translation unit. +# CPU kernels and SIMD dispatch. +# generic.c is always built; avx.c and neon.c each guarded by arch (__AVX2__ / __ARM_NEON). KERNEL_SRCS := act.c gemm.c norm.c rope.c attn.c threadpool.c generic.c avx.c neon.c BLAS_SRCS := backend_blas.c model.c model_blas.c forward.c late.c late_blas.c alloc.c \ @@ -45,22 +45,17 @@ all: $(STATIC) $(SHARED) $(eval $(call static_archive,$(STATIC),$(OBJS))) $(eval $(call shared_library,$(SHARED),$(OBJS),,$(BACKEND_LDFLAGS) $(YYJSON_LDFLAGS) -lm)) -# yyjson is vendored under deps/yyjson; build its archive before linking the -# shared library (the static libffwd.a does not embed it — consumers link both). + YYJSON := $(ROOT)/deps/yyjson $(OUTDIR)/libyyjson.a: $(MAKE) -C $(YYJSON) $(SHARED): $(OUTDIR)/libyyjson.a -# The export map is generated from the public ABI (the FFWD_API declarations in -# ffwd.h) instead of being committed, so it cannot drift from the header. It is -# regenerated when the header or the generator changes, and the shared library -# relinks when it does. + +# The export map is auto-generated from the FFWD_API declarations in ffwd.h. $(EXPORT_FILE): ffwd.h $(ROOT)/devtools/gen_exports.py python3 $(ROOT)/devtools/gen_exports.py --outdir $(ROOT) $(SHARED): $(EXPORT_FILE) -# Removes only this MODE/BACKEND's objects and artifacts; the root `make clean` -# removes the whole build/ tree. clean: $(RM) -r $(OBJDIR) $(STATIC) $(SHARED) $(SHARED).dSYM diff --git a/libffwd/act.c b/libffwd/act.c index 688dd71..2495311 100644 --- a/libffwd/act.c +++ b/libffwd/act.c @@ -1,6 +1,4 @@ -/* - * act.c - activations - */ +/* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU variants. */ #include <math.h> #include <stdlib.h> @@ -29,13 +27,6 @@ # define M_PI 3.14159265358979323846 #endif -/* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU - * variants. */ - -/* ======================================================================== - * Activation Functions - * ======================================================================== */ - void silu_mul_inplace(float *gate, const float *up, int n) { #if defined(__APPLE__) && defined(USE_BLAS) enum { CHUNK = 4096 }; diff --git a/libffwd/alloc.c b/libffwd/alloc.c index 6af854a..828258e 100644 --- a/libffwd/alloc.c +++ b/libffwd/alloc.c @@ -4,10 +4,11 @@ #include <stdint.h> #include <limits.h> -/* Sequence-capacity growth policy for the forward-pass workspaces: for new - * growth, never allocate fewer than the minimum, and round up to the - * granularity so small length changes do not trigger a realloc on every call. - * Existing larger capacity is preserved. */ +/* Sequence-capacity growth policy for the forward-pass workspaces: + * For new growth: + * - never allocate less than the minimum required size. + * - round up to the granularity so small length changes do not trigger a realloc on every call. + * - keep existing capacity if it is already larger. */ #define FFWD_MIN_WORKSPACE_SEQ_CAP 16 #define FFWD_WORKSPACE_SEQ_GRANULARITY 16 diff --git a/libffwd/alloc.h b/libffwd/alloc.h index 0712e21..a12ee7a 100644 --- a/libffwd/alloc.h +++ b/libffwd/alloc.h @@ -3,11 +3,9 @@ #include <stddef.h> -/* Overflow-checked size arithmetic and typed (re)allocation helpers shared by - * the model loader, the forward pass, and the late-interaction code. Every - * helper returns -1 (or NULL) on overflow or allocation failure so callers can - * bail without a partial buffer. realloc_* keep the original pointer intact on - * failure; the caller still owns it. */ +/* Checked size arithmetic and typed allocation helpers. + * Return -1 or NULL on overflow/allocation failure. + * realloc_* leave the original pointer unchanged on failure. */ /* out = a * b, or -1 if the product overflows size_t. */ int mul_size(size_t a, size_t b, size_t *out); @@ -15,20 +13,19 @@ int mul_size(size_t a, size_t b, size_t *out); /* out = a + b, or -1 if the sum overflows size_t. */ int add_size(size_t a, size_t b, size_t *out); -/* Round a requested sequence capacity up to the workspace growth granularity, - * never shrinking below the current capacity. Returns -1 on bad input/overflow. */ +/* Grow sequence capacity without shrinking. */ int grow_cap(int current, int needed, int *out); -/* realloc *ptr to hold `count` floats; *ptr unchanged on failure. */ +/* realloc *ptr to hold count floats; *ptr unchanged on failure. */ int realloc_floats(float **ptr, size_t count); -/* malloc `count` floats, NULL on overflow/failure. */ +/* malloc count floats, NULL on overflow/failure. */ float *malloc_floats(size_t count); /* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */ int realloc_floats_2d(float **ptr, int rows, int cols); -/* realloc *ptr to hold `count` ints; *ptr unchanged on failure. */ +/* realloc *ptr to hold count ints; *ptr unchanged on failure. */ int realloc_ints(int **ptr, size_t count); #endif /* FFWD_ALLOC_H */ diff --git a/libffwd/attn.c b/libffwd/attn.c index 6134064..57c665a 100644 --- a/libffwd/attn.c +++ b/libffwd/attn.c @@ -1,7 +1,3 @@ -/* - * attn.c - attention - */ - #include "kernels.h" #include "impl.h" #include "threadpool.h" @@ -25,9 +21,7 @@ /* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE) * GQA, each with an online-softmax and a BLAS-GEMM path. */ -/* ======================================================================== - * Attention Operations - * ======================================================================== */ +/* Attention operations */ float dot_f32(const float *a, const float *b, int n) { return dot_f32_impl(a, b, n); } diff --git a/libffwd/avx.c b/libffwd/avx.c index 41259a1..7cbbd5a 100644 --- a/libffwd/avx.c +++ b/libffwd/avx.c @@ -1,12 +1,14 @@ -/* - * avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) +/* x86 SIMD hot kernels: AVX2+FMA, plus AVX-512 when available. * - * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. - * Uses AVX-512F+BW for 16-wide bf16 matvec (dominant for seq_len==1), and - * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data). + * BF16 -> F32 conversion: + * load 16 uint16 values, zero-extend to 32-bit, then shift left 16. * - * The bf16 matvec processes 4 output rows simultaneously to reduce instruction - * overhead and improve out-of-order execution on memory-bound workloads. + * Uses AVX-512F+BW for 16-wide BF16 matvec, dominant when seq_len == 1. + * Uses AVX2+FMA / AVX-512F for F32 attention helpers, which operate on + * cache-resident data. + * + * BF16 matvec processes 4 output rows at once to reduce instruction overhead + * and improve out-of-order execution on memory-bound workloads. */ #include "impl.h" @@ -16,9 +18,7 @@ # include <immintrin.h> # include <string.h> -/* ===================================================================== - * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise - * ===================================================================== */ +/* BF16 matvec: uses AVX-512F+BW when available, AVX2 fallback otherwise */ # if defined(__AVX512F__) && defined(__AVX512BW__) diff --git a/libffwd/backend_blas.c b/libffwd/backend_blas.c index 9e3d376..7bd1d0d 100644 --- a/libffwd/backend_blas.c +++ b/libffwd/backend_blas.c @@ -1,6 +1,10 @@ -/* backend_blas.c - CPU backend (BLAS), implements the engine API in ffwd.h. - * Linked only into CPU builds. The CPU model loads on the caller's thread, so - * ffwd_open does the full load and activate/worker_free are no-ops. */ +/* CPU backend. + * Implements the engine API in ffwd.h and is linked only into CPU builds. + * + * The CPU model loads on the caller's thread, so ffwd_open does the full load. + * activate and worker_free are no-ops. + */ + #include "internal.h" #include "threadpool.h" #include "platform.h" diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c index a925404..6adbe95 100644 --- a/libffwd/backend_cuda.c +++ b/libffwd/backend_cuda.c @@ -1,7 +1,13 @@ -/* backend_cuda.c - NVIDIA GPU backend (CUDA), implements the engine API in ffwd.h. - * Linked only into CUDA builds. The context is owned by the inference thread: - * open records what to load, activate creates the context, worker_free destroys - * it. CUDA encodes on the device and scores MaxSim on the host. */ +/* NVIDIA CUDA backend. + * Implements the API in ffwd.h and is linked only into CUDA builds. + * + * The context is owned by the inference thread: + * open records what to load, activate creates the context, and worker_free + * destroys it. + * + * CUDA encodes on the device and scores MaxSim on the host. + */ + #include "internal.h" #include "cuda.h" #include "platform.h" diff --git a/libffwd/backend_mlx.c b/libffwd/backend_mlx.c index 932e550..841aad1 100644 --- a/libffwd/backend_mlx.c +++ b/libffwd/backend_mlx.c @@ -1,7 +1,12 @@ -/* backend_mlx.c - Apple GPU backend (MLX), implements the engine API in ffwd.h. - * Linked only into Apple Silicon GPU builds. MLX streams are thread-local, so - * the context is created and destroyed on the inference thread: open only - * records what to load, activate creates the context, worker_free destroys it. */ +/* Metal backend. + * Implements the API in ffwd.h and is linked only into Apple Metal builds. + * + * MLX streams are thread-local, so the context is created and destroyed on the + * inference thread. + * open only records what to load; activate creates the context; worker_free + * destroys it. + */ + #include "internal.h" #include "mlx.h" #include "platform.h" diff --git a/libffwd/bpe.c b/libffwd/bpe.c index b3c8a8c..5a4d184 100644 --- a/libffwd/bpe.c +++ b/libffwd/bpe.c @@ -8,9 +8,7 @@ #include <stdlib.h> #include <string.h> -/* ======================================================================== - * GPT-2 Bytes-to-Unicode Mapping - * ======================================================================== */ +/* GPT-2 bytes-to-unicode mapping */ static int gpt2_byte_to_unicode[256]; static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ @@ -124,9 +122,7 @@ static char *decode_gpt2_token(const char *token_str) { return (char *)bytes; } -/* ======================================================================== - * String->int hash map (open addressing) - * ======================================================================== */ +/* String->int hash map (open addressing) */ typedef struct { char *key; /* NULL means empty slot */ @@ -252,9 +248,7 @@ static int map_get_pair(const str_int_entry_t *map, int cap, const char *a, cons return -1; } -/* ======================================================================== - * BPE helpers - * ======================================================================== */ +/* BPE helpers */ static int utf8_char_len(unsigned char c) { if ((c & 0x80) == 0) @@ -319,9 +313,7 @@ static char *str_concat2(const char *a, const char *b) { return r; } -/* ======================================================================== - * Integer-pair BPE merges: (left_id, right_id) -> (rank, merged_id) - * ======================================================================== */ +/* Integer-pair BPE merges: (left_id, right_id) -> (rank, merged_id) */ typedef struct { uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */ @@ -1253,11 +1245,14 @@ static int encode_bpe_word_into_slow(const tok_bpe_t *tok, return 0; } -/* Integer-pair BPE merge: token ids in a doubly-linked array with cached - * pair ranks; each merge updates only the two adjacent pairs. Falls back to - * the string-based loop when the int tables are unavailable. Identical - * output: initial ids are the byte-level single-char token ids and every - * merge result is the vocab id of the concatenated pair. */ +/* Integer-pair BPE merge. + * Uses token ids in a doubly-linked array with cached pair ranks. + * Each merge updates only the two adjacent pairs. + * + * Falls back to the string-based loop when int tables are unavailable. + * + * Output is identical: initial ids are byte-level single-char token ids, and + * each merge result is the vocab id of the concatenated pair. */ static int encode_bpe_word_into(const tok_bpe_t *tok, tok_bpe_workspace_t *ws, const char *mapped, @@ -1650,9 +1645,7 @@ static int load_merges_map(tok_bpe_t *tok, const char *merges_path) { return 0; } -/* ======================================================================== - * Public API - * ======================================================================== */ +/* Public API */ tok_bpe_t *tok_bpe_load(const char *vocab_json_path) { FILE *f = fopen(vocab_json_path, "rb"); @@ -1688,10 +1681,11 @@ tok_bpe_t *tok_bpe_load(const char *vocab_json_path) { return NULL; } - /* vocab.json is a flat { token: id } object. Walk it once for the max id to - * size the tables, then again to fill them. yyjson unescapes each key - * (including \uXXXX) to the same byte-level GPT-2 form the old scanner - * produced; decode_gpt2_token then maps that to the raw token text. */ + /* vocab.json is a flat { token: id } object. + * Walk it once to find the max id and size the tables, then again to fill them. + * yyjson unescapes each key, including \uXXXX, to the same byte-level GPT-2 form + * the old scanner produced. + * decode_gpt2_token then maps that to the raw token text. */ int max_id = 0; size_t idx, n; yyjson_val *key, *val; diff --git a/libffwd/bpe.h b/libffwd/bpe.h index 5b16c52..5199fd0 100644 --- a/libffwd/bpe.h +++ b/libffwd/bpe.h @@ -1,5 +1,5 @@ /* - * bpe.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level). * * Supports: * - decode token ID -> UTF-8 text diff --git a/libffwd/build.h b/libffwd/build.h index a133163..6ab367e 100644 --- a/libffwd/build.h +++ b/libffwd/build.h @@ -1,8 +1,12 @@ -/* ffwd_build.h - build identity and capability information for the ffwd tools +/* Build identity and capability information. * * The real values come from the Makefile's VERSION_CFLAGS at compile time; the - * fallbacks below apply when those flags are absent. Front-end only: the - * inference library does not use these. */ + * fallbacks below apply when those flags are absent. The core library doesn't use + * these; it's for frontends only. + */ + +/* TODO: tools should not depend on the internal library. */ + #ifndef FFWD_BUILD_H #define FFWD_BUILD_H diff --git a/libffwd/config.c b/libffwd/config.c index 6531658..d9efb4d 100644 --- a/libffwd/config.c +++ b/libffwd/config.c @@ -9,7 +9,7 @@ #include <stdlib.h> #include <string.h> -/* Verbosity flag, read across the codebase for optional stderr diagnostics. +/* Verbosity flag for optional stderr diagnostics across the codebase. * Defined in config.c because every backend links it. */ int ffwd_verbose = 0; @@ -20,11 +20,13 @@ typedef enum { JSON_BOOL_INVALID = -1, } json_bool_t; -/* Field readers over a parsed JSON object. yyjson getters are NULL-tolerant, so - * a missing key or wrong-typed value falls back cleanly. These replace a - * substring scanner that could match a key nested anywhere in the document - * (e.g. a field inside a nested sub-config), which is exactly the failure mode - * a real parser removes. */ +/* Read fields from a parsed JSON object. + * yyjson getters tolerate NULL, so missing keys or wrong types cleanly return + * the fallback. + * + * This replaces substring scanning, which could match a key anywhere in the + * document, including inside nested sub-configs. A real parser avoids that + * failure mode. */ static int json_get_int(yyjson_val *obj, const char *key, int fallback) { yyjson_val *v = yyjson_obj_get(obj, key); if (yyjson_is_int(v)) diff --git a/libffwd/cuda.h b/libffwd/cuda.h index 851e412..22fa78f 100644 --- a/libffwd/cuda.h +++ b/libffwd/cuda.h @@ -1,13 +1,11 @@ /* - * ffwd_cuda.h - CUDA/cuBLAS backend + * CUDA/cuBLAS backend */ #ifndef FFWD_CUDA_H #define FFWD_CUDA_H -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying FFWD_API are exported - * from the shared library; everything else stays internal. */ +/* Public-API export annotation. */ #ifndef FFWD_API # if defined(__GNUC__) # define FFWD_API __attribute__((visibility("default"))) diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h index 03a8e26..c4708fa 100644 --- a/libffwd/ffwd.h +++ b/libffwd/ffwd.h @@ -1,13 +1,8 @@ -/* - * ffwd.h - ffwd API - */ - #ifndef FFWD_H #define FFWD_H -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying FFWD_API are exported - * from the shared library; everything else stays internal. */ +/* The libraries are built with -fvisibility=hidden, so only declarations carrying + * FFWD_API are exported from the shared library; everything else stays internal. */ #ifndef FFWD_API # if defined(__GNUC__) # define FFWD_API __attribute__((visibility("default"))) @@ -19,25 +14,21 @@ #include <stddef.h> #include <stdint.h> -/* ======================================================================== - * Shared constants and family defaults - * ======================================================================== */ +/* Shared constants and defaults */ #define FFWD_VOCAB_SIZE 151936 #define FFWD_HEAD_DIM 128 #define FFWD_MAX_LAYERS 64 /* upper bound for stack arrays */ #define FFWD_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ -/* Late-interaction (ColBERT) special-token id fallbacks and the query pad - * length, used when a model's tokenizer does not define the tokens itself. */ +/* Late-interaction (ColBERT) fallback special-token ids and query pad length. + * Used when the model tokenizer does not define these tokens itself. */ #define FFWD_LATE_QUERY_TOKENS 32 #define FFWD_LATE_MASK_TOKEN_ID 151642 #define FFWD_LATE_QUERY_PREFIX_ID 151669 #define FFWD_LATE_DOCUMENT_PREFIX_ID 151670 -/* ======================================================================== - * Model Configuration (populated from config.json at load time) - * ======================================================================== */ +/* Model Configuration (populated from config.json at load time) */ typedef enum { FFWD_ATTENTION_BIDIRECTIONAL = 0, @@ -86,12 +77,14 @@ typedef struct { ffwd_attention_mode_t attention_mode; ffwd_pooling_mode_t pooling_mode; int normalize_embeddings; - /* Qwen3-Embedding pools the last token, which is the tokenizer's - * <|endoftext|> suffix added by its post-processor - not the model's chat - * eos_token_id (<|im_end|>). Text frontends append it and resolve the id - * from the tokenizer exactly like the contextual separator - * (FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The - * model forward consumes token ids exactly and never appends. */ + /* Qwen3-Embedding pools the last token: the tokenizer's <|endoftext|> suffix + * added by its post-processor, not the model's chat eos_token_id (<|im_end|>). + * + * Text frontends append it and resolve its id from the tokenizer, just like the + * contextual separator. FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model + * fallback. + * + * Model forward consumes token ids exactly and never appends. */ int append_terminal_token; } ffwd_config_t; @@ -111,24 +104,31 @@ typedef struct { int n_spans; } ffwd_context_input_t; -/* ======================================================================== - * Engine API — the backend-selected inference handle +/* + * Backend-selected inference handle. + * + * This is the library's main public interface and the full supported surface of + * the shared library. Exactly one backend is linked at build time: CPU, Apple Metal, + * or NVIDIA CUDA. That backend implements these functions, so callers never branch + * on backend. + * + * The CLI, server, and language bindings use only these functions plus the math + * helpers below. + * + * Model loading has two phases so a GPU context can be created on the thread + * that owns it. * - * This is the library's primary public interface and the entire supported - * surface of the shared library. Exactly one backend (CPU, Apple Metal, or - * NVIDIA CUDA) is linked at build time and implements these functions; - * callers never branch on the backend. The CLI, the server, and any language - * binding use only these functions plus the math helpers below. + * ffwd_open runs on the caller's thread. It loads a CPU model immediately; in a + * GPU build, it only records what to load. * - * A model loads in two phases so a GPU context can be created on the thread - * that will own it. ffwd_open runs on the caller's thread and loads a CPU - * model immediately; a GPU build only records what to load. ffwd_activate then - * runs on the inference thread and creates the GPU context (a no-op for CPU). - * Teardown mirrors this: ffwd_worker_free releases the GPU context from the - * inference thread, ffwd_free releases the rest. - * ======================================================================== */ + * ffwd_activate runs on the inference thread and creates the GPU context. It is + * a no-op for CPU. + * + * Teardown mirrors activation: ffwd_worker_free releases the GPU context from + * the inference thread, and ffwd_free releases the rest. + */ -/* Opaque inference handle; its layout is private to the linked backend. */ +/* Opaque inference handle; layout is private to the linked backend. */ typedef struct embed ffwd_t; /* Backend tuning, parsed from the command line. A backend reads the fields it @@ -220,22 +220,27 @@ FFWD_API int ffwd_l2_normalize(float *vec, int dim); */ FFWD_API float ffwd_cosine_similarity(const float *a, const float *b, int dim); -/* ======================================================================== - * Tokenization — text into the token ids a model expects +/* + * Tokenization: text -> token ids the model expects. * - * The tokenizer is backend-independent (text -> ids is identical on CPU or - * GPU), so it is its own handle, separate from ffwd_t. ffwd_tok_open picks the - * tokenizer by the files in model_dir (WordPiece vocab.txt, SentencePiece - * .model, or byte-level vocab.json) and resolves the model's special-token ids. - * ffwd_tokenize then applies that model's layout: BERT and XLM-R wrap the ids - * with [CLS]..[SEP] (<s>..</s>), Qwen appends its terminal token. - * ======================================================================== */ + * Tokenization is backend-independent because text -> ids is the same on CPU + * and GPU, so it has its own handle, separate from ffwd_t. + * + * ffwd_tok_open selects the tokenizer from files in model_dir: WordPiece + * vocab.txt, SentencePiece .model, or byte-level vocab.json. It also resolves + * the model's special-token ids. + * + * ffwd_tokenize applies the model's token layout: BERT and XLM-R wrap ids with + * [CLS]..[SEP] (<s>..</s>), while Qwen appends its terminal token. + */ /* Opaque tokenizer handle; owns the tokenizer and its resolved special tokens. */ typedef struct ffwd_tok ffwd_tok_t; -/* Late-interaction tokens: ids, plus the document positions to score. For a - * query, keep is NULL (every position is kept) and n_keep == n_tokens. */ +/* Late-interaction tokens: + * token ids plus the document positions to score. + * For queries, keep is NULL because every position is kept, and + * n_keep == n_tokens. */ typedef struct { int *ids; int n_tokens; diff --git a/libffwd/forward.c b/libffwd/forward.c index d8e45f9..d5d2442 100644 --- a/libffwd/forward.c +++ b/libffwd/forward.c @@ -9,14 +9,12 @@ #include <math.h> #include <limits.h> -/* Below these packed-sequence lengths the fused BF16 row kernels beat widening - * the weight to F32 and dispatching through BLAS; above them, widen once. */ +/* For packed sequences below these lengths, fused BF16 row kernels beat + * widening the weight to F32 and using BLAS. For longer packed sequences, widen once. */ #define FFWD_BF16_QKV_FUSE_MAX_SEQ 16 #define FFWD_BF16_PAIR_FUSE_MAX_SEQ 4 -/* ======================================================================== - * Linear projections (weight-dtype dispatch) and workspace scratch growth - * ======================================================================== */ +/* Linear projections (weight-dtype dispatch) and workspace scratch growth */ static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { for (int i = 0; i < n; i++) @@ -123,14 +121,14 @@ static void linear_pair_weight(ffwd_workspace_t *ws, linear_nobias_weight(ws, b, x, wb, seq_len, in_dim, b_dim); } -/* ======================================================================== - * Packed forward pass +/* Packed forward pass. * - * The workspace-growth helpers these call (ensure_buffers, ensure_rope_cache, - * ensure_attention_scores, ensure_offsets) live in model_blas.c beside the - * workspace lifecycle and ffwd_workspace_nbytes, whose size accounting mirrors - * them. - * ======================================================================== */ + * Workspace-growth helpers called here live in model_blas.c beside the + * workspace lifecycle and ffwd_workspace_nbytes: + * ensure_buffers, ensure_rope_cache, ensure_attention_scores, ensure_offsets. + * + * ffwd_workspace_nbytes mirrors their size accounting. + */ static ffwd_attention_fn attention_for_config(const ffwd_config_t *cfg) { return cfg->attention_mode == FFWD_ATTENTION_CAUSAL @@ -298,13 +296,21 @@ int forward_packed_inplace(const ffwd_model_t *model, 0, model->config.n_layers, apply_final_norm); } -/* BERT-family forward: token + learned-position + token-type embeddings and an - * embedding LayerNorm, then post-norm encoder layers (bias on every projection, - * GeLU feed-forward, no RoPE, no per-head norm). Bidirectional attention is - * block-diagonal over the packed batch, so each sequence's positions restart at - * cfg->position_id_offset (0 for BERT, pad_id + 1 for RoBERTa/XLM-R). Leaves the - * final hidden states in ws->x; pooling reads them as-is because the last - * layer's LayerNorm is already the model's final normalization. */ +/* BERT-family forward pass. + * + * Uses token, learned-position, and token-type embeddings, then embedding + * LayerNorm. + * + * Encoder layers are post-norm, with bias on every projection, GeLU + * feed-forward, no RoPE, and no per-head norm. + * + * Bidirectional attention is block-diagonal over the packed batch. Each + * sequence's positions restart at cfg->position_id_offset: 0 for BERT, or + * pad_id + 1 for RoBERTa/XLM-R. + * + * Leaves final hidden states in ws->x. Pooling reads them as-is because the last + * layer's LayerNorm is already the model's final normalization. + */ static int forward_packed_bert(const ffwd_model_t *model, ffwd_workspace_t *ws, const ffwd_input_t *inputs, @@ -405,8 +411,8 @@ static int pool_embeddings(const ffwd_model_t *model, const ffwd_config_t *cfg = &model->config; int hidden = cfg->hidden_size; float eps = cfg->rms_norm_eps; - /* Qwen pools pre-final-norm states and applies the final RMSNorm here; BERT - * pools already-normed states (no final norm), signalled by a NULL norm. */ + /* Qwen pools pre-final-norm states, then applies final RMSNorm here. + * BERT pools already-normalized states, so a NULL norm means no final norm. */ const float *norm_weight = model->weights.norm; const float *x = ws->x; if (hidden <= 0 || !x) @@ -419,8 +425,7 @@ static int pool_embeddings(const ffwd_model_t *model, float *emb = out_embeddings + (size_t)b * hidden; if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) { - /* Single-token pooling: CLS takes the first token, last-token the - * final one. */ + /* Single-token pooling: CLS uses the first token; last-token uses the final token. */ int idx = cfg->pooling_mode == FFWD_POOL_CLS ? start : end - 1; const float *row = x + (size_t)idx * hidden; if (norm_weight) { @@ -460,9 +465,7 @@ static int pool_embeddings(const ffwd_model_t *model, return 0; } -/* ======================================================================== - * Public forward/embed APIs - * ======================================================================== */ +/* Forward/embed APIs */ int ffwd_model_forward_into(const ffwd_model_t *model, ffwd_workspace_t *ws, diff --git a/libffwd/gemm.c b/libffwd/gemm.c index bb54fb9..ab664f1 100644 --- a/libffwd/gemm.c +++ b/libffwd/gemm.c @@ -1,5 +1,5 @@ /* - * gemm.c - general Matrix Multiplication + * General matrix multiplication. */ #include "kernels.h" @@ -30,21 +30,18 @@ # define M_PI 3.14159265358979323846 #endif -/* Matrix and vector ops: GEMM, the dtype-dispatch linear kernels, and the - * BF16-fused matvec/QKV/pair workers, plus the elementwise residual add. */ +/* Matrix and vector ops: + * GEMM, dtype-dispatch linear kernels, BF16-fused matvec/QKV/pair workers, + * and element-wise residual add. */ -/* ======================================================================== - * Basic Element-wise Operations - * ======================================================================== */ +/* Basic element-wise operations */ void add_inplace(float *a, const float *b, int n) { for (int i = 0; i < n; i++) a[i] += b[i]; } -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ +/* Matrix operations */ void matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { #ifdef USE_BLAS @@ -425,7 +422,8 @@ void linear_nobias_bf16( bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); return; } - /* Callers route longer sequences through an F32-widened weight matrix in - * caller-owned scratch; this per-row path stays correct for any length. */ + /* Callers send longer sequences through an F32-widened weight matrix in + * caller-owned scratch. + * This per-row path remains correct for any length. */ bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); } diff --git a/libffwd/generic.c b/libffwd/generic.c index 8798107..5425247 100644 --- a/libffwd/generic.c +++ b/libffwd/generic.c @@ -1,5 +1,5 @@ /* - * generic.c - architecture-generic hot kernels / non-SIMD fallback + * Architecture-generic hot kernels / non-SIMD fallback. */ #include "impl.h" diff --git a/libffwd/impl.h b/libffwd/impl.h index 39baa40..2149cf2 100644 --- a/libffwd/impl.h +++ b/libffwd/impl.h @@ -1,15 +1,16 @@ /* - * impl.h - internal architecture dispatch for hot kernels + * Internal architecture dispatch for hot kernels. * - * The small vector primitives (dot product, scale, axpy, scale-add, scaled - * multiply) are defined here as static inline so they fold directly into the - * caller's loop: a row of RMSNorm or attention issues no call and the compiler - * keeps the row in registers across the reduction and the write-back. The - * generic scalar versions stay out of line in generic.c as a reference for the - * kernel tests and as the fallback on platforms with no SIMD path. + * Small vector primitives (dot product, scale, axpy, scale-add, scaled + * multiply) are static inline here so they fold into the caller's loop. + * For example, an RMSNorm or attention row has no call overhead, and the + * compiler can keep the row in registers across reduction and write-back. * - * bf16_matvec_fused is large (a full matrix-vector product) and is called once - * per output tile rather than inside a tight loop, so it stays out of line. + * Generic scalar versions stay out of line in generic.c. They are the reference + * for kernel tests and the fallback when no SIMD path is available. + * + * bf16_matvec_fused is a large full matrix-vector product, called once per + * output tile rather than inside a tight loop, so it stays out of line. */ #ifndef KERNELS_IMPL_H diff --git a/libffwd/internal.h b/libffwd/internal.h index a2b933a..6030c36 100644 --- a/libffwd/internal.h +++ b/libffwd/internal.h @@ -1,9 +1,3 @@ -/* - * internal.h - non-exported, in-tree API shared by the frontends, - * backends, tests, and benchmarks. The public ABI is ffwd.h; none of these - * symbols carry FFWD_API visibility or appear in the export maps. - */ - #ifndef FFWD_INTERNAL_H #define FFWD_INTERNAL_H @@ -17,7 +11,8 @@ typedef struct ffwd_workspace ffwd_workspace_t; typedef struct ffwd_late_model ffwd_late_model_t; typedef struct ffwd_late_workspace ffwd_late_workspace_t; -/* Backend-specific help text used by the in-tree frontends. */ +/* Help text used by the frontends. */ +/* TODO: tools should not depend on the internal API */ void ffwd_cli_help(FILE *f); void ffwd_server_help(FILE *f); @@ -91,7 +86,7 @@ int ffwd_model_encode_spans_batch(const ffwd_model_t *model, /* * Late-interaction CPU model API. The public engine path exposes reranking via - * ffwd_rerank(); these entry points are for in-tree tests/backends only. + * ffwd_rerank(); these entry points are for tests/backends only. */ ffwd_late_model_t *ffwd_late_model_load(const char *model_dir); void ffwd_late_model_free(ffwd_late_model_t *model); diff --git a/libffwd/kernels.h b/libffwd/kernels.h index 2a917e0..11312db 100644 --- a/libffwd/kernels.h +++ b/libffwd/kernels.h @@ -1,5 +1,5 @@ /* - * kernels.h - tkern public API: transformer inference kernels (CPU, float32) + * Transformer inference kernels (CPU, float32). * * Low-level math operations. All operate on float32 tensors in row-major order. */ @@ -10,15 +10,11 @@ #include <stddef.h> #include <stdint.h> -/* ======================================================================== - * Basic Operations - * ======================================================================== */ +/* Basic operations */ void add_inplace(float *a, const float *b, int n); -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ +/* Matrix operations */ /* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ void matmul_t(float *C, const float *A, const float *B, int M, int K, int N); @@ -62,9 +58,7 @@ void linear_nobias_bf16_qkv(float *q, /* Dot product using the best available local SIMD implementation. */ float dot_f32(const float *a, const float *b, int n); -/* ======================================================================== - * Normalization - * ======================================================================== */ +/* Normalization */ /* RMS Normalization: out = x / rms(x) * weight */ void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); @@ -75,9 +69,10 @@ void rms_norm(float *out, const float *x, const float *weight, int seq_len, int void rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); -/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps) - * + beta, with mean and biased (population) variance over the hidden axis per - * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ +/* Layer normalization for BERT-family models: + * out = gamma * (x - mean) / sqrt(var + eps) + beta. + * Mean and biased/population variance are computed over the hidden axis per row. + * Unlike RMSNorm, this subtracts the mean and adds a bias. */ void layer_norm(float *out, const float *x, const float *gamma, @@ -86,25 +81,21 @@ void layer_norm(float *out, int hidden, float eps); -/* ======================================================================== - * Activation Functions - * ======================================================================== */ +/* Activation functions */ void softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ void silu_mul_inplace(float *gate, const float *up, int n); -/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the - * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ +/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). + * This is the "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ void gelu_inplace(float *x, int n); /* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh): * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */ void gelu_tanh_inplace(float *x, int n); -/* ======================================================================== - * Attention Operations - * ======================================================================== */ +/* Attention operations */ /* * Bidirectional GQA attention over a packed/ragged batch. @@ -169,9 +160,7 @@ void causal_gqa_attention_packed_with_scratch(float *out, float *scratch, size_t scratch_bytes); -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ +/* Position embeddings */ /* * NeoX-style RoPE: compute cos/sin for positions. diff --git a/libffwd/late.c b/libffwd/late.c index e086ad3..9418a20 100644 --- a/libffwd/late.c +++ b/libffwd/late.c @@ -1,3 +1,16 @@ +/* + * Late-interaction model. + * + * A late-interaction model is a pooled base model plus a 1_Dense linear + * projection from hidden states to token vectors. + * + * Instead of pooling to one vector, it keeps one projected vector per kept token + * and scores query/doc pairs with MaxSim. + * + * Reuses the base model's packed forward pass and workspace. This layer only + * owns the projection, token-vector buffer, and scoring. + */ + #include "ffwd.h" #include "model_internal.h" #include "kernels.h" @@ -8,17 +21,6 @@ #include <limits.h> #include <float.h> -/* ======================================================================== - * Late-interaction model - * - * A late-interaction model is a pooled base model plus a 1_Dense linear - * projection from hidden states to token vectors. Instead of pooling to one - * vector, it keeps one projected vector per kept token and scores query/doc - * pairs with MaxSim. It reuses the base model's packed forward pass and - * workspace; only the projection, the token-vector buffer, and the scoring - * live here. - * ======================================================================== */ - struct ffwd_late_model { ffwd_model_t *base; safetensors_file_t *dense_sf; @@ -107,8 +109,9 @@ const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model) { int ffwd_late_model_token_dim(const ffwd_late_model_t *model) { return model ? model->token_dim : 0; } -/* Internal (internal.h): expose the base model and projection so GPU - * backends can upload them to device without seeing struct ffwd_late_model. */ +/* Internal (internal.h): + * expose the base model and projection so GPU backends can upload them to the + * device without seeing struct ffwd_late_model. */ ffwd_model_t *ffwd_late_model_base(const ffwd_late_model_t *model) { return model ? model->base : NULL; } diff --git a/libffwd/late_blas.c b/libffwd/late_blas.c index dabeb58..84b7f17 100644 --- a/libffwd/late_blas.c +++ b/libffwd/late_blas.c @@ -5,8 +5,9 @@ #include <stdlib.h> #include <string.h> -/* CPU late-interaction encoding. The late model loader and MaxSim scoring live - * in late.c because CUDA reuses those without linking the CPU transformer. */ +/* CPU late-interaction encoding. + * The late model loader and MaxSim scoring stay in late.c because CUDA reuses + * them without linking the CPU transformer. */ struct ffwd_late_workspace { const ffwd_late_model_t *model; diff --git a/libffwd/mlx.c b/libffwd/mlx.c index 0b6d8b7..d8f6d1e 100644 --- a/libffwd/mlx.c +++ b/libffwd/mlx.c @@ -13,9 +13,7 @@ #include <limits.h> #include <float.h> -/* ======================================================================== - * Per-layer MLX weight arrays - * ======================================================================== */ +/* Per-layer MLX weight arrays */ typedef struct { mlx_array w; @@ -33,9 +31,9 @@ typedef struct { mlx_array q_norm, k_norm; mlx_array input_norm, post_attn_norm; mlx_linear_t gate_proj, up_proj, down_proj; - /* BERT family (unset for Qwen3; see internal.h). wo/up_proj/down_proj - * and input_norm/post_attn_norm are reused as the BERT attention output, - * FFN matrices, and the two block LayerNorm weights. */ + /* BERT family only; unset for Qwen3. See internal.h. + * Reuses wo/up_proj/down_proj for BERT attention output and FFN matrices. + * Reuses input_norm/post_attn_norm for the two BERT block LayerNorm weights. */ mlx_array o_bias, attn_ln_bias, ffn_inter_bias, ffn_out_bias, ffn_ln_bias; } mlx_layer_t; @@ -44,7 +42,7 @@ struct ffwd_mlx_ctx { mlx_array embed_tokens; mlx_layer_t *layers; /* heap-allocated [n_layers] */ mlx_array norm; - /* BERT family (unset for Qwen3): learned positions, token-type, embed LN. */ + /* BERT family only; unset for Qwen3: learned positions, token-type, embed LN. */ mlx_array position_embeddings, token_type_embeddings, ffwd_ln_w, ffwd_ln_b; mlx_stream stream; multi_safetensors_t *ms; @@ -69,9 +67,7 @@ struct ffwd_mlx_late_vectors { int dim; }; -/* ======================================================================== - * Helpers - * ======================================================================== */ +/* Helpers */ static int arr_ok(mlx_array a) { return a.ctx != NULL; } @@ -158,10 +154,11 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, const in if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim)) return (mlx_array){0}; if (t->dtype == DTYPE_F16) { - /* No F16 forward path on MLX (the matmul fast path keys on BF16). Widen - * to F32 on the host at load so the whole forward runs in F32, mirroring - * the CPU loader. mlx_array_new_data copies its buffer, so the temporary - * is freed immediately. Public F16 snapshots load through here. */ + /* No F16 forward path on MLX; the matmul fast path uses BF16. + * Widen to F32 on the host at load time, so the full forward pass runs in F32, + * matching the CPU loader. + * mlx_array_new_data copies its buffer, so the temporary is freed immediately. + * Public F16 snapshots load through this path. */ int64_t n = safetensor_numel(t); float *tmp = (float *)malloc((size_t)n * sizeof(float)); if (!tmp) @@ -341,9 +338,7 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s) { *a = f; } -/* ======================================================================== - * Load - * ======================================================================== */ +/* Load */ static int mlx_model_dir_has_file(const char *model_dir, const char *rel) { char path[1024]; @@ -394,9 +389,9 @@ static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) { prepare_linear_transpose(&l->down_proj, s); } -/* BERT-family MLX loader, mirroring load_bert_weights on the CPU side. The - * shared post-load below (stream, transposes, optional BF16 upcast) runs after - * either family fills ctx->layers. */ +/* BERT-family MLX loader, matching CPU load_bert_weights. + * The shared post-load step below runs after either family fills ctx->layers: + * stream setup, transposes, and optional BF16 upcast. */ static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer_end) { const ffwd_config_t *c = &ctx->config; int h = c->hidden_size, inter = c->intermediate_size; @@ -625,9 +620,11 @@ static ffwd_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, loaded: ctx->stream = mlx_default_gpu_stream_new(); ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w); - /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the - * tiny norm vectors so they match the F32 activations in mlx_forward_layers; - * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */ + /* Non-quantized BF16: + * run RMSNorm and the residual stream in F32. + * Upcast the small norm vectors to match the F32 activations in + * mlx_forward_layers; keep the large matmul weights in BF16. + * Q8 keeps its existing BF16 norms. */ if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) { for (int i = layer_start; i < layer_end; i++) { mlx_layer_t *l = &ctx->layers[i]; @@ -647,7 +644,7 @@ loaded: upcast_f32_inplace(&l->ffn_ln_bias, ctx->stream); } upcast_f32_inplace(&ctx->norm, ctx->stream); - /* BERT embedding stage (no-ops for Qwen): learned position and token-type + /* BERT embedding stage; no-ops for Qwen: learned position and token-type * embeddings and the embedding LayerNorm all feed the F32 residual. */ upcast_f32_inplace(&ctx->position_embeddings, ctx->stream); upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream); @@ -704,9 +701,7 @@ ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, const ffwd_mlx return mlx_load_range_ex(model_dir, 0, -1, opts, 0); } -/* ======================================================================== - * Free - * ======================================================================== */ +/* Free */ static void free_mlx_layer(mlx_layer_t *l) { free_linear(&l->wq); @@ -827,9 +822,7 @@ const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx) { int ffwd_mlx_late_token_dim(const ffwd_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } -/* ======================================================================== - * Forward pass - * ======================================================================== */ +/* Forward pass */ /* Exact erf GeLU: 0.5 * x * (1 + erf(x / sqrt(2))). Frees nothing it is given. */ static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) { @@ -1569,7 +1562,7 @@ cleanup: /* Device-resident select/concat primitives. The server rerank path batches * candidates through ffwd_mlx_late_encode_docs_device, but the late-interaction - * verification harness (tests/integration/check_late_interaction.py) drives these directly + * verification harness (tests/check_late_interaction.py) drives these directly * to validate per-document gather and concat against the CPU MaxSim reference. */ ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, @@ -2125,7 +2118,7 @@ static int ffwd_mlx_encode_batch_dense(ffwd_mlx_ctx_t *ctx, mlx_array_free(x_normed); } - /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are + /* 5. Eval and copy [B, hidden] to CPU. pplx-embed embeddings are * intentionally unnormalized; callers should use cosine similarity. */ mlx_array emb_f32 = mlx_array_new(); mlx_astype(&emb_f32, emb, MLX_FLOAT32, S); diff --git a/libffwd/mlx.h b/libffwd/mlx.h index e4e2a8e..e2a6e12 100644 --- a/libffwd/mlx.h +++ b/libffwd/mlx.h @@ -1,15 +1,12 @@ /* - * ffwd_mlx.h - MLX GPU backend for Apple Silicon - * - * Uses Apple's mlx-c to run the transformer on Metal. + * Run the transformer on Metal using mlx-c. */ #ifndef FFWD_MLX_H #define FFWD_MLX_H -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying FFWD_API are exported - * from the shared library; everything else stays internal. */ +/* The libraries are built with -fvisibility=hidden, so only declarations carrying + * FFWD_API are exported from the shared library; everything else is internal. */ #ifndef FFWD_API # if defined(__GNUC__) # define FFWD_API __attribute__((visibility("default"))) @@ -79,13 +76,16 @@ FFWD_API void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors); -/* - * Encode n_docs documents in one padded transformer pass and pack each - * document's kept token vectors back-to-back into [total_keep, token_dim] in - * document order. out_offsets[n_docs + 1] receives the prefix sum of kept - * counts, so the result feeds ffwd_mlx_late_maxsim_batch_device directly. - * Returns NULL on error; the handle must not outlive ctx. - */ + +/* Encode n_docs documents in one padded transformer pass. + * + * Packs each document's kept token vectors back-to-back into [total_keep, token_dim] + * in document order. + * + * out_offsets[n_docs + 1] gets the prefix sum of kept counts, so the result can + * feed ffwd_mlx_late_maxsim_batch_device directly. + * + * Returns NULL on error. The handle must not outlive ctx. */ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx, const int *const *doc_ids, const int *n_tokens, @@ -94,9 +94,9 @@ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late int n_docs, int normalize, int *out_offsets); -/* Device-resident select/concat primitives, retained for the late-interaction - * verification harness (tests/integration/check_late_interaction.py); the server rerank - * path uses ffwd_mlx_late_encode_docs_device. */ +/* Device-resident select/concat primitives. + * Kept for the late-interaction verification harness (tests/check_late_interaction.py). + * The server rerank path uses ffwd_mlx_late_encode_docs_device. */ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat( ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count); FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, @@ -124,7 +124,7 @@ FFWD_API int ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding); /* - * Compute a true padded dense batch on MLX. + * Compute a true padded dense batch. * * out_embeddings is caller-provided [batch, hidden_size]. * Pooling and attention masking follow the loaded model configuration. diff --git a/libffwd/model.c b/libffwd/model.c index faecfd4..7346e32 100644 --- a/libffwd/model.c +++ b/libffwd/model.c @@ -8,9 +8,7 @@ #include <stdint.h> #include <string.h> -/* ======================================================================== - * Model-directory probes - * ======================================================================== */ +/* Model-directory probes */ static int model_dir_has_file(const char *model_dir, const char *rel) { char path[1024]; @@ -29,9 +27,7 @@ int model_dir_has_late_projection(const char *model_dir) { model_dir_has_file(model_dir, "1_Dense/model.safetensors"); } -/* ======================================================================== - * Weight tensor loading and validation (direct mmap pointers into safetensors) - * ======================================================================== */ +/* Weight tensor loading and validation (direct mmap pointers into safetensors) */ static size_t dtype_size(safetensor_dtype_t dtype) { switch (dtype) { @@ -125,8 +121,7 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } -/* Widen a safetensors tensor to f32 (caller frees). libsafetensors is a pure - * reader now, so the dtype conversion lives here, next to its only callers. */ +/* Widen a safetensors tensor to f32. Caller frees. */ static float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { int64_t n = safetensor_numel(t); if (n <= 0) @@ -206,15 +201,19 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i return safetensors_get_f32(sf, t); } -/* ======================================================================== - * Model / workspace load and free - * ======================================================================== */ +/* Model / workspace load and free */ -/* BERT-family weight loader. The tensor names differ entirely from the Qwen - * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry - * point. Reused ffwd_layer_t slots carry their BERT meaning: wo is the - * attention output dense, input_norm/post_attn_norm are the two block LayerNorm - * weights, up_proj/down_proj are the two FFN matrices (see internal.h). */ +/* BERT-family weight loader. + * Tensor names differ completely from Qwen + * (encoder.layer.N.*, embeddings.*), so this family has its own entry point. + * + * Reused ffwd_layer_t slots keep their BERT meanings: + * wo is the attention output dense; + * input_norm/post_attn_norm are the two block LayerNorm weights; + * up_proj/down_proj are the two FFN matrices. + * + * See internal.h. + */ static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { const ffwd_config_t *cfg = &model->config; diff --git a/libffwd/model_internal.h b/libffwd/model_internal.h index 97de064..b828e67 100644 --- a/libffwd/model_internal.h +++ b/libffwd/model_internal.h @@ -1,19 +1,24 @@ +/* Cross-file declarations private to CPU model code. + * + * model_types.h holds the shared struct definitions; this header holds the + * shared functions. + * + * These are not part of the public API (ffwd.h) and do not use FFWD_API + * visibility. They are shared between translation units that used to be one + * file: model loading, the forward pass, and late interaction. + * + * Frontends and GPU backends do not include this header. + */ + #ifndef FFWD_MODEL_INTERNAL_H #define FFWD_MODEL_INTERNAL_H #include "model_types.h" #include "alloc.h" -/* Cross-file function declarations private to the CPU model code. The companion - * to model_types.h: that header holds the shared struct definitions, this one the - * shared functions. None are part of the public API (ffwd.h) and none carry - * FFWD_API visibility; they are shared between the translation units (model - * loading, the forward pass, and late interaction) that used to live in one - * file. The frontends and the GPU backends do not include this header. */ - -/* Validate that a safetensor matches an expected dtype/rank/shape and that its - * bytes lie within the mapped file. bf16_ok permits either 16-bit float format - * (BF16 or F16) in addition to F32. */ +/* Validate that a safetensor has the expected dtype, rank, and shape, and that + * its bytes are within the mapped file. bf16_ok also allows either 16-bit float + * format, BF16 or F16, in addition to F32. */ int tensor_has_supported_shape(const safetensors_file_t *sf, const safetensor_t *t, const char *name, @@ -21,10 +26,14 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, int ndim, int bf16_ok); -/* Build a weight ref from a located tensor: F32/BF16 borrow the mmap, F16 is - * widened to an owned F32 buffer (no fused F16 kernel). Shared by the model - * loader and the late-interaction projection loader; the owner frees ref.data - * when ref.owned is set. Returns ref.data == NULL on allocation failure. */ +/* Build a weight ref from a located tensor. + * F32/BF16 borrow the mmap; F16 is widened to an owned F32 buffer because there + * is no fused F16 kernel. + * + * Shared by the model loader and late-interaction projection loader. + * The owner frees ref.data when ref.owned is set. + * + * Returns ref.data == NULL on allocation failure. */ ffwd_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t); /* True if model_dir carries a 1_Dense projection (a late-interaction model). */ @@ -35,9 +44,9 @@ int model_dir_has_late_projection(const char *model_dir); ffwd_model_t * model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late); -/* Workspace scratch growth, lazy per forward call. Defined in model.c beside the - * workspace lifecycle (ffwd_workspace_nbytes mirrors their sizing); called from - * the forward pass in forward.c. */ +/* Workspace scratch grows lazily per forward call. + * Defined in model.c beside the workspace lifecycle. ffwd_workspace_nbytes mirrors this sizing. + * Called from the forward pass in forward.c. */ int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq); void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch); int ensure_rope_cache(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int n_pos); diff --git a/libffwd/model_types.h b/libffwd/model_types.h index 345e0cd..4f01ca2 100644 --- a/libffwd/model_types.h +++ b/libffwd/model_types.h @@ -1,9 +1,10 @@ -/* - * model_types.h - in-memory layout of a loaded model: the model and workspace - * structs, the per-layer weight references, and the accessors that expose them - * to the GPU backends. Shared by the CPU model code and the CUDA backend; not - * part of the public ABI (ffwd.h). Companion to model_internal.h, which - * declares the CPU model's private functions. +/* In-memory layout of a loaded model: model/workspace structs, per-layer weight + * references, and accessors for GPU backends. + * + * Shared by CPU model code and the CUDA backend. Not part of the public ABI + * (ffwd.h). + * + * model_internal.h declares the CPU model's private functions. */ #ifndef FFWD_MODEL_TYPES_H diff --git a/libffwd/neon.c b/libffwd/neon.c index 7f44d82..50cf649 100644 --- a/libffwd/neon.c +++ b/libffwd/neon.c @@ -1,5 +1,5 @@ /* - * neon.c - ARM NEON hot kernels + * ARM NEON hot kernels. */ #include "impl.h" diff --git a/libffwd/norm.c b/libffwd/norm.c index 27fd4c4..57c93db 100644 --- a/libffwd/norm.c +++ b/libffwd/norm.c @@ -1,5 +1,6 @@ /* - * norm.c - Normalization + * Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the + * per-head RMSNorm used for QK-norm. */ #include "kernels.h" @@ -21,12 +22,7 @@ # define M_PI 3.14159265358979323846 #endif -/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the - * per-head RMSNorm used for QK-norm. */ - -/* ======================================================================== - * Normalization - * ======================================================================== */ +/* Normalization */ /* Rows split across the pool only when the tensor is big enough to pay * for the dispatch. */ diff --git a/libffwd/platform.h b/libffwd/platform.h index 0575a43..7d1c7ee 100644 --- a/libffwd/platform.h +++ b/libffwd/platform.h @@ -1,9 +1,11 @@ -/* platform.h - compile-time backend selection. +/* Compile-time backend selection. * * The build passes -DUSE_GPU for a GPU build. This header maps that flag and the - * host platform to a concrete backend: Apple Silicon uses MLX, any other GPU - * uses CUDA, and a non-GPU build is CPU. Use USE_GPU for the CPU/GPU split; - * USE_MLX and USE_CUDA only where a specific accelerator API is called. */ + * host platform to a backend: + * Use USE_GPU for the CPU/GPU split; USE_MLX and USE_CUDA only where a specific + * accelerator API is called. + */ + #ifndef FFWD_PLATFORM_H #define FFWD_PLATFORM_H diff --git a/libffwd/rope.c b/libffwd/rope.c index 12057b2..29cbb17 100644 --- a/libffwd/rope.c +++ b/libffwd/rope.c @@ -1,5 +1,5 @@ /* - * rope.c - rotary position embeddings + * Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ #include "kernels.h" @@ -29,11 +29,7 @@ # define M_PI 3.14159265358979323846 #endif -/* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ - -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ +/* Position embeddings */ void compute_rope_neox( float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { diff --git a/libffwd/safetensors.c b/libffwd/safetensors.c index 0efbd57..f2c9b26 100644 --- a/libffwd/safetensors.c +++ b/libffwd/safetensors.c @@ -9,9 +9,8 @@ #include <sys/stat.h> #include <unistd.h> -/* ======================================================================== - * Minimal JSON parser for safetensors header - * ======================================================================== */ +/* Minimal JSON parser for safetensors header */ +/* TODO: consider replacing with yyjson */ static void skip_whitespace(const char **p) { while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') @@ -262,9 +261,7 @@ static int parse_header(safetensors_file_t *sf) { return 0; } -/* ======================================================================== - * Single file operations - * ======================================================================== */ +/* Single file operations */ safetensors_file_t *safetensors_open(const char *path) { int fd = open(path, O_RDONLY); @@ -343,9 +340,7 @@ int64_t safetensor_numel(const safetensor_t *t) { return n; } -/* ======================================================================== - * Multi-shard operations - * ======================================================================== */ +/* Multi-shard operations */ static int cmp_shard_names(const void *a, const void *b) { const char *sa = (const char *)a; diff --git a/libffwd/safetensors.h b/libffwd/safetensors.h index 13d4cba..f2ea9fd 100644 --- a/libffwd/safetensors.h +++ b/libffwd/safetensors.h @@ -1,5 +1,5 @@ /* - * safetensors.h - Safetensors file format reader (multi-shard support) + * Safetensors file format reader with multi-shard support. */ #ifndef SAFETENSORS_H @@ -40,8 +40,7 @@ typedef struct { safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; } safetensors_file_t; -/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup - */ +/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ typedef struct { safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; int num_shards; diff --git a/libffwd/sentencepiece.c b/libffwd/sentencepiece.c index 2372972..e5e8393 100644 --- a/libffwd/sentencepiece.c +++ b/libffwd/sentencepiece.c @@ -1,5 +1,5 @@ /* - * sentencepiece.c - SentencePiece Unigram tokenizer. + * SentencePiece Unigram tokenizer. * * The implementation reads the small subset of ModelProto needed at runtime: * pieces, scores, piece types, trainer model_type, and normalizer options. The diff --git a/libffwd/sentencepiece.h b/libffwd/sentencepiece.h index 7f5314a..b79f47c 100644 --- a/libffwd/sentencepiece.h +++ b/libffwd/sentencepiece.h @@ -1,8 +1,8 @@ /* - * sentencepiece.h - SentencePiece tokenizer for XLM-R style models. + * SentencePiece tokenizer for XLM-R style models. * - * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`, - * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R + * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model, + * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R * and multilingual E5. It encodes text only; frontends add the model's special * token layout, e.g. <s> ... </s> for XLM-R. */ diff --git a/libffwd/threadpool.c b/libffwd/threadpool.c index 2521a6d..6cb4252 100644 --- a/libffwd/threadpool.c +++ b/libffwd/threadpool.c @@ -1,5 +1,5 @@ /* - * tp.c - CPU thread pool + * CPU thread pool. */ #if defined(__linux__) && !defined(_GNU_SOURCE) @@ -116,15 +116,19 @@ void tp_set_threads(int n) { } #ifndef __APPLE__ -/* CPU bandwidth this process may actually use, in whole cores, or 0 if there - * is no cgroup limit. Containers (Docker --cpus, Kubernetes, RunPod) expose - * every host CPU through sysconf()/cpuset, but the CFS quota caps the CPU time - * the process really gets. Sizing the BLAS/thread pool to the host count then - * oversubscribes that quota: the threads spend the period's budget early and - * the scheduler throttles them for the rest of each period, which adds large, - * bursty stalls (e.g. a 16-thread default against a 5.1-core quota ran the - * 0.6B model ~2x slower than a 5-thread pool). Floor, not ceil, so a fully - * busy pool stays under the quota and never trips throttling. */ +/* CPU quota available to this process, in whole cores, or 0 if uncapped. + * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host + * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process + * can actually use. + * + * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota: + * threads use the period budget early, then get throttled for the rest of the + * period, causing large bursty stalls. Example: a 16-thread default against a + * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool. + * + * Floor instead of ceil, so a fully busy pool stays under quota and avoids + * throttling. + */ static int cgroup_cpu_quota(void) { /* cgroup v2: "/sys/fs/cgroup/cpu.max" is "<quota> <period>", or * "max <period>" when unlimited. */ diff --git a/libffwd/threadpool.h b/libffwd/threadpool.h index 1dcc80a..f9739a1 100644 --- a/libffwd/threadpool.h +++ b/libffwd/threadpool.h @@ -1,12 +1,14 @@ #ifndef TP_THREADPOOL_H #define TP_THREADPOOL_H -/* Persistent worker pool for the CPU math kernels: a fixed set of threads that - * sleep until parallel_for() hands them a callback, plus the CPU-count helper - * that sizes the pool (cgroup-quota aware). Separated from the math so the - * kernels depend only on the parallel_for / tp_num_threads interface, not on - * the pool's internals. kernels.h includes this, so kernel callers need no - * extra include. */ +/* Persistent worker pool for CPU kernels. + * Uses a fixed thread set that sleeps until parallel_for() gives it a callback. + * Also provides the cgroup-quota-aware CPU-count helper used to size the pool. + * + * Kept separate from the kernels so they depend only on parallel_for and + * tp_num_threads, not pool internals. + * + * kernels.h includes this, so kernel callers need no extra include. */ /* Worker callback: (thread index, total threads, user arg). */ typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c index df5861e..2af151a 100644 --- a/libffwd/tokenize.c +++ b/libffwd/tokenize.c @@ -1,10 +1,14 @@ /* - * tokenize.c - model-aware tokenization: text -> the token ids a model expects. + * Model-aware tokenization: text -> token ids the model expects. * - * Backend-independent, so this is its own handle (ffwd_tok_t), not part of the - * per-backend ffwd_t. ffwd_tok_open picks the tokenizer by the files present - * and resolves the model's special-token ids; ffwd_tokenize applies that - * model's layout (BERT/XLM-R [CLS]..[SEP], Qwen appended terminal token). + * Backend-independent, so this uses its own handle, ffwd_tok_t, instead of the + * per-backend ffwd_t. + * + * ffwd_tok_open selects the tokenizer from the files present and resolves the + * model's special-token ids. + * + * ffwd_tokenize applies the model's token layout: BERT/XLM-R [CLS]..[SEP], or + * Qwen appended terminal token. */ #include "ffwd.h" diff --git a/libffwd/vec.c b/libffwd/vec.c index f848be0..ec6b27d 100644 --- a/libffwd/vec.c +++ b/libffwd/vec.c @@ -2,13 +2,7 @@ #include <math.h> -/* ======================================================================== - * Vector helpers - * - * Standalone post-processing on embedding vectors: callers (CLI, server) use - * these to L2-normalize outputs and to score pooled embeddings with cosine - * similarity. They touch no model or tensor internals, only the public API. - * ======================================================================== */ +/* Vector helpers */ int ffwd_l2_normalize(float *vec, int dim) { if (!vec || dim <= 0) diff --git a/libffwd/wordpiece.c b/libffwd/wordpiece.c index d3f62dd..dc0d995 100644 --- a/libffwd/wordpiece.c +++ b/libffwd/wordpiece.c @@ -1,21 +1,24 @@ /* - * tok_wp_tokenizer.c - BERT-family WordPiece tokenizer. + * BERT-family WordPiece tokenizer. * - * Pipeline (matching transformers' BertTokenizer): + * Pipeline matching transformers' BertTokenizer: * BasicTokenizer: clean control/whitespace -> space CJK ideographs -> - * whitespace split -> (lowercase + strip accents) -> split on punctuation - * WordPiece: greedy longest-match-first against vocab.txt, continuation - * pieces marked "##", whole token -> [UNK] when any piece fails to match. + * whitespace split -> lowercase + strip accents, if enabled -> + * split on punctuation. + * WordPiece: greedy longest-match-first against vocab.txt, with continuation + * pieces marked "##". If any piece fails, the whole token becomes [UNK]. * - * Unicode handling is full and exact against transformers, driven by generated - * tables (tok_wp_unicode.h): punctuation and control by Unicode category, - * whitespace by the complete Zs set, the CJK split by the BERT ideograph ranges, - * and the do_lower_case fold by a lowercase+NFD+strip-marks table (with Hangul - * decomposed by formula). Cased and uncased multilingual encoders (e.g. - * bert-base-multilingual-cased / -uncased) match Hugging Face exactly, including - * Greek, Cyrillic, Vietnamese and Korean. The fold is per-codepoint, matching - * transformers, which lowercases character by character (no context-sensitive - * Greek final sigma). + * Unicode handling fully matches transformers using generated tables + * (unicode.h): Unicode category for punctuation/control, full Zs set for + * whitespace, BERT ideograph ranges for CJK splitting, and a lowercase + NFD + + * strip-marks table for do_lower_case, with Hangul decomposed by formula. + * + * Cased and uncased multilingual encoders, such as + * bert-base-multilingual-cased / -uncased, match Hugging Face exactly, + * including Greek, Cyrillic, Vietnamese, and Korean. + * + * Folding is per-codepoint, matching transformers, which lowercases one + * character at a time, with no context-sensitive Greek final sigma. */ #include "wordpiece.h" @@ -27,7 +30,7 @@ #include <stdlib.h> #include <string.h> -/* ===================== string -> int open-addressing map ===================== */ +/* string -> int open-addressing map */ typedef struct { const char *key; /* points into tok->id_to_token; not owned */ @@ -55,7 +58,7 @@ static void map_put(wp_entry_t *map, int cap, const char *key, int val) { int pos = (int)(fnv1a_hash(key) & mask); while (map[pos].key) { if (!strcmp(map[pos].key, key)) { - map[pos].val = val; /* keep the first (lowest) id, like a dict insert */ + map[pos].val = val; /* keep the first/lowest id, like a dict insert */ return; } pos = (int)((pos + 1) & (int)mask); @@ -77,10 +80,10 @@ static int map_get(const wp_entry_t *map, int cap, const char *key) { return -1; } -/* ============================== UTF-8 helpers ================================ */ +/* UTF-8 helpers */ -/* Decode one codepoint. Returns bytes consumed (>=1); on a malformed lead byte - * emits U+FFFD and consumes one byte so iteration always advances. */ +/* Decode one codepoint. Returns bytes consumed (>=1). + * On malformed lead byte, emits U+FFFD and consumes one byte so iteration advances. */ static int utf8_decode(const unsigned char *p, const unsigned char *end, uint32_t *cp) { unsigned char c = p[0]; if (c < 0x80) { @@ -140,7 +143,7 @@ static int utf8_encode(uint32_t cp, char out[4]) { return 4; } -/* ========================== Unicode classification ========================== */ +/* Unicode classification */ static int wp_is_whitespace(uint32_t cp) { if (cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r') @@ -164,9 +167,9 @@ static int wp_in_ranges(uint32_t cp, const wp_range_t *r, int n) { return 0; } -/* transformers' _is_control: tab/newline/carriage-return are whitespace, not - * control (they fall through to wp_is_whitespace); everything else is Unicode - * category Cc, Cf, or Co. */ +/* Match transformers' _is_control: + * tab/newline/carriage-return are whitespace, not control, and fall through to + * wp_is_whitespace; everything else is Unicode category Cc, Cf, or Co. */ static int wp_is_control(uint32_t cp) { if (cp == '\t' || cp == '\n' || cp == '\r') return 0; @@ -184,10 +187,10 @@ static int wp_is_cjk(uint32_t cp) { (cp >= 0xF900 && cp <= 0xFAFF) || (cp >= 0x2F800 && cp <= 0x2FA1F); } -/* Hangul syllable decomposition constants (the standard algorithm). Hangul - * syllables are not cased, so their do_lower_case fold is the canonical NFD into - * leading (L), vowel (V), and optional trailing (T) jamo - none combining marks - * - which keeps them out of the generated WP_FOLD table. */ +/* Hangul syllable decomposition constants for the standard algorithm. + * Hangul syllables are not cased, so their do_lower_case fold is canonical NFD: + * leading (L), vowel (V), and optional trailing (T) jamo. + * None are combining marks, so they stay out of the generated WP_FOLD table. */ enum { WP_HANGUL_S = 0xAC00, WP_HANGUL_L = 0x1100, @@ -198,10 +201,12 @@ enum { WP_HANGUL_SCOUNT = 11172 }; -/* Apply the do_lower_case fold (Python str.lower then NFD then drop combining - * marks) to one codepoint, writing 0..3 output codepoints to out and the count - * to *n_out. 0 outputs means the codepoint is a dropped mark. Hangul decomposes - * by formula; everything else is the generated WP_FOLD table or an identity. */ +/* Apply do_lower_case to one codepoint: + * lowercase using Python-compatible str.lower behavior, then NFD, + * then drop combining marks. + * Writes 0..3 output codepoints to out and the count to *n_out. + * 0 outputs means the codepoint was a dropped mark. + * Hangul decomposes by formula; all else uses WP_FOLD or identity. */ static void wp_fold(uint32_t cp, uint32_t out[3], int *n_out) { if (cp >= WP_HANGUL_S && cp < WP_HANGUL_S + WP_HANGUL_SCOUNT) { int s = (int)(cp - WP_HANGUL_S); @@ -235,7 +240,7 @@ static void wp_fold(uint32_t cp, uint32_t out[3], int *n_out) { *n_out = 1; } -/* =============================== workspace ================================== */ +/* workspace */ struct tok_wp_workspace { uint32_t *cps; /* cleaned + CJK-spaced codepoints of the whole input */ @@ -310,7 +315,7 @@ void tok_wp_workspace_free(tok_wp_workspace_t *ws) { free(ws); } -/* ============================== WordPiece core ============================== */ +/* WordPiece core */ /* Append the WordPiece ids of one BasicTokenizer sub-word (codepoints * word[0..n)) to ws->ids[*n_out]. The whole sub-word maps to [UNK] when it is @@ -381,8 +386,8 @@ tok_wp_subword(const tok_wp_t *tok, tok_wp_workspace_t *ws, const uint32_t *word return 0; } -/* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. Returns the - * token count, or -1 on allocation failure. */ +/* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. + * Returns the token count, or -1 on allocation failure. */ static int tok_wp_run(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text) { /* clean_text + tokenize_chinese_chars, in one pass over the input. */ const unsigned char *p = (const unsigned char *)text; @@ -460,7 +465,7 @@ static int tok_wp_run(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *t return n_out; } -/* ================================ loading =================================== */ +/* loading */ static char *read_file(const char *path, long *out_len) { FILE *f = fopen(path, "rb"); @@ -493,7 +498,7 @@ static char *read_file(const char *path, long *out_len) { } /* do_lower_case from config.json: default true (BERT uncased). A - * lightweight scan is enough - we only need one boolean and avoid a JSON dep + * lightweight scan is enough; we only need one boolean and avoid a JSON dep * here, mirroring how the config layer reads small flags. */ static int read_do_lower_case(const char *model_dir) { char path[2048]; @@ -595,7 +600,7 @@ tok_wp_t *tok_wp_load(const char *model_dir) { return tok; } -/* ============================== public API ================================= */ +/* public API*/ const char *tok_wp_decode(const tok_wp_t *tok, int id) { if (!tok || id < 0 || id >= tok->vocab_size) diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h index c93140a..f8a63d6 100644 --- a/libffwd/wordpiece.h +++ b/libffwd/wordpiece.h @@ -1,15 +1,16 @@ /* - * wordpiece.h - BERT-family WordPiece tokenizer. + * BERT-family WordPiece tokenizer. * - * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional - * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting) - * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes - * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the - * byte-level BPE backend leaves terminal/separator tokens to the frontend. + * Implements BERT tokenization: BasicTokenizer (cleanup, optional lowercase + * and accent stripping, CJK spacing, whitespace splitting, and punctuation + * splitting), then greedy longest-match WordPiece using vocab.txt. * - * The public surface mirrors bpe.h (load / encode with a reusable - * workspace / decode / resolve a named special token / free) so both can sit - * behind one tokenizer interface later. + * Encodes text only. The frontend adds [CLS]/[SEP], just as the byte-level BPE + * backend leaves terminal and separator tokens to the frontend. + * + * Public API mirrors bpe.h: load, encode with reusable workspace, decode, + * resolve named special tokens, and free. This lets both tokenizers share one + * tokenizer interface later. */ #ifndef TOK_WP_H @@ -29,28 +30,29 @@ typedef struct { typedef struct tok_wp_workspace tok_wp_workspace_t; -/* Load from <model_dir>/vocab.txt, reading do_lower_case from - * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on - * error (missing vocab.txt, missing [UNK], allocation failure). */ +/* Load <model_dir>/vocab.txt and do_lower_case from + * <model_dir>/tokenizer_config.json; defaults to lowercase. + * Returns NULL on error: missing vocab.txt, missing [UNK], or allocation failure. */ tok_wp_t *tok_wp_load(const char *model_dir); -/* Decode a single id to its vocab token string (internal pointer), or NULL. */ +/* Decode one id to its vocab token string, or NULL. + * Returns an internal pointer. */ const char *tok_wp_decode(const tok_wp_t *tok, int id); /* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */ int tok_wp_token_id(const tok_wp_t *tok, const char *token); -/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on - * error (and *out_n_tokens = 0). */ +/* Encode UTF-8 text into a malloc'd id array and set *out_n_tokens. + * Returns NULL on error and sets *out_n_tokens = 0. */ int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n_tokens); -/* Reusable scratch for allocation-light repeated encoding. */ +/* Reusable scratch space for repeated encoding with fewer allocations. */ tok_wp_workspace_t *tok_wp_workspace_new(void); void tok_wp_workspace_free(tok_wp_workspace_t *ws); -/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1 - * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to - * the number of tokens needed/emitted in all cases. */ +/* Encode into caller-provided out_ids[0..out_cap-1]. + * Returns 0 on success, -1 on error, or -2 if out_cap is too small. + * If non-NULL, *out_n_tokens is always set to tokens needed/emitted. */ int tok_wp_encode_into(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text, @@ -58,8 +60,8 @@ int tok_wp_encode_into(const tok_wp_t *tok, int out_cap, int *out_n_tokens); -/* Same tokenization as tok_wp_encode(), reusing ws internally and - * performing only the final returned int[] allocation. */ +/* Same tokenization as tok_wp_encode(). + * Reuses ws internally and only allocates the returned int[]. */ int *tok_wp_encode_with_workspace(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text, |
