diff options
96 files changed, 792 insertions, 746 deletions
diff --git a/bench/bench.h b/bench/bench.h index 8be685c..2c11fa4 100644 --- a/bench/bench.h +++ b/bench/bench.h @@ -1,4 +1,4 @@ -/* bench/bench.h - minimal microbenchmark harness. +/* Minimal microbenchmark harness. * * A benchmark function receives a bench_state_t and must run its measured * loop exactly b->n times. Call bench_begin(b) after setup so allocation diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c index 55f859b..d302396 100644 --- a/bench/bench_kernels.c +++ b/bench/bench_kernels.c @@ -1,10 +1,12 @@ -/* bench/bench_kernels.c - regression microbenchmarks for the hot kernels. +/* Regression microbenchmarks for the hot kernels. * * Shapes are the real pplx-embed-v1-0.6b dimensions (hidden 1024, 16 query / * 8 KV heads x head_dim 128, intermediate 3072) so a slowdown here predicts - * a slowdown in inference. Single-threaded by default for stable numbers; - * --threads N opts into the pool. Build and record via `make bench`, - * compare records with bench/benchstat.py. + * a slowdown in inference. + * + * Single-threaded by default for stable numbers; --threads N opts into the pool. + * + * Build and record via make bench, compare records with bench/benchstat.py. */ #include "bench.h" @@ -133,7 +135,7 @@ static void bm_bf16_widen(bench_state_t *b) { free(src); } -/* ---- attention pieces ---- */ +/* attention pieces */ static void bm_attn_scores(bench_state_t *b) { /* one head: scores[seq,seq] = Q[seq,hd] @ K[seq,hd]^T */ @@ -171,7 +173,7 @@ static void bm_rope(bench_state_t *b) { free(x); } -/* ---- norms, activations, reductions ---- */ +/* norms, activations, reductions */ static void bm_rms_norm(bench_state_t *b) { float *x = alloc_f32((size_t)SEQ * HIDDEN, 23); diff --git a/bench/bench_late_rerank.c b/bench/bench_late_rerank.c index 987874c..88fc611 100644 --- a/bench/bench_late_rerank.c +++ b/bench/bench_late_rerank.c @@ -1,9 +1,13 @@ -/* bench/bench_late_rerank.c - late-interaction MaxSim rerank benchmark. - * Build via `make bench-late-rerank`; run with: +/* Late-interaction MaxSim rerank benchmark. + * + * Build via make bench-late-rerank; run with: * ./bench_late_rerank MODEL_DIR CANDIDATES DOC_REPEAT RUNS + * * Encodes a query and N candidate documents to token vectors, then times the * packed MaxSim rerank. check it against bench_late_rerank.py, which builds and - * runs this binary. */ + * runs this binary. + */ + #include "internal.h" #ifdef CHECK_MLX # include "mlx.h" diff --git a/bench/bench_late_rerank.py b/bench/bench_late_rerank.py index 99bb817..489ea58 100755 --- a/bench/bench_late_rerank.py +++ b/bench/bench_late_rerank.py @@ -2,7 +2,7 @@ """Benchmark late-interaction MaxSim reranking shapes. Builds the C bench (bench/bench_late_rerank.c) via the Makefile and runs it -against a late model directory (e.g. pplx-embed-v1-late-0.6b). python3 only. +against a late model directory (e.g. pplx-embed-v1-late-0.6b). bench/bench_late_rerank.py --model-dir DIR """ diff --git a/bench/bench_matrix.py b/bench/bench_matrix.py index 7e0d72c..e0d99e2 100755 --- a/bench/bench_matrix.py +++ b/bench/bench_matrix.py @@ -15,7 +15,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -# Each backend is its own binary now: `make <backend>` builds the CLI into +# Each backend is its own binary now: make <backend> builds the CLI into # build/release/<dir>/ffwd-cli (and refreshes the ./ffwd-cli root symlink). BUILD_DIR = {"cpu": "blas", "mlx": "mlx", "cuda": "cuda"} diff --git a/bench/bench_model.c b/bench/bench_model.c index 7cb6458..a274285 100644 --- a/bench/bench_model.c +++ b/bench/bench_model.c @@ -1,11 +1,11 @@ -/* bench/bench_model.c - end-to-end embedding throughput against real - * weights, for regression tracking (kernels are covered by bench_kernels.c; - * this catches slowdowns in the orchestration between them). +/* End-to-end embedding throughput against real weights, for regression tracking. * + * Kernels are covered by bench_kernels.c; this catches slowdowns in the orchestration + * between them: * make bench-model MODEL_DIR=/path/to/pplx-embed-v1-0.6b * - * Uses deterministic synthetic token ids, so only config.json and - * model.safetensors are needed (no tokenizer). Single-threaded by default; + * Uses deterministic synthetic token ids, so only config.json and model.safetensors + * are needed (no tokenizer). Single-threaded by default; * --threads N opts into the pool. */ diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index a5a16ee..7c06bdd 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -1,5 +1,6 @@ -/* bench/bench_tokenizer.c - tokenizer encode-path benchmark. - * Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */ +/* Tokenizer encode-path benchmark. + * Build via make bench-tokenizer; accepts a model dir or vocab.json. + */ #include "bpe.h" #include "sentencepiece.h" diff --git a/bench/benchstat.py b/bench/benchstat.py index 7f8b695..8036c97 100755 --- a/bench/benchstat.py +++ b/bench/benchstat.py @@ -4,13 +4,20 @@ usage: benchstat.py OLD.json NEW.json [--alpha 0.05] [--threshold 3.0] -Per benchmark: median ns/op of both sample sets, the delta, and a -Mann-Whitney U significance test. A delta only counts when it is BOTH -significant at --alpha AND at least --threshold percent: back-to-back -runs of an unchanged binary drift a couple of percent with machine state -(thermals, core scheduling), so smaller shifts are below the -reproducibility floor even when statistically clean. Everything else -prints as "~" - treat it as noise, not improvement or regression. +Per benchmark: +median ns/op for both sample sets, the delta, and a Mann-Whitney U +significance test. + +A delta counts only when it is both: +- significant at --alpha +- at least --threshold percent + +Back-to-back runs of an unchanged binary can drift a few percent with machine +state, such as thermals and core scheduling. Smaller shifts are below the +reproducibility floor even when statistically clean. + +Everything else prints as "~"; treat it as noise, not improvement or regression. + Exits 1 if any benchmark regresses past both bars. """ @@ -10,11 +10,14 @@ BACKEND ?= blas MODE ?= release prefix ?= /usr/local -# Per-configuration build tree under the repo root. Each MODE/BACKEND pair gets -# its own objects and artifacts, so switching configs never reuses another's -# output and needs no clean in between. A consumer sets ROOT (path to the repo -# root) and COMPONENT (a unique name) before including this file; objects land in -# $(OBJDIR), final artifacts in $(OUTDIR). +# Per-configuration build tree under the repo root. +# Each MODE/BACKEND pair gets its own objects and artifacts, so switching configs +# never reuses another config's output and needs no clean. +# +# A consumer sets ROOT, the repo root path, and COMPONENT, a unique name, before +# including this file. +# +# Objects go in $(OBJDIR); final artifacts go in $(OUTDIR). ROOT ?= . COMPONENT ?= misc BUILDREL := build/$(MODE)/$(BACKEND) @@ -34,8 +37,7 @@ ifeq ($(strip $(OPENBLAS_LDFLAGS)),) OPENBLAS_LDFLAGS = -lopenblas endif -# yyjson is vendored under deps/yyjson and built into a per-config static archive -# (deps/yyjson/Makefile), so there is no system or pkg-config dependency. +# yyjson is vendored under deps/yyjson and built into a per-config static archive. # YYJSON_CFLAGS resolves <yyjson.h> to the vendored header; YYJSON_LDFLAGS names # the archive by full path (as the server links libae.a), which links the same # way on macOS and Linux. Consumers build it via $(MAKE) -C deps/yyjson. @@ -45,20 +47,24 @@ YYJSON_LDFLAGS := $(OUTDIR)/libyyjson.a MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null) MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null) -# Allow FP reassociation and contraction so reduction loops vectorize, but do -# not assume finite math or approximate libm calls: full -ffast-math produced -# NaN embeddings on GCC/OpenBLAS Linux builds (-ffinite-math-only and -# -fapprox-func are the unsafe parts and stay off). -# -fvisibility=hidden pairs with FFWD_API in the public headers: a new -# public function without the annotation will be missing from the shared lib. -# _DEFAULT_SOURCE exposes POSIX/BSD declarations (strdup, strndup, ...) under -# -std=c11: strict C11 defines __STRICT_ANSI__, so glibc otherwise hides them, -# and an implicit int return truncates the 64-bit pointer (crashes on Linux; -# macOS declares them regardless, which hid this). +# Allow FP reassociation and contraction so reduction loops vectorize, without +# assuming finite math or approximate libm calls. +# +# Full -ffast-math produced NaN embeddings on GCC/OpenBLAS Linux builds. +# The unsafe parts, -ffinite-math-only and -fapprox-func, stay off. +# +# -fvisibility=hidden pairs with FFWD_API in public headers. A new public +# function without FFWD_API will be missing from the shared library. +# +# _DEFAULT_SOURCE exposes POSIX/BSD declarations such as strdup and strndup under +# -std=c11. Strict C11 defines __STRICT_ANSI__, so glibc otherwise hides them. +# An implicit int return then truncates the 64-bit pointer and crashes on Linux. +# macOS declares them regardless, which hid this. +# # -Werror=implicit-function-declaration turns a missing prototype into a build -# failure instead of a buried -Wall warning: that is exactly how the strdup -# truncation above slipped through (an implicit int return -> mangled pointer, -# crashing only on Linux). It is a hard error in C23 anyway. +# failure instead of a buried -Wall warning. That is how the strdup truncation +# above slipped through: implicit int return -> mangled pointer -> Linux-only +# crash. It is also a hard error in C23. CFLAGS_BASE = -Wall -Wextra -O3 $(ARCH_FLAGS) -fPIC -fvisibility=hidden \ -D_DEFAULT_SOURCE -Werror=implicit-function-declaration \ -fno-math-errno -ffp-contract=fast -fno-trapping-math \ @@ -99,13 +105,14 @@ else HOST_BLAS_LDFLAGS = $(OPENBLAS_LDFLAGS) endif -# Each backend defines only the flags its own compiled sources need. -# blas: the CPU build; the kernel files call cblas, so USE_BLAS is on. -# mlx: the Apple GPU build runs all math on the GPU and compiles no kernel -# files, so it defines neither USE_BLAS nor links Accelerate. USE_GPU selects -# MLX via platform.h. -# cuda: the NVIDIA GPU build still compiles the CPU kernel files for host-side -# work, so it keeps USE_BLAS (OpenBLAS) alongside the CUDA libraries. +# Each backend defines only the flags needed by its compiled sources. +# +# - blas: CPU build: Kernel files call cblas, so USE_BLAS is enabled. +# - mlx: Apple Metal build: All math runs on the GPU and no kernel files are +# compiled, so it defines neither USE_BLAS nor links Accelerate. +# USE_GPU selects MLX through platform.h. +# - cuda: NVIDIA CUDA build: Still compiles CPU kernel files for host-side work, +# so it keeps USE_BLAS/OpenBLAS along with the CUDA libraries. BLAS_BACKEND_CFLAGS = $(HOST_BLAS_CFLAGS) BLAS_BACKEND_LDFLAGS = $(HOST_BLAS_LDFLAGS) MLX_BACKEND_CFLAGS = -DUSE_GPU -I$(MLXC_PREFIX)/include diff --git a/devtools/container-build-test.sh b/devtools/container-build-test.sh index 3437868..24c8f81 100755 --- a/devtools/container-build-test.sh +++ b/devtools/container-build-test.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash -# Build, test and bench the OpenBLAS backend in an Ubuntu container to catch -# Linux portability breakage a macOS/Accelerate build hides. -# It finds the repo root via git, bind-mounts it at /work, installs the Runpod -# toolchain, re-execs itself inside with --in-container, and runs -# make cpu/test/bench/bench-server-utils/debug, then `make clean`. -# Skip clean with KEEP=1. Override the image with IMAGE=ubuntu:22.04. +# Build, test, and benchmark the OpenBLAS backend in an Ubuntu container. +# Finds the repo root with git, bind-mounts it at /work, installs the RunPod +# toolchain, re-execs itself inside with --in-container, and runs: +# make cpu/test/bench/bench-server-utils/debug, then make clean. +# Set KEEP=1 to skip clean. Set IMAGE=ubuntu:22.04 to override the image. set -euo pipefail IMAGE="${IMAGE:-ubuntu:24.04}" @@ -69,7 +68,7 @@ run_in_container() { echo "==================== ALL STEPS COMPLETED ====================" } -# --- host side: launch the container and re-exec this script inside it -------- +# Launch the container and re-exec this script inside it. launch_on_host() { command -v container >/dev/null 2>&1 || { echo "error: Apple 'container' CLI not found on PATH" >&2; exit 1; } diff --git a/libffwd/Makefile b/libffwd/Makefile index 955f93e..2372697 100644 --- a/libffwd/Makefile +++ b/libffwd/Makefile @@ -9,15 +9,15 @@ SHARED := $(OUTDIR)/lib$(LIB).$(SHARED_EXT) CFLAGS += -std=c11 -pthread $(BACKEND_CFLAGS) CPPFLAGS += -I. SHARED_FLAGS += -pthread + # Restrict the shared library's exported symbols to the public ABI. SHARED_LDFLAGS += $(EXPORT_FLAGS) -# Tokenizer, config, safetensors reader, and L2/cosine helpers: every backend. +# Tokenizer, config, safetensors reader, and vector helpers. COMMON_SRCS := config.c vec.c tokenize.c bpe.c sentencepiece.c wordpiece.c safetensors.c -# CPU math kernels and SIMD dispatch. generic.c is always built; avx.c and -# neon.c each guard their entire body by arch (__AVX2__ / __ARM_NEON), so the -# off-target file compiles to an empty translation unit. +# CPU kernels and SIMD dispatch. +# generic.c is always built; avx.c and neon.c each guarded by arch (__AVX2__ / __ARM_NEON). KERNEL_SRCS := act.c gemm.c norm.c rope.c attn.c threadpool.c generic.c avx.c neon.c BLAS_SRCS := backend_blas.c model.c model_blas.c forward.c late.c late_blas.c alloc.c \ @@ -45,22 +45,17 @@ all: $(STATIC) $(SHARED) $(eval $(call static_archive,$(STATIC),$(OBJS))) $(eval $(call shared_library,$(SHARED),$(OBJS),,$(BACKEND_LDFLAGS) $(YYJSON_LDFLAGS) -lm)) -# yyjson is vendored under deps/yyjson; build its archive before linking the -# shared library (the static libffwd.a does not embed it — consumers link both). + YYJSON := $(ROOT)/deps/yyjson $(OUTDIR)/libyyjson.a: $(MAKE) -C $(YYJSON) $(SHARED): $(OUTDIR)/libyyjson.a -# The export map is generated from the public ABI (the FFWD_API declarations in -# ffwd.h) instead of being committed, so it cannot drift from the header. It is -# regenerated when the header or the generator changes, and the shared library -# relinks when it does. + +# The export map is auto-generated from the FFWD_API declarations in ffwd.h. $(EXPORT_FILE): ffwd.h $(ROOT)/devtools/gen_exports.py python3 $(ROOT)/devtools/gen_exports.py --outdir $(ROOT) $(SHARED): $(EXPORT_FILE) -# Removes only this MODE/BACKEND's objects and artifacts; the root `make clean` -# removes the whole build/ tree. clean: $(RM) -r $(OBJDIR) $(STATIC) $(SHARED) $(SHARED).dSYM diff --git a/libffwd/act.c b/libffwd/act.c index 688dd71..2495311 100644 --- a/libffwd/act.c +++ b/libffwd/act.c @@ -1,6 +1,4 @@ -/* - * act.c - activations - */ +/* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU variants. */ #include <math.h> #include <stdlib.h> @@ -29,13 +27,6 @@ # define M_PI 3.14159265358979323846 #endif -/* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU - * variants. */ - -/* ======================================================================== - * Activation Functions - * ======================================================================== */ - void silu_mul_inplace(float *gate, const float *up, int n) { #if defined(__APPLE__) && defined(USE_BLAS) enum { CHUNK = 4096 }; diff --git a/libffwd/alloc.c b/libffwd/alloc.c index 6af854a..828258e 100644 --- a/libffwd/alloc.c +++ b/libffwd/alloc.c @@ -4,10 +4,11 @@ #include <stdint.h> #include <limits.h> -/* Sequence-capacity growth policy for the forward-pass workspaces: for new - * growth, never allocate fewer than the minimum, and round up to the - * granularity so small length changes do not trigger a realloc on every call. - * Existing larger capacity is preserved. */ +/* Sequence-capacity growth policy for the forward-pass workspaces: + * For new growth: + * - never allocate less than the minimum required size. + * - round up to the granularity so small length changes do not trigger a realloc on every call. + * - keep existing capacity if it is already larger. */ #define FFWD_MIN_WORKSPACE_SEQ_CAP 16 #define FFWD_WORKSPACE_SEQ_GRANULARITY 16 diff --git a/libffwd/alloc.h b/libffwd/alloc.h index 0712e21..a12ee7a 100644 --- a/libffwd/alloc.h +++ b/libffwd/alloc.h @@ -3,11 +3,9 @@ #include <stddef.h> -/* Overflow-checked size arithmetic and typed (re)allocation helpers shared by - * the model loader, the forward pass, and the late-interaction code. Every - * helper returns -1 (or NULL) on overflow or allocation failure so callers can - * bail without a partial buffer. realloc_* keep the original pointer intact on - * failure; the caller still owns it. */ +/* Checked size arithmetic and typed allocation helpers. + * Return -1 or NULL on overflow/allocation failure. + * realloc_* leave the original pointer unchanged on failure. */ /* out = a * b, or -1 if the product overflows size_t. */ int mul_size(size_t a, size_t b, size_t *out); @@ -15,20 +13,19 @@ int mul_size(size_t a, size_t b, size_t *out); /* out = a + b, or -1 if the sum overflows size_t. */ int add_size(size_t a, size_t b, size_t *out); -/* Round a requested sequence capacity up to the workspace growth granularity, - * never shrinking below the current capacity. Returns -1 on bad input/overflow. */ +/* Grow sequence capacity without shrinking. */ int grow_cap(int current, int needed, int *out); -/* realloc *ptr to hold `count` floats; *ptr unchanged on failure. */ +/* realloc *ptr to hold count floats; *ptr unchanged on failure. */ int realloc_floats(float **ptr, size_t count); -/* malloc `count` floats, NULL on overflow/failure. */ +/* malloc count floats, NULL on overflow/failure. */ float *malloc_floats(size_t count); /* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */ int realloc_floats_2d(float **ptr, int rows, int cols); -/* realloc *ptr to hold `count` ints; *ptr unchanged on failure. */ +/* realloc *ptr to hold count ints; *ptr unchanged on failure. */ int realloc_ints(int **ptr, size_t count); #endif /* FFWD_ALLOC_H */ diff --git a/libffwd/attn.c b/libffwd/attn.c index 6134064..57c665a 100644 --- a/libffwd/attn.c +++ b/libffwd/attn.c @@ -1,7 +1,3 @@ -/* - * attn.c - attention - */ - #include "kernels.h" #include "impl.h" #include "threadpool.h" @@ -25,9 +21,7 @@ /* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE) * GQA, each with an online-softmax and a BLAS-GEMM path. */ -/* ======================================================================== - * Attention Operations - * ======================================================================== */ +/* Attention operations */ float dot_f32(const float *a, const float *b, int n) { return dot_f32_impl(a, b, n); } diff --git a/libffwd/avx.c b/libffwd/avx.c index 41259a1..7cbbd5a 100644 --- a/libffwd/avx.c +++ b/libffwd/avx.c @@ -1,12 +1,14 @@ -/* - * avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) +/* x86 SIMD hot kernels: AVX2+FMA, plus AVX-512 when available. * - * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. - * Uses AVX-512F+BW for 16-wide bf16 matvec (dominant for seq_len==1), and - * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data). + * BF16 -> F32 conversion: + * load 16 uint16 values, zero-extend to 32-bit, then shift left 16. * - * The bf16 matvec processes 4 output rows simultaneously to reduce instruction - * overhead and improve out-of-order execution on memory-bound workloads. + * Uses AVX-512F+BW for 16-wide BF16 matvec, dominant when seq_len == 1. + * Uses AVX2+FMA / AVX-512F for F32 attention helpers, which operate on + * cache-resident data. + * + * BF16 matvec processes 4 output rows at once to reduce instruction overhead + * and improve out-of-order execution on memory-bound workloads. */ #include "impl.h" @@ -16,9 +18,7 @@ # include <immintrin.h> # include <string.h> -/* ===================================================================== - * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise - * ===================================================================== */ +/* BF16 matvec: uses AVX-512F+BW when available, AVX2 fallback otherwise */ # if defined(__AVX512F__) && defined(__AVX512BW__) diff --git a/libffwd/backend_blas.c b/libffwd/backend_blas.c index 9e3d376..7bd1d0d 100644 --- a/libffwd/backend_blas.c +++ b/libffwd/backend_blas.c @@ -1,6 +1,10 @@ -/* backend_blas.c - CPU backend (BLAS), implements the engine API in ffwd.h. - * Linked only into CPU builds. The CPU model loads on the caller's thread, so - * ffwd_open does the full load and activate/worker_free are no-ops. */ +/* CPU backend. + * Implements the engine API in ffwd.h and is linked only into CPU builds. + * + * The CPU model loads on the caller's thread, so ffwd_open does the full load. + * activate and worker_free are no-ops. + */ + #include "internal.h" #include "threadpool.h" #include "platform.h" diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c index a925404..6adbe95 100644 --- a/libffwd/backend_cuda.c +++ b/libffwd/backend_cuda.c @@ -1,7 +1,13 @@ -/* backend_cuda.c - NVIDIA GPU backend (CUDA), implements the engine API in ffwd.h. - * Linked only into CUDA builds. The context is owned by the inference thread: - * open records what to load, activate creates the context, worker_free destroys - * it. CUDA encodes on the device and scores MaxSim on the host. */ +/* NVIDIA CUDA backend. + * Implements the API in ffwd.h and is linked only into CUDA builds. + * + * The context is owned by the inference thread: + * open records what to load, activate creates the context, and worker_free + * destroys it. + * + * CUDA encodes on the device and scores MaxSim on the host. + */ + #include "internal.h" #include "cuda.h" #include "platform.h" diff --git a/libffwd/backend_mlx.c b/libffwd/backend_mlx.c index 932e550..841aad1 100644 --- a/libffwd/backend_mlx.c +++ b/libffwd/backend_mlx.c @@ -1,7 +1,12 @@ -/* backend_mlx.c - Apple GPU backend (MLX), implements the engine API in ffwd.h. - * Linked only into Apple Silicon GPU builds. MLX streams are thread-local, so - * the context is created and destroyed on the inference thread: open only - * records what to load, activate creates the context, worker_free destroys it. */ +/* Metal backend. + * Implements the API in ffwd.h and is linked only into Apple Metal builds. + * + * MLX streams are thread-local, so the context is created and destroyed on the + * inference thread. + * open only records what to load; activate creates the context; worker_free + * destroys it. + */ + #include "internal.h" #include "mlx.h" #include "platform.h" diff --git a/libffwd/bpe.c b/libffwd/bpe.c index b3c8a8c..5a4d184 100644 --- a/libffwd/bpe.c +++ b/libffwd/bpe.c @@ -8,9 +8,7 @@ #include <stdlib.h> #include <string.h> -/* ======================================================================== - * GPT-2 Bytes-to-Unicode Mapping - * ======================================================================== */ +/* GPT-2 bytes-to-unicode mapping */ static int gpt2_byte_to_unicode[256]; static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ @@ -124,9 +122,7 @@ static char *decode_gpt2_token(const char *token_str) { return (char *)bytes; } -/* ======================================================================== - * String->int hash map (open addressing) - * ======================================================================== */ +/* String->int hash map (open addressing) */ typedef struct { char *key; /* NULL means empty slot */ @@ -252,9 +248,7 @@ static int map_get_pair(const str_int_entry_t *map, int cap, const char *a, cons return -1; } -/* ======================================================================== - * BPE helpers - * ======================================================================== */ +/* BPE helpers */ static int utf8_char_len(unsigned char c) { if ((c & 0x80) == 0) @@ -319,9 +313,7 @@ static char *str_concat2(const char *a, const char *b) { return r; } -/* ======================================================================== - * Integer-pair BPE merges: (left_id, right_id) -> (rank, merged_id) - * ======================================================================== */ +/* Integer-pair BPE merges: (left_id, right_id) -> (rank, merged_id) */ typedef struct { uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */ @@ -1253,11 +1245,14 @@ static int encode_bpe_word_into_slow(const tok_bpe_t *tok, return 0; } -/* Integer-pair BPE merge: token ids in a doubly-linked array with cached - * pair ranks; each merge updates only the two adjacent pairs. Falls back to - * the string-based loop when the int tables are unavailable. Identical - * output: initial ids are the byte-level single-char token ids and every - * merge result is the vocab id of the concatenated pair. */ +/* Integer-pair BPE merge. + * Uses token ids in a doubly-linked array with cached pair ranks. + * Each merge updates only the two adjacent pairs. + * + * Falls back to the string-based loop when int tables are unavailable. + * + * Output is identical: initial ids are byte-level single-char token ids, and + * each merge result is the vocab id of the concatenated pair. */ static int encode_bpe_word_into(const tok_bpe_t *tok, tok_bpe_workspace_t *ws, const char *mapped, @@ -1650,9 +1645,7 @@ static int load_merges_map(tok_bpe_t *tok, const char *merges_path) { return 0; } -/* ======================================================================== - * Public API - * ======================================================================== */ +/* Public API */ tok_bpe_t *tok_bpe_load(const char *vocab_json_path) { FILE *f = fopen(vocab_json_path, "rb"); @@ -1688,10 +1681,11 @@ tok_bpe_t *tok_bpe_load(const char *vocab_json_path) { return NULL; } - /* vocab.json is a flat { token: id } object. Walk it once for the max id to - * size the tables, then again to fill them. yyjson unescapes each key - * (including \uXXXX) to the same byte-level GPT-2 form the old scanner - * produced; decode_gpt2_token then maps that to the raw token text. */ + /* vocab.json is a flat { token: id } object. + * Walk it once to find the max id and size the tables, then again to fill them. + * yyjson unescapes each key, including \uXXXX, to the same byte-level GPT-2 form + * the old scanner produced. + * decode_gpt2_token then maps that to the raw token text. */ int max_id = 0; size_t idx, n; yyjson_val *key, *val; diff --git a/libffwd/bpe.h b/libffwd/bpe.h index 5b16c52..5199fd0 100644 --- a/libffwd/bpe.h +++ b/libffwd/bpe.h @@ -1,5 +1,5 @@ /* - * bpe.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level). * * Supports: * - decode token ID -> UTF-8 text diff --git a/libffwd/build.h b/libffwd/build.h index a133163..6ab367e 100644 --- a/libffwd/build.h +++ b/libffwd/build.h @@ -1,8 +1,12 @@ -/* ffwd_build.h - build identity and capability information for the ffwd tools +/* Build identity and capability information. * * The real values come from the Makefile's VERSION_CFLAGS at compile time; the - * fallbacks below apply when those flags are absent. Front-end only: the - * inference library does not use these. */ + * fallbacks below apply when those flags are absent. The core library doesn't use + * these; it's for frontends only. + */ + +/* TODO: tools should not depend on the internal library. */ + #ifndef FFWD_BUILD_H #define FFWD_BUILD_H diff --git a/libffwd/config.c b/libffwd/config.c index 6531658..d9efb4d 100644 --- a/libffwd/config.c +++ b/libffwd/config.c @@ -9,7 +9,7 @@ #include <stdlib.h> #include <string.h> -/* Verbosity flag, read across the codebase for optional stderr diagnostics. +/* Verbosity flag for optional stderr diagnostics across the codebase. * Defined in config.c because every backend links it. */ int ffwd_verbose = 0; @@ -20,11 +20,13 @@ typedef enum { JSON_BOOL_INVALID = -1, } json_bool_t; -/* Field readers over a parsed JSON object. yyjson getters are NULL-tolerant, so - * a missing key or wrong-typed value falls back cleanly. These replace a - * substring scanner that could match a key nested anywhere in the document - * (e.g. a field inside a nested sub-config), which is exactly the failure mode - * a real parser removes. */ +/* Read fields from a parsed JSON object. + * yyjson getters tolerate NULL, so missing keys or wrong types cleanly return + * the fallback. + * + * This replaces substring scanning, which could match a key anywhere in the + * document, including inside nested sub-configs. A real parser avoids that + * failure mode. */ static int json_get_int(yyjson_val *obj, const char *key, int fallback) { yyjson_val *v = yyjson_obj_get(obj, key); if (yyjson_is_int(v)) diff --git a/libffwd/cuda.h b/libffwd/cuda.h index 851e412..22fa78f 100644 --- a/libffwd/cuda.h +++ b/libffwd/cuda.h @@ -1,13 +1,11 @@ /* - * ffwd_cuda.h - CUDA/cuBLAS backend + * CUDA/cuBLAS backend */ #ifndef FFWD_CUDA_H #define FFWD_CUDA_H -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying FFWD_API are exported - * from the shared library; everything else stays internal. */ +/* Public-API export annotation. */ #ifndef FFWD_API # if defined(__GNUC__) # define FFWD_API __attribute__((visibility("default"))) diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h index 03a8e26..c4708fa 100644 --- a/libffwd/ffwd.h +++ b/libffwd/ffwd.h @@ -1,13 +1,8 @@ -/* - * ffwd.h - ffwd API - */ - #ifndef FFWD_H #define FFWD_H -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying FFWD_API are exported - * from the shared library; everything else stays internal. */ +/* The libraries are built with -fvisibility=hidden, so only declarations carrying + * FFWD_API are exported from the shared library; everything else stays internal. */ #ifndef FFWD_API # if defined(__GNUC__) # define FFWD_API __attribute__((visibility("default"))) @@ -19,25 +14,21 @@ #include <stddef.h> #include <stdint.h> -/* ======================================================================== - * Shared constants and family defaults - * ======================================================================== */ +/* Shared constants and defaults */ #define FFWD_VOCAB_SIZE 151936 #define FFWD_HEAD_DIM 128 #define FFWD_MAX_LAYERS 64 /* upper bound for stack arrays */ #define FFWD_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ -/* Late-interaction (ColBERT) special-token id fallbacks and the query pad - * length, used when a model's tokenizer does not define the tokens itself. */ +/* Late-interaction (ColBERT) fallback special-token ids and query pad length. + * Used when the model tokenizer does not define these tokens itself. */ #define FFWD_LATE_QUERY_TOKENS 32 #define FFWD_LATE_MASK_TOKEN_ID 151642 #define FFWD_LATE_QUERY_PREFIX_ID 151669 #define FFWD_LATE_DOCUMENT_PREFIX_ID 151670 -/* ======================================================================== - * Model Configuration (populated from config.json at load time) - * ======================================================================== */ +/* Model Configuration (populated from config.json at load time) */ typedef enum { FFWD_ATTENTION_BIDIRECTIONAL = 0, @@ -86,12 +77,14 @@ typedef struct { ffwd_attention_mode_t attention_mode; ffwd_pooling_mode_t pooling_mode; int normalize_embeddings; - /* Qwen3-Embedding pools the last token, which is the tokenizer's - * <|endoftext|> suffix added by its post-processor - not the model's chat - * eos_token_id (<|im_end|>). Text frontends append it and resolve the id - * from the tokenizer exactly like the contextual separator - * (FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The - * model forward consumes token ids exactly and never appends. */ + /* Qwen3-Embedding pools the last token: the tokenizer's <|endoftext|> suffix + * added by its post-processor, not the model's chat eos_token_id (<|im_end|>). + * + * Text frontends append it and resolve its id from the tokenizer, just like the + * contextual separator. FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model + * fallback. + * + * Model forward consumes token ids exactly and never appends. */ int append_terminal_token; } ffwd_config_t; @@ -111,24 +104,31 @@ typedef struct { int n_spans; } ffwd_context_input_t; -/* ======================================================================== - * Engine API — the backend-selected inference handle +/* + * Backend-selected inference handle. + * + * This is the library's main public interface and the full supported surface of + * the shared library. Exactly one backend is linked at build time: CPU, Apple Metal, + * or NVIDIA CUDA. That backend implements these functions, so callers never branch + * on backend. + * + * The CLI, server, and language bindings use only these functions plus the math + * helpers below. + * + * Model loading has two phases so a GPU context can be created on the thread + * that owns it. * - * This is the library's primary public interface and the entire supported - * surface of the shared library. Exactly one backend (CPU, Apple Metal, or - * NVIDIA CUDA) is linked at build time and implements these functions; - * callers never branch on the backend. The CLI, the server, and any language - * binding use only these functions plus the math helpers below. + * ffwd_open runs on the caller's thread. It loads a CPU model immediately; in a + * GPU build, it only records what to load. * - * A model loads in two phases so a GPU context can be created on the thread - * that will own it. ffwd_open runs on the caller's thread and loads a CPU - * model immediately; a GPU build only records what to load. ffwd_activate then - * runs on the inference thread and creates the GPU context (a no-op for CPU). - * Teardown mirrors this: ffwd_worker_free releases the GPU context from the - * inference thread, ffwd_free releases the rest. - * ======================================================================== */ + * ffwd_activate runs on the inference thread and creates the GPU context. It is + * a no-op for CPU. + * + * Teardown mirrors activation: ffwd_worker_free releases the GPU context from + * the inference thread, and ffwd_free releases the rest. + */ -/* Opaque inference handle; its layout is private to the linked backend. */ +/* Opaque inference handle; layout is private to the linked backend. */ typedef struct embed ffwd_t; /* Backend tuning, parsed from the command line. A backend reads the fields it @@ -220,22 +220,27 @@ FFWD_API int ffwd_l2_normalize(float *vec, int dim); */ FFWD_API float ffwd_cosine_similarity(const float *a, const float *b, int dim); -/* ======================================================================== - * Tokenization — text into the token ids a model expects +/* + * Tokenization: text -> token ids the model expects. * - * The tokenizer is backend-independent (text -> ids is identical on CPU or - * GPU), so it is its own handle, separate from ffwd_t. ffwd_tok_open picks the - * tokenizer by the files in model_dir (WordPiece vocab.txt, SentencePiece - * .model, or byte-level vocab.json) and resolves the model's special-token ids. - * ffwd_tokenize then applies that model's layout: BERT and XLM-R wrap the ids - * with [CLS]..[SEP] (<s>..</s>), Qwen appends its terminal token. - * ======================================================================== */ + * Tokenization is backend-independent because text -> ids is the same on CPU + * and GPU, so it has its own handle, separate from ffwd_t. + * + * ffwd_tok_open selects the tokenizer from files in model_dir: WordPiece + * vocab.txt, SentencePiece .model, or byte-level vocab.json. It also resolves + * the model's special-token ids. + * + * ffwd_tokenize applies the model's token layout: BERT and XLM-R wrap ids with + * [CLS]..[SEP] (<s>..</s>), while Qwen appends its terminal token. + */ /* Opaque tokenizer handle; owns the tokenizer and its resolved special tokens. */ typedef struct ffwd_tok ffwd_tok_t; -/* Late-interaction tokens: ids, plus the document positions to score. For a - * query, keep is NULL (every position is kept) and n_keep == n_tokens. */ +/* Late-interaction tokens: + * token ids plus the document positions to score. + * For queries, keep is NULL because every position is kept, and + * n_keep == n_tokens. */ typedef struct { int *ids; int n_tokens; diff --git a/libffwd/forward.c b/libffwd/forward.c index d8e45f9..d5d2442 100644 --- a/libffwd/forward.c +++ b/libffwd/forward.c @@ -9,14 +9,12 @@ #include <math.h> #include <limits.h> -/* Below these packed-sequence lengths the fused BF16 row kernels beat widening - * the weight to F32 and dispatching through BLAS; above them, widen once. */ +/* For packed sequences below these lengths, fused BF16 row kernels beat + * widening the weight to F32 and using BLAS. For longer packed sequences, widen once. */ #define FFWD_BF16_QKV_FUSE_MAX_SEQ 16 #define FFWD_BF16_PAIR_FUSE_MAX_SEQ 4 -/* ======================================================================== - * Linear projections (weight-dtype dispatch) and workspace scratch growth - * ======================================================================== */ +/* Linear projections (weight-dtype dispatch) and workspace scratch growth */ static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { for (int i = 0; i < n; i++) @@ -123,14 +121,14 @@ static void linear_pair_weight(ffwd_workspace_t *ws, linear_nobias_weight(ws, b, x, wb, seq_len, in_dim, b_dim); } -/* ======================================================================== - * Packed forward pass +/* Packed forward pass. * - * The workspace-growth helpers these call (ensure_buffers, ensure_rope_cache, - * ensure_attention_scores, ensure_offsets) live in model_blas.c beside the - * workspace lifecycle and ffwd_workspace_nbytes, whose size accounting mirrors - * them. - * ======================================================================== */ + * Workspace-growth helpers called here live in model_blas.c beside the + * workspace lifecycle and ffwd_workspace_nbytes: + * ensure_buffers, ensure_rope_cache, ensure_attention_scores, ensure_offsets. + * + * ffwd_workspace_nbytes mirrors their size accounting. + */ static ffwd_attention_fn attention_for_config(const ffwd_config_t *cfg) { return cfg->attention_mode == FFWD_ATTENTION_CAUSAL @@ -298,13 +296,21 @@ int forward_packed_inplace(const ffwd_model_t *model, 0, model->config.n_layers, apply_final_norm); } -/* BERT-family forward: token + learned-position + token-type embeddings and an - * embedding LayerNorm, then post-norm encoder layers (bias on every projection, - * GeLU feed-forward, no RoPE, no per-head norm). Bidirectional attention is - * block-diagonal over the packed batch, so each sequence's positions restart at - * cfg->position_id_offset (0 for BERT, pad_id + 1 for RoBERTa/XLM-R). Leaves the - * final hidden states in ws->x; pooling reads them as-is because the last - * layer's LayerNorm is already the model's final normalization. */ +/* BERT-family forward pass. + * + * Uses token, learned-position, and token-type embeddings, then embedding + * LayerNorm. + * + * Encoder layers are post-norm, with bias on every projection, GeLU + * feed-forward, no RoPE, and no per-head norm. + * + * Bidirectional attention is block-diagonal over the packed batch. Each + * sequence's positions restart at cfg->position_id_offset: 0 for BERT, or + * pad_id + 1 for RoBERTa/XLM-R. + * + * Leaves final hidden states in ws->x. Pooling reads them as-is because the last + * layer's LayerNorm is already the model's final normalization. + */ static int forward_packed_bert(const ffwd_model_t *model, ffwd_workspace_t *ws, const ffwd_input_t *inputs, @@ -405,8 +411,8 @@ static int pool_embeddings(const ffwd_model_t *model, const ffwd_config_t *cfg = &model->config; int hidden = cfg->hidden_size; float eps = cfg->rms_norm_eps; - /* Qwen pools pre-final-norm states and applies the final RMSNorm here; BERT - * pools already-normed states (no final norm), signalled by a NULL norm. */ + /* Qwen pools pre-final-norm states, then applies final RMSNorm here. + * BERT pools already-normalized states, so a NULL norm means no final norm. */ const float *norm_weight = model->weights.norm; const float *x = ws->x; if (hidden <= 0 || !x) @@ -419,8 +425,7 @@ static int pool_embeddings(const ffwd_model_t *model, float *emb = out_embeddings + (size_t)b * hidden; if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) { - /* Single-token pooling: CLS takes the first token, last-token the - * final one. */ + /* Single-token pooling: CLS uses the first token; last-token uses the final token. */ int idx = cfg->pooling_mode == FFWD_POOL_CLS ? start : end - 1; const float *row = x + (size_t)idx * hidden; if (norm_weight) { @@ -460,9 +465,7 @@ static int pool_embeddings(const ffwd_model_t *model, return 0; } -/* ======================================================================== - * Public forward/embed APIs - * ======================================================================== */ +/* Forward/embed APIs */ int ffwd_model_forward_into(const ffwd_model_t *model, ffwd_workspace_t *ws, diff --git a/libffwd/gemm.c b/libffwd/gemm.c index bb54fb9..ab664f1 100644 --- a/libffwd/gemm.c +++ b/libffwd/gemm.c @@ -1,5 +1,5 @@ /* - * gemm.c - general Matrix Multiplication + * General matrix multiplication. */ #include "kernels.h" @@ -30,21 +30,18 @@ # define M_PI 3.14159265358979323846 #endif -/* Matrix and vector ops: GEMM, the dtype-dispatch linear kernels, and the - * BF16-fused matvec/QKV/pair workers, plus the elementwise residual add. */ +/* Matrix and vector ops: + * GEMM, dtype-dispatch linear kernels, BF16-fused matvec/QKV/pair workers, + * and element-wise residual add. */ -/* ======================================================================== - * Basic Element-wise Operations - * ======================================================================== */ +/* Basic element-wise operations */ void add_inplace(float *a, const float *b, int n) { for (int i = 0; i < n; i++) a[i] += b[i]; } -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ +/* Matrix operations */ void matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { #ifdef USE_BLAS @@ -425,7 +422,8 @@ void linear_nobias_bf16( bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); return; } - /* Callers route longer sequences through an F32-widened weight matrix in - * caller-owned scratch; this per-row path stays correct for any length. */ + /* Callers send longer sequences through an F32-widened weight matrix in + * caller-owned scratch. + * This per-row path remains correct for any length. */ bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); } diff --git a/libffwd/generic.c b/libffwd/generic.c index 8798107..5425247 100644 --- a/libffwd/generic.c +++ b/libffwd/generic.c @@ -1,5 +1,5 @@ /* - * generic.c - architecture-generic hot kernels / non-SIMD fallback + * Architecture-generic hot kernels / non-SIMD fallback. */ #include "impl.h" diff --git a/libffwd/impl.h b/libffwd/impl.h index 39baa40..2149cf2 100644 --- a/libffwd/impl.h +++ b/libffwd/impl.h @@ -1,15 +1,16 @@ /* - * impl.h - internal architecture dispatch for hot kernels + * Internal architecture dispatch for hot kernels. * - * The small vector primitives (dot product, scale, axpy, scale-add, scaled - * multiply) are defined here as static inline so they fold directly into the - * caller's loop: a row of RMSNorm or attention issues no call and the compiler - * keeps the row in registers across the reduction and the write-back. The - * generic scalar versions stay out of line in generic.c as a reference for the - * kernel tests and as the fallback on platforms with no SIMD path. + * Small vector primitives (dot product, scale, axpy, scale-add, scaled + * multiply) are static inline here so they fold into the caller's loop. + * For example, an RMSNorm or attention row has no call overhead, and the + * compiler can keep the row in registers across reduction and write-back. * - * bf16_matvec_fused is large (a full matrix-vector product) and is called once - * per output tile rather than inside a tight loop, so it stays out of line. + * Generic scalar versions stay out of line in generic.c. They are the reference + * for kernel tests and the fallback when no SIMD path is available. + * + * bf16_matvec_fused is a large full matrix-vector product, called once per + * output tile rather than inside a tight loop, so it stays out of line. */ #ifndef KERNELS_IMPL_H diff --git a/libffwd/internal.h b/libffwd/internal.h index a2b933a..6030c36 100644 --- a/libffwd/internal.h +++ b/libffwd/internal.h @@ -1,9 +1,3 @@ -/* - * internal.h - non-exported, in-tree API shared by the frontends, - * backends, tests, and benchmarks. The public ABI is ffwd.h; none of these - * symbols carry FFWD_API visibility or appear in the export maps. - */ - #ifndef FFWD_INTERNAL_H #define FFWD_INTERNAL_H @@ -17,7 +11,8 @@ typedef struct ffwd_workspace ffwd_workspace_t; typedef struct ffwd_late_model ffwd_late_model_t; typedef struct ffwd_late_workspace ffwd_late_workspace_t; -/* Backend-specific help text used by the in-tree frontends. */ +/* Help text used by the frontends. */ +/* TODO: tools should not depend on the internal API */ void ffwd_cli_help(FILE *f); void ffwd_server_help(FILE *f); @@ -91,7 +86,7 @@ int ffwd_model_encode_spans_batch(const ffwd_model_t *model, /* * Late-interaction CPU model API. The public engine path exposes reranking via - * ffwd_rerank(); these entry points are for in-tree tests/backends only. + * ffwd_rerank(); these entry points are for tests/backends only. */ ffwd_late_model_t *ffwd_late_model_load(const char *model_dir); void ffwd_late_model_free(ffwd_late_model_t *model); diff --git a/libffwd/kernels.h b/libffwd/kernels.h index 2a917e0..11312db 100644 --- a/libffwd/kernels.h +++ b/libffwd/kernels.h @@ -1,5 +1,5 @@ /* - * kernels.h - tkern public API: transformer inference kernels (CPU, float32) + * Transformer inference kernels (CPU, float32). * * Low-level math operations. All operate on float32 tensors in row-major order. */ @@ -10,15 +10,11 @@ #include <stddef.h> #include <stdint.h> -/* ======================================================================== - * Basic Operations - * ======================================================================== */ +/* Basic operations */ void add_inplace(float *a, const float *b, int n); -/* ======================================================================== - * Matrix Operations - * ======================================================================== */ +/* Matrix operations */ /* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ void matmul_t(float *C, const float *A, const float *B, int M, int K, int N); @@ -62,9 +58,7 @@ void linear_nobias_bf16_qkv(float *q, /* Dot product using the best available local SIMD implementation. */ float dot_f32(const float *a, const float *b, int n); -/* ======================================================================== - * Normalization - * ======================================================================== */ +/* Normalization */ /* RMS Normalization: out = x / rms(x) * weight */ void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); @@ -75,9 +69,10 @@ void rms_norm(float *out, const float *x, const float *weight, int seq_len, int void rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); -/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps) - * + beta, with mean and biased (population) variance over the hidden axis per - * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ +/* Layer normalization for BERT-family models: + * out = gamma * (x - mean) / sqrt(var + eps) + beta. + * Mean and biased/population variance are computed over the hidden axis per row. + * Unlike RMSNorm, this subtracts the mean and adds a bias. */ void layer_norm(float *out, const float *x, const float *gamma, @@ -86,25 +81,21 @@ void layer_norm(float *out, int hidden, float eps); -/* ======================================================================== - * Activation Functions - * ======================================================================== */ +/* Activation functions */ void softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ void silu_mul_inplace(float *gate, const float *up, int n); -/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the - * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ +/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). + * This is the "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ void gelu_inplace(float *x, int n); /* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh): * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */ void gelu_tanh_inplace(float *x, int n); -/* ======================================================================== - * Attention Operations - * ======================================================================== */ +/* Attention operations */ /* * Bidirectional GQA attention over a packed/ragged batch. @@ -169,9 +160,7 @@ void causal_gqa_attention_packed_with_scratch(float *out, float *scratch, size_t scratch_bytes); -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ +/* Position embeddings */ /* * NeoX-style RoPE: compute cos/sin for positions. diff --git a/libffwd/late.c b/libffwd/late.c index e086ad3..9418a20 100644 --- a/libffwd/late.c +++ b/libffwd/late.c @@ -1,3 +1,16 @@ +/* + * Late-interaction model. + * + * A late-interaction model is a pooled base model plus a 1_Dense linear + * projection from hidden states to token vectors. + * + * Instead of pooling to one vector, it keeps one projected vector per kept token + * and scores query/doc pairs with MaxSim. + * + * Reuses the base model's packed forward pass and workspace. This layer only + * owns the projection, token-vector buffer, and scoring. + */ + #include "ffwd.h" #include "model_internal.h" #include "kernels.h" @@ -8,17 +21,6 @@ #include <limits.h> #include <float.h> -/* ======================================================================== - * Late-interaction model - * - * A late-interaction model is a pooled base model plus a 1_Dense linear - * projection from hidden states to token vectors. Instead of pooling to one - * vector, it keeps one projected vector per kept token and scores query/doc - * pairs with MaxSim. It reuses the base model's packed forward pass and - * workspace; only the projection, the token-vector buffer, and the scoring - * live here. - * ======================================================================== */ - struct ffwd_late_model { ffwd_model_t *base; safetensors_file_t *dense_sf; @@ -107,8 +109,9 @@ const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model) { int ffwd_late_model_token_dim(const ffwd_late_model_t *model) { return model ? model->token_dim : 0; } -/* Internal (internal.h): expose the base model and projection so GPU - * backends can upload them to device without seeing struct ffwd_late_model. */ +/* Internal (internal.h): + * expose the base model and projection so GPU backends can upload them to the + * device without seeing struct ffwd_late_model. */ ffwd_model_t *ffwd_late_model_base(const ffwd_late_model_t *model) { return model ? model->base : NULL; } diff --git a/libffwd/late_blas.c b/libffwd/late_blas.c index dabeb58..84b7f17 100644 --- a/libffwd/late_blas.c +++ b/libffwd/late_blas.c @@ -5,8 +5,9 @@ #include <stdlib.h> #include <string.h> -/* CPU late-interaction encoding. The late model loader and MaxSim scoring live - * in late.c because CUDA reuses those without linking the CPU transformer. */ +/* CPU late-interaction encoding. + * The late model loader and MaxSim scoring stay in late.c because CUDA reuses + * them without linking the CPU transformer. */ struct ffwd_late_workspace { const ffwd_late_model_t *model; diff --git a/libffwd/mlx.c b/libffwd/mlx.c index 0b6d8b7..d8f6d1e 100644 --- a/libffwd/mlx.c +++ b/libffwd/mlx.c @@ -13,9 +13,7 @@ #include <limits.h> #include <float.h> -/* ======================================================================== - * Per-layer MLX weight arrays - * ======================================================================== */ +/* Per-layer MLX weight arrays */ typedef struct { mlx_array w; @@ -33,9 +31,9 @@ typedef struct { mlx_array q_norm, k_norm; mlx_array input_norm, post_attn_norm; mlx_linear_t gate_proj, up_proj, down_proj; - /* BERT family (unset for Qwen3; see internal.h). wo/up_proj/down_proj - * and input_norm/post_attn_norm are reused as the BERT attention output, - * FFN matrices, and the two block LayerNorm weights. */ + /* BERT family only; unset for Qwen3. See internal.h. + * Reuses wo/up_proj/down_proj for BERT attention output and FFN matrices. + * Reuses input_norm/post_attn_norm for the two BERT block LayerNorm weights. */ mlx_array o_bias, attn_ln_bias, ffn_inter_bias, ffn_out_bias, ffn_ln_bias; } mlx_layer_t; @@ -44,7 +42,7 @@ struct ffwd_mlx_ctx { mlx_array embed_tokens; mlx_layer_t *layers; /* heap-allocated [n_layers] */ mlx_array norm; - /* BERT family (unset for Qwen3): learned positions, token-type, embed LN. */ + /* BERT family only; unset for Qwen3: learned positions, token-type, embed LN. */ mlx_array position_embeddings, token_type_embeddings, ffwd_ln_w, ffwd_ln_b; mlx_stream stream; multi_safetensors_t *ms; @@ -69,9 +67,7 @@ struct ffwd_mlx_late_vectors { int dim; }; -/* ======================================================================== - * Helpers - * ======================================================================== */ +/* Helpers */ static int arr_ok(mlx_array a) { return a.ctx != NULL; } @@ -158,10 +154,11 @@ static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, const in if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim)) return (mlx_array){0}; if (t->dtype == DTYPE_F16) { - /* No F16 forward path on MLX (the matmul fast path keys on BF16). Widen - * to F32 on the host at load so the whole forward runs in F32, mirroring - * the CPU loader. mlx_array_new_data copies its buffer, so the temporary - * is freed immediately. Public F16 snapshots load through here. */ + /* No F16 forward path on MLX; the matmul fast path uses BF16. + * Widen to F32 on the host at load time, so the full forward pass runs in F32, + * matching the CPU loader. + * mlx_array_new_data copies its buffer, so the temporary is freed immediately. + * Public F16 snapshots load through this path. */ int64_t n = safetensor_numel(t); float *tmp = (float *)malloc((size_t)n * sizeof(float)); if (!tmp) @@ -341,9 +338,7 @@ static void upcast_f32_inplace(mlx_array *a, mlx_stream s) { *a = f; } -/* ======================================================================== - * Load - * ======================================================================== */ +/* Load */ static int mlx_model_dir_has_file(const char *model_dir, const char *rel) { char path[1024]; @@ -394,9 +389,9 @@ static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) { prepare_linear_transpose(&l->down_proj, s); } -/* BERT-family MLX loader, mirroring load_bert_weights on the CPU side. The - * shared post-load below (stream, transposes, optional BF16 upcast) runs after - * either family fills ctx->layers. */ +/* BERT-family MLX loader, matching CPU load_bert_weights. + * The shared post-load step below runs after either family fills ctx->layers: + * stream setup, transposes, and optional BF16 upcast. */ static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer_end) { const ffwd_config_t *c = &ctx->config; int h = c->hidden_size, inter = c->intermediate_size; @@ -625,9 +620,11 @@ static ffwd_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, loaded: ctx->stream = mlx_default_gpu_stream_new(); ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w); - /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the - * tiny norm vectors so they match the F32 activations in mlx_forward_layers; - * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */ + /* Non-quantized BF16: + * run RMSNorm and the residual stream in F32. + * Upcast the small norm vectors to match the F32 activations in + * mlx_forward_layers; keep the large matmul weights in BF16. + * Q8 keeps its existing BF16 norms. */ if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) { for (int i = layer_start; i < layer_end; i++) { mlx_layer_t *l = &ctx->layers[i]; @@ -647,7 +644,7 @@ loaded: upcast_f32_inplace(&l->ffn_ln_bias, ctx->stream); } upcast_f32_inplace(&ctx->norm, ctx->stream); - /* BERT embedding stage (no-ops for Qwen): learned position and token-type + /* BERT embedding stage; no-ops for Qwen: learned position and token-type * embeddings and the embedding LayerNorm all feed the F32 residual. */ upcast_f32_inplace(&ctx->position_embeddings, ctx->stream); upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream); @@ -704,9 +701,7 @@ ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, const ffwd_mlx return mlx_load_range_ex(model_dir, 0, -1, opts, 0); } -/* ======================================================================== - * Free - * ======================================================================== */ +/* Free */ static void free_mlx_layer(mlx_layer_t *l) { free_linear(&l->wq); @@ -827,9 +822,7 @@ const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx) { int ffwd_mlx_late_token_dim(const ffwd_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } -/* ======================================================================== - * Forward pass - * ======================================================================== */ +/* Forward pass */ /* Exact erf GeLU: 0.5 * x * (1 + erf(x / sqrt(2))). Frees nothing it is given. */ static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) { @@ -1569,7 +1562,7 @@ cleanup: /* Device-resident select/concat primitives. The server rerank path batches * candidates through ffwd_mlx_late_encode_docs_device, but the late-interaction - * verification harness (tests/integration/check_late_interaction.py) drives these directly + * verification harness (tests/check_late_interaction.py) drives these directly * to validate per-document gather and concat against the CPU MaxSim reference. */ ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, @@ -2125,7 +2118,7 @@ static int ffwd_mlx_encode_batch_dense(ffwd_mlx_ctx_t *ctx, mlx_array_free(x_normed); } - /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are + /* 5. Eval and copy [B, hidden] to CPU. pplx-embed embeddings are * intentionally unnormalized; callers should use cosine similarity. */ mlx_array emb_f32 = mlx_array_new(); mlx_astype(&emb_f32, emb, MLX_FLOAT32, S); diff --git a/libffwd/mlx.h b/libffwd/mlx.h index e4e2a8e..e2a6e12 100644 --- a/libffwd/mlx.h +++ b/libffwd/mlx.h @@ -1,15 +1,12 @@ /* - * ffwd_mlx.h - MLX GPU backend for Apple Silicon - * - * Uses Apple's mlx-c to run the transformer on Metal. + * Run the transformer on Metal using mlx-c. */ #ifndef FFWD_MLX_H #define FFWD_MLX_H -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying FFWD_API are exported - * from the shared library; everything else stays internal. */ +/* The libraries are built with -fvisibility=hidden, so only declarations carrying + * FFWD_API are exported from the shared library; everything else is internal. */ #ifndef FFWD_API # if defined(__GNUC__) # define FFWD_API __attribute__((visibility("default"))) @@ -79,13 +76,16 @@ FFWD_API void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs); FFWD_API int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors); -/* - * Encode n_docs documents in one padded transformer pass and pack each - * document's kept token vectors back-to-back into [total_keep, token_dim] in - * document order. out_offsets[n_docs + 1] receives the prefix sum of kept - * counts, so the result feeds ffwd_mlx_late_maxsim_batch_device directly. - * Returns NULL on error; the handle must not outlive ctx. - */ + +/* Encode n_docs documents in one padded transformer pass. + * + * Packs each document's kept token vectors back-to-back into [total_keep, token_dim] + * in document order. + * + * out_offsets[n_docs + 1] gets the prefix sum of kept counts, so the result can + * feed ffwd_mlx_late_maxsim_batch_device directly. + * + * Returns NULL on error. The handle must not outlive ctx. */ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx, const int *const *doc_ids, const int *n_tokens, @@ -94,9 +94,9 @@ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late int n_docs, int normalize, int *out_offsets); -/* Device-resident select/concat primitives, retained for the late-interaction - * verification harness (tests/integration/check_late_interaction.py); the server rerank - * path uses ffwd_mlx_late_encode_docs_device. */ +/* Device-resident select/concat primitives. + * Kept for the late-interaction verification harness (tests/check_late_interaction.py). + * The server rerank path uses ffwd_mlx_late_encode_docs_device. */ FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat( ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count); FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, @@ -124,7 +124,7 @@ FFWD_API int ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding); /* - * Compute a true padded dense batch on MLX. + * Compute a true padded dense batch. * * out_embeddings is caller-provided [batch, hidden_size]. * Pooling and attention masking follow the loaded model configuration. diff --git a/libffwd/model.c b/libffwd/model.c index faecfd4..7346e32 100644 --- a/libffwd/model.c +++ b/libffwd/model.c @@ -8,9 +8,7 @@ #include <stdint.h> #include <string.h> -/* ======================================================================== - * Model-directory probes - * ======================================================================== */ +/* Model-directory probes */ static int model_dir_has_file(const char *model_dir, const char *rel) { char path[1024]; @@ -29,9 +27,7 @@ int model_dir_has_late_projection(const char *model_dir) { model_dir_has_file(model_dir, "1_Dense/model.safetensors"); } -/* ======================================================================== - * Weight tensor loading and validation (direct mmap pointers into safetensors) - * ======================================================================== */ +/* Weight tensor loading and validation (direct mmap pointers into safetensors) */ static size_t dtype_size(safetensor_dtype_t dtype) { switch (dtype) { @@ -125,8 +121,7 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } -/* Widen a safetensors tensor to f32 (caller frees). libsafetensors is a pure - * reader now, so the dtype conversion lives here, next to its only callers. */ +/* Widen a safetensors tensor to f32. Caller frees. */ static float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { int64_t n = safetensor_numel(t); if (n <= 0) @@ -206,15 +201,19 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i return safetensors_get_f32(sf, t); } -/* ======================================================================== - * Model / workspace load and free - * ======================================================================== */ +/* Model / workspace load and free */ -/* BERT-family weight loader. The tensor names differ entirely from the Qwen - * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry - * point. Reused ffwd_layer_t slots carry their BERT meaning: wo is the - * attention output dense, input_norm/post_attn_norm are the two block LayerNorm - * weights, up_proj/down_proj are the two FFN matrices (see internal.h). */ +/* BERT-family weight loader. + * Tensor names differ completely from Qwen + * (encoder.layer.N.*, embeddings.*), so this family has its own entry point. + * + * Reused ffwd_layer_t slots keep their BERT meanings: + * wo is the attention output dense; + * input_norm/post_attn_norm are the two block LayerNorm weights; + * up_proj/down_proj are the two FFN matrices. + * + * See internal.h. + */ static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { const ffwd_config_t *cfg = &model->config; diff --git a/libffwd/model_internal.h b/libffwd/model_internal.h index 97de064..b828e67 100644 --- a/libffwd/model_internal.h +++ b/libffwd/model_internal.h @@ -1,19 +1,24 @@ +/* Cross-file declarations private to CPU model code. + * + * model_types.h holds the shared struct definitions; this header holds the + * shared functions. + * + * These are not part of the public API (ffwd.h) and do not use FFWD_API + * visibility. They are shared between translation units that used to be one + * file: model loading, the forward pass, and late interaction. + * + * Frontends and GPU backends do not include this header. + */ + #ifndef FFWD_MODEL_INTERNAL_H #define FFWD_MODEL_INTERNAL_H #include "model_types.h" #include "alloc.h" -/* Cross-file function declarations private to the CPU model code. The companion - * to model_types.h: that header holds the shared struct definitions, this one the - * shared functions. None are part of the public API (ffwd.h) and none carry - * FFWD_API visibility; they are shared between the translation units (model - * loading, the forward pass, and late interaction) that used to live in one - * file. The frontends and the GPU backends do not include this header. */ - -/* Validate that a safetensor matches an expected dtype/rank/shape and that its - * bytes lie within the mapped file. bf16_ok permits either 16-bit float format - * (BF16 or F16) in addition to F32. */ +/* Validate that a safetensor has the expected dtype, rank, and shape, and that + * its bytes are within the mapped file. bf16_ok also allows either 16-bit float + * format, BF16 or F16, in addition to F32. */ int tensor_has_supported_shape(const safetensors_file_t *sf, const safetensor_t *t, const char *name, @@ -21,10 +26,14 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, int ndim, int bf16_ok); -/* Build a weight ref from a located tensor: F32/BF16 borrow the mmap, F16 is - * widened to an owned F32 buffer (no fused F16 kernel). Shared by the model - * loader and the late-interaction projection loader; the owner frees ref.data - * when ref.owned is set. Returns ref.data == NULL on allocation failure. */ +/* Build a weight ref from a located tensor. + * F32/BF16 borrow the mmap; F16 is widened to an owned F32 buffer because there + * is no fused F16 kernel. + * + * Shared by the model loader and late-interaction projection loader. + * The owner frees ref.data when ref.owned is set. + * + * Returns ref.data == NULL on allocation failure. */ ffwd_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t); /* True if model_dir carries a 1_Dense projection (a late-interaction model). */ @@ -35,9 +44,9 @@ int model_dir_has_late_projection(const char *model_dir); ffwd_model_t * model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late); -/* Workspace scratch growth, lazy per forward call. Defined in model.c beside the - * workspace lifecycle (ffwd_workspace_nbytes mirrors their sizing); called from - * the forward pass in forward.c. */ +/* Workspace scratch grows lazily per forward call. + * Defined in model.c beside the workspace lifecycle. ffwd_workspace_nbytes mirrors this sizing. + * Called from the forward pass in forward.c. */ int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq); void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch); int ensure_rope_cache(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int n_pos); diff --git a/libffwd/model_types.h b/libffwd/model_types.h index 345e0cd..4f01ca2 100644 --- a/libffwd/model_types.h +++ b/libffwd/model_types.h @@ -1,9 +1,10 @@ -/* - * model_types.h - in-memory layout of a loaded model: the model and workspace - * structs, the per-layer weight references, and the accessors that expose them - * to the GPU backends. Shared by the CPU model code and the CUDA backend; not - * part of the public ABI (ffwd.h). Companion to model_internal.h, which - * declares the CPU model's private functions. +/* In-memory layout of a loaded model: model/workspace structs, per-layer weight + * references, and accessors for GPU backends. + * + * Shared by CPU model code and the CUDA backend. Not part of the public ABI + * (ffwd.h). + * + * model_internal.h declares the CPU model's private functions. */ #ifndef FFWD_MODEL_TYPES_H diff --git a/libffwd/neon.c b/libffwd/neon.c index 7f44d82..50cf649 100644 --- a/libffwd/neon.c +++ b/libffwd/neon.c @@ -1,5 +1,5 @@ /* - * neon.c - ARM NEON hot kernels + * ARM NEON hot kernels. */ #include "impl.h" diff --git a/libffwd/norm.c b/libffwd/norm.c index 27fd4c4..57c93db 100644 --- a/libffwd/norm.c +++ b/libffwd/norm.c @@ -1,5 +1,6 @@ /* - * norm.c - Normalization + * Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the + * per-head RMSNorm used for QK-norm. */ #include "kernels.h" @@ -21,12 +22,7 @@ # define M_PI 3.14159265358979323846 #endif -/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the - * per-head RMSNorm used for QK-norm. */ - -/* ======================================================================== - * Normalization - * ======================================================================== */ +/* Normalization */ /* Rows split across the pool only when the tensor is big enough to pay * for the dispatch. */ diff --git a/libffwd/platform.h b/libffwd/platform.h index 0575a43..7d1c7ee 100644 --- a/libffwd/platform.h +++ b/libffwd/platform.h @@ -1,9 +1,11 @@ -/* platform.h - compile-time backend selection. +/* Compile-time backend selection. * * The build passes -DUSE_GPU for a GPU build. This header maps that flag and the - * host platform to a concrete backend: Apple Silicon uses MLX, any other GPU - * uses CUDA, and a non-GPU build is CPU. Use USE_GPU for the CPU/GPU split; - * USE_MLX and USE_CUDA only where a specific accelerator API is called. */ + * host platform to a backend: + * Use USE_GPU for the CPU/GPU split; USE_MLX and USE_CUDA only where a specific + * accelerator API is called. + */ + #ifndef FFWD_PLATFORM_H #define FFWD_PLATFORM_H diff --git a/libffwd/rope.c b/libffwd/rope.c index 12057b2..29cbb17 100644 --- a/libffwd/rope.c +++ b/libffwd/rope.c @@ -1,5 +1,5 @@ /* - * rope.c - rotary position embeddings + * Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ #include "kernels.h" @@ -29,11 +29,7 @@ # define M_PI 3.14159265358979323846 #endif -/* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ - -/* ======================================================================== - * Position Embeddings - * ======================================================================== */ +/* Position embeddings */ void compute_rope_neox( float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { diff --git a/libffwd/safetensors.c b/libffwd/safetensors.c index 0efbd57..f2c9b26 100644 --- a/libffwd/safetensors.c +++ b/libffwd/safetensors.c @@ -9,9 +9,8 @@ #include <sys/stat.h> #include <unistd.h> -/* ======================================================================== - * Minimal JSON parser for safetensors header - * ======================================================================== */ +/* Minimal JSON parser for safetensors header */ +/* TODO: consider replacing with yyjson */ static void skip_whitespace(const char **p) { while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') @@ -262,9 +261,7 @@ static int parse_header(safetensors_file_t *sf) { return 0; } -/* ======================================================================== - * Single file operations - * ======================================================================== */ +/* Single file operations */ safetensors_file_t *safetensors_open(const char *path) { int fd = open(path, O_RDONLY); @@ -343,9 +340,7 @@ int64_t safetensor_numel(const safetensor_t *t) { return n; } -/* ======================================================================== - * Multi-shard operations - * ======================================================================== */ +/* Multi-shard operations */ static int cmp_shard_names(const void *a, const void *b) { const char *sa = (const char *)a; diff --git a/libffwd/safetensors.h b/libffwd/safetensors.h index 13d4cba..f2ea9fd 100644 --- a/libffwd/safetensors.h +++ b/libffwd/safetensors.h @@ -1,5 +1,5 @@ /* - * safetensors.h - Safetensors file format reader (multi-shard support) + * Safetensors file format reader with multi-shard support. */ #ifndef SAFETENSORS_H @@ -40,8 +40,7 @@ typedef struct { safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; } safetensors_file_t; -/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup - */ +/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ typedef struct { safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; int num_shards; diff --git a/libffwd/sentencepiece.c b/libffwd/sentencepiece.c index 2372972..e5e8393 100644 --- a/libffwd/sentencepiece.c +++ b/libffwd/sentencepiece.c @@ -1,5 +1,5 @@ /* - * sentencepiece.c - SentencePiece Unigram tokenizer. + * SentencePiece Unigram tokenizer. * * The implementation reads the small subset of ModelProto needed at runtime: * pieces, scores, piece types, trainer model_type, and normalizer options. The diff --git a/libffwd/sentencepiece.h b/libffwd/sentencepiece.h index 7f5314a..b79f47c 100644 --- a/libffwd/sentencepiece.h +++ b/libffwd/sentencepiece.h @@ -1,8 +1,8 @@ /* - * sentencepiece.h - SentencePiece tokenizer for XLM-R style models. + * SentencePiece tokenizer for XLM-R style models. * - * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`, - * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R + * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model, + * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R * and multilingual E5. It encodes text only; frontends add the model's special * token layout, e.g. <s> ... </s> for XLM-R. */ diff --git a/libffwd/threadpool.c b/libffwd/threadpool.c index 2521a6d..6cb4252 100644 --- a/libffwd/threadpool.c +++ b/libffwd/threadpool.c @@ -1,5 +1,5 @@ /* - * tp.c - CPU thread pool + * CPU thread pool. */ #if defined(__linux__) && !defined(_GNU_SOURCE) @@ -116,15 +116,19 @@ void tp_set_threads(int n) { } #ifndef __APPLE__ -/* CPU bandwidth this process may actually use, in whole cores, or 0 if there - * is no cgroup limit. Containers (Docker --cpus, Kubernetes, RunPod) expose - * every host CPU through sysconf()/cpuset, but the CFS quota caps the CPU time - * the process really gets. Sizing the BLAS/thread pool to the host count then - * oversubscribes that quota: the threads spend the period's budget early and - * the scheduler throttles them for the rest of each period, which adds large, - * bursty stalls (e.g. a 16-thread default against a 5.1-core quota ran the - * 0.6B model ~2x slower than a 5-thread pool). Floor, not ceil, so a fully - * busy pool stays under the quota and never trips throttling. */ +/* CPU quota available to this process, in whole cores, or 0 if uncapped. + * Containers such as Docker --cpus, Kubernetes, and RunPod may expose every host + * CPU through sysconf()/cpuset, while CFS quota limits the CPU time the process + * can actually use. + * + * Sizing the BLAS/thread pool to the host CPU count oversubscribes that quota: + * threads use the period budget early, then get throttled for the rest of the + * period, causing large bursty stalls. Example: a 16-thread default against a + * 5.1-core quota ran the 0.6B model about 2x slower than a 5-thread pool. + * + * Floor instead of ceil, so a fully busy pool stays under quota and avoids + * throttling. + */ static int cgroup_cpu_quota(void) { /* cgroup v2: "/sys/fs/cgroup/cpu.max" is "<quota> <period>", or * "max <period>" when unlimited. */ diff --git a/libffwd/threadpool.h b/libffwd/threadpool.h index 1dcc80a..f9739a1 100644 --- a/libffwd/threadpool.h +++ b/libffwd/threadpool.h @@ -1,12 +1,14 @@ #ifndef TP_THREADPOOL_H #define TP_THREADPOOL_H -/* Persistent worker pool for the CPU math kernels: a fixed set of threads that - * sleep until parallel_for() hands them a callback, plus the CPU-count helper - * that sizes the pool (cgroup-quota aware). Separated from the math so the - * kernels depend only on the parallel_for / tp_num_threads interface, not on - * the pool's internals. kernels.h includes this, so kernel callers need no - * extra include. */ +/* Persistent worker pool for CPU kernels. + * Uses a fixed thread set that sleeps until parallel_for() gives it a callback. + * Also provides the cgroup-quota-aware CPU-count helper used to size the pool. + * + * Kept separate from the kernels so they depend only on parallel_for and + * tp_num_threads, not pool internals. + * + * kernels.h includes this, so kernel callers need no extra include. */ /* Worker callback: (thread index, total threads, user arg). */ typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c index df5861e..2af151a 100644 --- a/libffwd/tokenize.c +++ b/libffwd/tokenize.c @@ -1,10 +1,14 @@ /* - * tokenize.c - model-aware tokenization: text -> the token ids a model expects. + * Model-aware tokenization: text -> token ids the model expects. * - * Backend-independent, so this is its own handle (ffwd_tok_t), not part of the - * per-backend ffwd_t. ffwd_tok_open picks the tokenizer by the files present - * and resolves the model's special-token ids; ffwd_tokenize applies that - * model's layout (BERT/XLM-R [CLS]..[SEP], Qwen appended terminal token). + * Backend-independent, so this uses its own handle, ffwd_tok_t, instead of the + * per-backend ffwd_t. + * + * ffwd_tok_open selects the tokenizer from the files present and resolves the + * model's special-token ids. + * + * ffwd_tokenize applies the model's token layout: BERT/XLM-R [CLS]..[SEP], or + * Qwen appended terminal token. */ #include "ffwd.h" diff --git a/libffwd/vec.c b/libffwd/vec.c index f848be0..ec6b27d 100644 --- a/libffwd/vec.c +++ b/libffwd/vec.c @@ -2,13 +2,7 @@ #include <math.h> -/* ======================================================================== - * Vector helpers - * - * Standalone post-processing on embedding vectors: callers (CLI, server) use - * these to L2-normalize outputs and to score pooled embeddings with cosine - * similarity. They touch no model or tensor internals, only the public API. - * ======================================================================== */ +/* Vector helpers */ int ffwd_l2_normalize(float *vec, int dim) { if (!vec || dim <= 0) diff --git a/libffwd/wordpiece.c b/libffwd/wordpiece.c index d3f62dd..dc0d995 100644 --- a/libffwd/wordpiece.c +++ b/libffwd/wordpiece.c @@ -1,21 +1,24 @@ /* - * tok_wp_tokenizer.c - BERT-family WordPiece tokenizer. + * BERT-family WordPiece tokenizer. * - * Pipeline (matching transformers' BertTokenizer): + * Pipeline matching transformers' BertTokenizer: * BasicTokenizer: clean control/whitespace -> space CJK ideographs -> - * whitespace split -> (lowercase + strip accents) -> split on punctuation - * WordPiece: greedy longest-match-first against vocab.txt, continuation - * pieces marked "##", whole token -> [UNK] when any piece fails to match. + * whitespace split -> lowercase + strip accents, if enabled -> + * split on punctuation. + * WordPiece: greedy longest-match-first against vocab.txt, with continuation + * pieces marked "##". If any piece fails, the whole token becomes [UNK]. * - * Unicode handling is full and exact against transformers, driven by generated - * tables (tok_wp_unicode.h): punctuation and control by Unicode category, - * whitespace by the complete Zs set, the CJK split by the BERT ideograph ranges, - * and the do_lower_case fold by a lowercase+NFD+strip-marks table (with Hangul - * decomposed by formula). Cased and uncased multilingual encoders (e.g. - * bert-base-multilingual-cased / -uncased) match Hugging Face exactly, including - * Greek, Cyrillic, Vietnamese and Korean. The fold is per-codepoint, matching - * transformers, which lowercases character by character (no context-sensitive - * Greek final sigma). + * Unicode handling fully matches transformers using generated tables + * (unicode.h): Unicode category for punctuation/control, full Zs set for + * whitespace, BERT ideograph ranges for CJK splitting, and a lowercase + NFD + + * strip-marks table for do_lower_case, with Hangul decomposed by formula. + * + * Cased and uncased multilingual encoders, such as + * bert-base-multilingual-cased / -uncased, match Hugging Face exactly, + * including Greek, Cyrillic, Vietnamese, and Korean. + * + * Folding is per-codepoint, matching transformers, which lowercases one + * character at a time, with no context-sensitive Greek final sigma. */ #include "wordpiece.h" @@ -27,7 +30,7 @@ #include <stdlib.h> #include <string.h> -/* ===================== string -> int open-addressing map ===================== */ +/* string -> int open-addressing map */ typedef struct { const char *key; /* points into tok->id_to_token; not owned */ @@ -55,7 +58,7 @@ static void map_put(wp_entry_t *map, int cap, const char *key, int val) { int pos = (int)(fnv1a_hash(key) & mask); while (map[pos].key) { if (!strcmp(map[pos].key, key)) { - map[pos].val = val; /* keep the first (lowest) id, like a dict insert */ + map[pos].val = val; /* keep the first/lowest id, like a dict insert */ return; } pos = (int)((pos + 1) & (int)mask); @@ -77,10 +80,10 @@ static int map_get(const wp_entry_t *map, int cap, const char *key) { return -1; } -/* ============================== UTF-8 helpers ================================ */ +/* UTF-8 helpers */ -/* Decode one codepoint. Returns bytes consumed (>=1); on a malformed lead byte - * emits U+FFFD and consumes one byte so iteration always advances. */ +/* Decode one codepoint. Returns bytes consumed (>=1). + * On malformed lead byte, emits U+FFFD and consumes one byte so iteration advances. */ static int utf8_decode(const unsigned char *p, const unsigned char *end, uint32_t *cp) { unsigned char c = p[0]; if (c < 0x80) { @@ -140,7 +143,7 @@ static int utf8_encode(uint32_t cp, char out[4]) { return 4; } -/* ========================== Unicode classification ========================== */ +/* Unicode classification */ static int wp_is_whitespace(uint32_t cp) { if (cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r') @@ -164,9 +167,9 @@ static int wp_in_ranges(uint32_t cp, const wp_range_t *r, int n) { return 0; } -/* transformers' _is_control: tab/newline/carriage-return are whitespace, not - * control (they fall through to wp_is_whitespace); everything else is Unicode - * category Cc, Cf, or Co. */ +/* Match transformers' _is_control: + * tab/newline/carriage-return are whitespace, not control, and fall through to + * wp_is_whitespace; everything else is Unicode category Cc, Cf, or Co. */ static int wp_is_control(uint32_t cp) { if (cp == '\t' || cp == '\n' || cp == '\r') return 0; @@ -184,10 +187,10 @@ static int wp_is_cjk(uint32_t cp) { (cp >= 0xF900 && cp <= 0xFAFF) || (cp >= 0x2F800 && cp <= 0x2FA1F); } -/* Hangul syllable decomposition constants (the standard algorithm). Hangul - * syllables are not cased, so their do_lower_case fold is the canonical NFD into - * leading (L), vowel (V), and optional trailing (T) jamo - none combining marks - * - which keeps them out of the generated WP_FOLD table. */ +/* Hangul syllable decomposition constants for the standard algorithm. + * Hangul syllables are not cased, so their do_lower_case fold is canonical NFD: + * leading (L), vowel (V), and optional trailing (T) jamo. + * None are combining marks, so they stay out of the generated WP_FOLD table. */ enum { WP_HANGUL_S = 0xAC00, WP_HANGUL_L = 0x1100, @@ -198,10 +201,12 @@ enum { WP_HANGUL_SCOUNT = 11172 }; -/* Apply the do_lower_case fold (Python str.lower then NFD then drop combining - * marks) to one codepoint, writing 0..3 output codepoints to out and the count - * to *n_out. 0 outputs means the codepoint is a dropped mark. Hangul decomposes - * by formula; everything else is the generated WP_FOLD table or an identity. */ +/* Apply do_lower_case to one codepoint: + * lowercase using Python-compatible str.lower behavior, then NFD, + * then drop combining marks. + * Writes 0..3 output codepoints to out and the count to *n_out. + * 0 outputs means the codepoint was a dropped mark. + * Hangul decomposes by formula; all else uses WP_FOLD or identity. */ static void wp_fold(uint32_t cp, uint32_t out[3], int *n_out) { if (cp >= WP_HANGUL_S && cp < WP_HANGUL_S + WP_HANGUL_SCOUNT) { int s = (int)(cp - WP_HANGUL_S); @@ -235,7 +240,7 @@ static void wp_fold(uint32_t cp, uint32_t out[3], int *n_out) { *n_out = 1; } -/* =============================== workspace ================================== */ +/* workspace */ struct tok_wp_workspace { uint32_t *cps; /* cleaned + CJK-spaced codepoints of the whole input */ @@ -310,7 +315,7 @@ void tok_wp_workspace_free(tok_wp_workspace_t *ws) { free(ws); } -/* ============================== WordPiece core ============================== */ +/* WordPiece core */ /* Append the WordPiece ids of one BasicTokenizer sub-word (codepoints * word[0..n)) to ws->ids[*n_out]. The whole sub-word maps to [UNK] when it is @@ -381,8 +386,8 @@ tok_wp_subword(const tok_wp_t *tok, tok_wp_workspace_t *ws, const uint32_t *word return 0; } -/* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. Returns the - * token count, or -1 on allocation failure. */ +/* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. + * Returns the token count, or -1 on allocation failure. */ static int tok_wp_run(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text) { /* clean_text + tokenize_chinese_chars, in one pass over the input. */ const unsigned char *p = (const unsigned char *)text; @@ -460,7 +465,7 @@ static int tok_wp_run(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *t return n_out; } -/* ================================ loading =================================== */ +/* loading */ static char *read_file(const char *path, long *out_len) { FILE *f = fopen(path, "rb"); @@ -493,7 +498,7 @@ static char *read_file(const char *path, long *out_len) { } /* do_lower_case from config.json: default true (BERT uncased). A - * lightweight scan is enough - we only need one boolean and avoid a JSON dep + * lightweight scan is enough; we only need one boolean and avoid a JSON dep * here, mirroring how the config layer reads small flags. */ static int read_do_lower_case(const char *model_dir) { char path[2048]; @@ -595,7 +600,7 @@ tok_wp_t *tok_wp_load(const char *model_dir) { return tok; } -/* ============================== public API ================================= */ +/* public API*/ const char *tok_wp_decode(const tok_wp_t *tok, int id) { if (!tok || id < 0 || id >= tok->vocab_size) diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h index c93140a..f8a63d6 100644 --- a/libffwd/wordpiece.h +++ b/libffwd/wordpiece.h @@ -1,15 +1,16 @@ /* - * wordpiece.h - BERT-family WordPiece tokenizer. + * BERT-family WordPiece tokenizer. * - * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional - * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting) - * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes - * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the - * byte-level BPE backend leaves terminal/separator tokens to the frontend. + * Implements BERT tokenization: BasicTokenizer (cleanup, optional lowercase + * and accent stripping, CJK spacing, whitespace splitting, and punctuation + * splitting), then greedy longest-match WordPiece using vocab.txt. * - * The public surface mirrors bpe.h (load / encode with a reusable - * workspace / decode / resolve a named special token / free) so both can sit - * behind one tokenizer interface later. + * Encodes text only. The frontend adds [CLS]/[SEP], just as the byte-level BPE + * backend leaves terminal and separator tokens to the frontend. + * + * Public API mirrors bpe.h: load, encode with reusable workspace, decode, + * resolve named special tokens, and free. This lets both tokenizers share one + * tokenizer interface later. */ #ifndef TOK_WP_H @@ -29,28 +30,29 @@ typedef struct { typedef struct tok_wp_workspace tok_wp_workspace_t; -/* Load from <model_dir>/vocab.txt, reading do_lower_case from - * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on - * error (missing vocab.txt, missing [UNK], allocation failure). */ +/* Load <model_dir>/vocab.txt and do_lower_case from + * <model_dir>/tokenizer_config.json; defaults to lowercase. + * Returns NULL on error: missing vocab.txt, missing [UNK], or allocation failure. */ tok_wp_t *tok_wp_load(const char *model_dir); -/* Decode a single id to its vocab token string (internal pointer), or NULL. */ +/* Decode one id to its vocab token string, or NULL. + * Returns an internal pointer. */ const char *tok_wp_decode(const tok_wp_t *tok, int id); /* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */ int tok_wp_token_id(const tok_wp_t *tok, const char *token); -/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on - * error (and *out_n_tokens = 0). */ +/* Encode UTF-8 text into a malloc'd id array and set *out_n_tokens. + * Returns NULL on error and sets *out_n_tokens = 0. */ int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n_tokens); -/* Reusable scratch for allocation-light repeated encoding. */ +/* Reusable scratch space for repeated encoding with fewer allocations. */ tok_wp_workspace_t *tok_wp_workspace_new(void); void tok_wp_workspace_free(tok_wp_workspace_t *ws); -/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1 - * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to - * the number of tokens needed/emitted in all cases. */ +/* Encode into caller-provided out_ids[0..out_cap-1]. + * Returns 0 on success, -1 on error, or -2 if out_cap is too small. + * If non-NULL, *out_n_tokens is always set to tokens needed/emitted. */ int tok_wp_encode_into(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text, @@ -58,8 +60,8 @@ int tok_wp_encode_into(const tok_wp_t *tok, int out_cap, int *out_n_tokens); -/* Same tokenization as tok_wp_encode(), reusing ws internally and - * performing only the final returned int[] allocation. */ +/* Same tokenization as tok_wp_encode(). + * Reuses ws internally and only allocates the returned int[]. */ int *tok_wp_encode_with_workspace(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text, diff --git a/tests/bert_dump.c b/tests/bert_dump.c index 8f820ee..5e27ce6 100644 --- a/tests/bert_dump.c +++ b/tests/bert_dump.c @@ -1,10 +1,15 @@ -/* Parity driver for the BERT-family forward. Loads a sentence-transformers - * checkpoint, wraps each argument with [CLS]/[SEP], runs the BERT forward - * (mean pool + L2 normalize inside ffwd_model_encode_into), and prints - * "<dim> <f> <f> ..." per line. check_bert_parity.py builds this with - * `make parity-bert-driver` and compares against SentenceTransformer.encode. - * This one needs the real model weights, so it stays a manual --model-dir - * check rather than a hermetic stored-vector test. */ +/* Parity driver for the BERT-family forward. + * + * Loads a sentence-transformers checkpoint, wraps each argument with [CLS]/[SEP], + * runs the BERT forward (mean pool + L2 normalize inside ffwd_model_encode_into), + * and prints "<dim> <f> <f> ..." per line. + * + * check_bert_parity.py builds this with `make parity-bert-driver` and compares against + * SentenceTransformer.encode. + * + * Needs the real model weights to run. + */ + #include "internal.h" #include "wordpiece.h" diff --git a/tests/check_bert_parity.py b/tests/check_bert_parity.py index 2500a5f..93a51f4 100755 --- a/tests/check_bert_parity.py +++ b/tests/check_bert_parity.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Compare the C BERT-family forward against SentenceTransformers. +"""Compare the BERT-family forward against SentenceTransformers. Builds a small C driver that loads the WordPiece tokenizer and the BERT model from a sentence-transformers checkpoint, wraps each text with [CLS]/[SEP], @@ -7,8 +7,7 @@ encodes it (BERT forward + mean pool + L2 normalize), and prints the vector. The reference is SentenceTransformer(model).encode(normalize_embeddings=True), which runs the same pipeline. Defaults to all-MiniLM-L6-v2. -This one needs the real model weights to run the C forward, so it stays a -manual --model-dir check rather than a hermetic stored-vector test. +Needs the real model weights to run the C forward. uv run --python 3.12 --with-requirements requirements-reference.txt \ tests/check_bert_parity.py --model-dir DIR diff --git a/tests/check_contextual_batch_parity.py b/tests/check_contextual_batch_parity.py index 6cd7021..e7f1e94 100755 --- a/tests/check_contextual_batch_parity.py +++ b/tests/check_contextual_batch_parity.py @@ -1,11 +1,16 @@ #!/usr/bin/env python3 -"""Run the contextual batch-versus-singleton parity check. +"""Run the contextual batch-vs-singleton parity check. -The C side is tests/contextual_batch.c, built by the Makefile target -`parity-context-driver`; this script builds it and runs it with the requested -parameters. It compares the single-document path against the batched path in C -(no external reference), so it needs a model directory but no Python ML stack. +The C test is tests/contextual_batch.c, built by the Makefile target +`parity-context-driver`. +This script builds the driver and runs it with the requested parameters. It +compares the single-document path against the batched path in C, with no +external reference. + +Requires a model directory. + +Usage: tests/check_contextual_batch_parity.py --model-dir DIR """ diff --git a/tests/check_late_interaction.py b/tests/check_late_interaction.py index e9dedb4..5bfb02f 100755 --- a/tests/check_late_interaction.py +++ b/tests/check_late_interaction.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 """Smoke-check late-interaction token vectors and MaxSim ranking. -The C side is tests/late_check.c, built by the Makefile target -`late-check-driver`. Plain mode runs the driver against a real late model and -checks the expected ranking; it needs only python3 and a late model directory -(e.g. pplx-embed-v1-late-0.6b). With --compare-pylate it also compares the C -token vectors against PyLate, which needs the pylate package: +The C test is tests/late_check.c, built by the Makefile target +`late-check-driver`. +Plain mode runs the driver against a real late model and checks the expected +ranking. It requires only python3 and a late model directory, such as +pplx-embed-v1-late-0.6b. + +With --compare-pylate, it also compares C token vectors against PyLate. This +requires the pylate package. + +Usage: tests/check_late_interaction.py --model-dir DIR uv run --python 3.12 --with pylate tests/check_late_interaction.py \ --model-dir DIR --compare-pylate diff --git a/tests/check_perplexity_sdk_compat.py b/tests/check_perplexity_sdk_compat.py index f0511f6..2e9e949 100755 --- a/tests/check_perplexity_sdk_compat.py +++ b/tests/check_perplexity_sdk_compat.py @@ -66,9 +66,7 @@ CONTEXT_DOCS = { } -# ------------------------------------------------------------------------- # Server launch (skipped when --base-url points at a running server) -# ------------------------------------------------------------------------- def ensure_server() -> None: if not SERVER_BIN.exists(): @@ -120,9 +118,7 @@ def wait_ready(base_url: str, api_key: str, model: str, die_with_log(proc, log_path, "server did not become ready in time") -# ------------------------------------------------------------------------- # SDK checks -# ------------------------------------------------------------------------- def decode_int8(embedding: str) -> np.ndarray: raw = base64.b64decode(embedding) diff --git a/tests/check_server_api.py b/tests/check_server_api.py index cfdc6aa..23f923e 100755 --- a/tests/check_server_api.py +++ b/tests/check_server_api.py @@ -64,9 +64,7 @@ def expect(cond: bool, what: str) -> None: fail(what) -# -------------------------------------------------------------------------- # Build + fixture + launch -# -------------------------------------------------------------------------- def ensure_tool(path: str, make_dir: str, make_target: str) -> None: """Build a Make target if its output binary is missing.""" diff --git a/tests/check_tokenizer_parity.py b/tests/check_tokenizer_parity.py index acea177..70eea3c 100755 --- a/tests/check_tokenizer_parity.py +++ b/tests/check_tokenizer_parity.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -"""Check the C BPE tokenizer against stored reference vectors. +"""Check the BPE tokenizer against stored reference vectors. The reference ids live in tests/test-vectors/bpe/expected.json, generated from -the Qwen tokenizer by devtools/gen_test_vectors.py. This builds the C BPE driver -(tests/tokenizer_dump), runs it against the checked-in vocab.json + merges.txt, -and compares the ids exactly. It imports no transformers and downloads no model, -so it is hermetic and fast. +the Qwen tokenizer by devtools/gen_test_vectors.py. + +This builds the C BPE driver (tests/tokenizer_dump), runs it against the checked-in +vocab.json + merges.txt, and compares the ids exactly. tests/check_tokenizer_parity.py tests/check_tokenizer_parity.py --cc clang diff --git a/tests/check_wordpiece_parity.py b/tests/check_wordpiece_parity.py index b03da7e..8f09394 100755 --- a/tests/check_wordpiece_parity.py +++ b/tests/check_wordpiece_parity.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -"""Check the C WordPiece tokenizer against stored reference vectors. +"""Check the WordPiece tokenizer against stored reference vectors. The reference ids live in tests/test-vectors/wordpiece/expected.json, generated -from the slow BertTokenizer by devtools/gen_test_vectors.py. This builds the C -WordPiece driver (tests/wordpiece_dump), runs it against the checked-in -vocab.txt + tokenizer_config.json, and compares the ids exactly. It imports no -transformers and downloads no model, so it is hermetic and fast. +from the slow BertTokenizer by devtools/gen_test_vectors.py. + +This builds the C WordPiece driver (tests/wordpiece_dump), runs it against the checked-in +vocab.txt + tokenizer_config.json, and compares the ids exactly. tests/check_wordpiece_parity.py tests/check_wordpiece_parity.py --cc clang diff --git a/tests/check_workspace_api.py b/tests/check_workspace_api.py index 05f0e8c..40e51b6 100755 --- a/tests/check_workspace_api.py +++ b/tests/check_workspace_api.py @@ -1,16 +1,20 @@ #!/usr/bin/env python3 -"""Run the workspace / model API smoke test against real weights. +"""Run the workspace/model API smoke test with real weights. -The C side is tests/test_workspace.c, the same source `make test` builds with -the synthetic fixture. Given a MODEL_DIR it loads real weights from that -directory instead and exercises the workspace and batch APIs. Needs only -python3 and a model directory (with vocab.json), no reference stack. +The C test is tests/test_workspace.c, the same source `make test` builds with +the synthetic fixture. With MODEL_DIR, it loads real weights from that directory +instead and exercises the workspace and batch APIs. -For mean-pooled models, the test additionally checks that a whole-sequence span -pools to the embedding and that half-spans recombine by token-count weighting. -For last-token/CLS models, those mean-span identities are skipped because the -sentence embedding and contextual span APIs intentionally pool different rows. +Requires only python3 and a model directory with vocab.json; no reference stack. +For mean-pooled models, also checks that: +- a whole-sequence span pools to the embedding +- half-spans recombine by token-count weighting + +For last-token/CLS models, these mean-span checks are skipped because sentence +embedding and contextual span APIs intentionally pool different rows. + +Usage: tests/check_workspace_api.py --model-dir DIR """ diff --git a/tests/contextual_batch.c b/tests/contextual_batch.c index 9700e82..1c4b5be 100644 --- a/tests/contextual_batch.c +++ b/tests/contextual_batch.c @@ -3,7 +3,9 @@ * No external library is involved; it checks the C code against itself. With a * positive run count it also times the batched path. check_contextual_batch_parity.py * builds this with `make parity-context-driver` (or parity-context-driver-mlx) - * and passes the model dir plus tolerances as positional arguments. */ + * and passes the model dir plus tolerances as positional arguments. + */ + #include "internal.h" #ifdef CHECK_MLX # include "mlx.h" diff --git a/tests/gen_fixture.c b/tests/gen_fixture.c index bf01fd6..0a52fb9 100644 --- a/tests/gen_fixture.c +++ b/tests/gen_fixture.c @@ -1,4 +1,4 @@ -/* gen_fixture.c - write a tiny model fixture to a directory. +/* Write a tiny model fixture to a directory. * * Thin command-line front end over tests/model_fixture.h, so out-of-process * callers (shell scripts, Python) can materialize the same synthetic model the diff --git a/tests/late_check.c b/tests/late_check.c index e0335e8..bb868d3 100644 --- a/tests/late_check.c +++ b/tests/late_check.c @@ -1,8 +1,11 @@ /* Late-interaction smoke driver: encodes a query and three documents to * token vectors, runs scalar and batched MaxSim, and checks the expected - * ranking. With --json it emits vectors and scores for the optional PyLate - * comparison in check_late_interaction.py, which builds this via - * `make late-check-driver` (or late-check-driver-mlx). */ + * ranking. + * With --json it emits vectors and scores for the optional PyLate comparison + * in check_late_interaction.py, which builds this via make late-check-driver + * (or late-check-driver-mlx). + */ + #include "internal.h" #ifdef CHECK_MLX # include "mlx.h" diff --git a/tests/model_fixture.h b/tests/model_fixture.h index bd613b2..fd2a282 100644 --- a/tests/model_fixture.h +++ b/tests/model_fixture.h @@ -1,4 +1,4 @@ -/* tests/model_fixture.h - shared model+tokenizer fixture writer. */ +/* Shared model+tokenizer fixture writer. */ #ifndef MODEL_FIXTURE_H #define MODEL_FIXTURE_H diff --git a/tests/test_bf16_model.c b/tests/test_bf16_model.c index 6c4b2a4..8b18adc 100644 --- a/tests/test_bf16_model.c +++ b/tests/test_bf16_model.c @@ -1,5 +1,5 @@ -/* tests/test_bf16_model.c - F32 vs BF16 vs F16 loader/dispatch parity on a tiny - * synthetic model. Hermetic: builds its own model dirs. Runs via `make test`. +/* F32 vs BF16 vs F16 loader/dispatch parity on a tiny synthetic model. + * Hermetic: builds its own model dirs. Runs via make test. * F16 has no fused kernel: the loader widens it to F32 once at load, so an F16 * copy must match the F32 copy as closely as the BF16 copy does. */ diff --git a/tests/test_bpe.c b/tests/test_bpe.c index 1674b4e..02e5078 100644 --- a/tests/test_bpe.c +++ b/tests/test_bpe.c @@ -1,8 +1,9 @@ -/* tests/test_tokenizer.c - hermetic byte-level BPE tokenizer tests. +/* Hermetic byte-level BPE tokenizer tests. * Synthesizes a complete GPT-2-style vocab.json (all 256 byte tokens plus * merged tokens) and merges.txt, then checks exact token ids through the * integer-pair fast path, and that dropping the integer tables (forcing the - * string-based fallback) produces identical ids. Runs via `make test`. */ + * string-based fallback) produces identical ids. Runs via make test. + */ #include "bpe.h" #include "tok_fixture.h" diff --git a/tests/test_cls.c b/tests/test_cls.c index b6e2dfb..b929a81 100644 --- a/tests/test_cls.c +++ b/tests/test_cls.c @@ -1,5 +1,6 @@ -/* tests/test_cls.c - hermetic CLS (first-token) pooling across the load path, - * the encode path, and the public pooling API. */ +/* Hermetic CLS (first-token) pooling across the load path, the encode path, and the + * public pooling API. + */ #include "internal.h" #include "tiny_model.h" diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 37847c1..f7e4680 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -1,6 +1,6 @@ -/* tests/test_kernels.c - golden tests for the low-level math kernels. - * Runs via `make test`, `make -C tests test_kernels test_kernels_generic`, - * or `./all.bash check-kernel-golden`. No model files required. */ +/* Golden tests for the low-level math kernels. + * Runs via make test, make -C tests test_kernels test_kernels_generic. No model files required. + */ #include "kernels.h" #include "threadpool.h" diff --git a/tests/test_late.c b/tests/test_late.c index 37a1bf1..2bf2053 100644 --- a/tests/test_late.c +++ b/tests/test_late.c @@ -1,6 +1,6 @@ -/* tests/test_late.c - late-interaction (ColBERT-style MaxSim) API tests. +/* Late-interaction (ColBERT-style MaxSim) API tests. * Hermetic: synthesizes a tiny base model, a 1_Dense projection head, and a - * byte-complete tokenizer fixture. Runs via `make test`. */ + * byte-complete tokenizer fixture. Runs via make test. */ #include "internal.h" #include "bpe.h" diff --git a/tests/test_qwen2.c b/tests/test_qwen2.c index 9567aad..a322c33 100644 --- a/tests/test_qwen2.c +++ b/tests/test_qwen2.c @@ -1,4 +1,4 @@ -/* tests/test_qwen2.c - hermetic GTE-Qwen2 block and task semantics. */ +/* Hermetic GTE-Qwen2 block and task semantics. */ #include "internal.h" #include "tiny_model.h" diff --git a/tests/test_qwen3.c b/tests/test_qwen3.c index 50f7e02..a685f03 100644 --- a/tests/test_qwen3.c +++ b/tests/test_qwen3.c @@ -1,4 +1,4 @@ -/* tests/test_qwen3.c - hermetic Qwen3 model semantics. */ +/* Hermetic Qwen3 model semantics. */ #include "internal.h" #include "tiny_model.h" diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c index 5f748cf..c9a06fb 100644 --- a/tests/test_safetensors.c +++ b/tests/test_safetensors.c @@ -1,6 +1,6 @@ -/* tests/test_safetensors.c - safetensors parser limit and rejection tests. - * Hermetic: synthesizes every fixture in a temp directory. Runs via - * `make test`. */ +/* safetensors parser limit and rejection tests. + * Hermetic: synthesizes every fixture in a temp directory. + */ #include "safetensors.h" diff --git a/tests/test_sentencepiece.c b/tests/test_sentencepiece.c index c04b5c1..cfbe1d1 100644 --- a/tests/test_sentencepiece.c +++ b/tests/test_sentencepiece.c @@ -1,4 +1,4 @@ -/* tests/test_sentencepiece.c - hermetic SentencePiece Unigram checks. */ +/* Hermetic SentencePiece Unigram checks. */ #include "sentencepiece.h" diff --git a/tests/test_wordpiece.c b/tests/test_wordpiece.c index 289260c..2e95f3e 100644 --- a/tests/test_wordpiece.c +++ b/tests/test_wordpiece.c @@ -1,7 +1,7 @@ -/* tests/test_wordpiece.c - hermetic WordPiece tokenizer checks. A synthetic - * vocab exercises greedy ## matching, [UNK], casing, accent stripping, - * punctuation splitting, CJK char-splitting, special-token resolution, and the - * workspace encode paths. No model files or network needed. */ +/* Hermetic WordPiece tokenizer checks. A synthetic vocab exercises greedy + * ## matching, [UNK], casing, accent stripping, punctuation splitting, CJK char-splitting, + * special-token resolution, and the workspace encode paths. + */ #include "wordpiece.h" diff --git a/tests/test_workspace.c b/tests/test_workspace.c index ca5b136..1e0f4a4 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -1,7 +1,7 @@ -/* tests/test_workspace.c - model/workspace API lifecycle tests. +/* Model/workspace API lifecycle tests. * Hermetic by default (synthesizes a tiny model + tokenizer fixture); * pass a MODEL_DIR to run the same checks against real weights, which is - * what tests/integration/check_workspace_api.py does. Runs via `make test`. */ + * what tests/integration/check_workspace_api.py does. Runs via make test. */ #include "internal.h" #include "bpe.h" diff --git a/tests/test_xlm_roberta.c b/tests/test_xlm_roberta.c index 4370855..82e4159 100644 --- a/tests/test_xlm_roberta.c +++ b/tests/test_xlm_roberta.c @@ -1,4 +1,4 @@ -/* tests/test_xlm_roberta.c - hermetic XLM-R/RoBERTa bridge checks. */ +/* Hermetic XLM-R/RoBERTa bridge checks. */ #include "internal.h" #include "tiny_model.h" diff --git a/tests/tiny_model.h b/tests/tiny_model.h index 58b58ea..cc9bc36 100644 --- a/tests/tiny_model.h +++ b/tests/tiny_model.h @@ -1,6 +1,7 @@ -/* tests/tiny_model.h - synthesize a tiny 1-layer model directory (config.json - * + model.safetensors) for hermetic tests. Values are BF16-representable so an - * F32 and a BF16 copy of the same model produce identical embeddings. */ +/* Synthesize a tiny 1-layer model directory (config.json + model.safetensors) for hermetic + * tests. Values are BF16-representable so an F32 and a BF16 copy of the same model produce + * identical embeddings. + */ #ifndef TINY_MODEL_H #define TINY_MODEL_H diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h index 9aa4170..7deeb75 100644 --- a/tests/tok_fixture.h +++ b/tests/tok_fixture.h @@ -1,7 +1,6 @@ -/* tests/tok_fixture.h - synthesize a complete GPT-2-style byte-level BPE - * tokenizer fixture into a directory: vocab.json with all 256 byte tokens - * (ids 0-255, so any input tokenizes) plus a few merged tokens (ids 256+), - * and the matching merges.txt. */ +/* Synthesize a complete GPT-2-style byte-level BPE tokenizer fixture into a directory: + * vocab.json with all 256 byte tokens (ids 0-255, so any input tokenizes) plus a few merged + * tokens (ids 256+), and the matching merges.txt. */ #ifndef TOK_FIXTURE_H #define TOK_FIXTURE_H diff --git a/tests/tokenizer_dump.c b/tests/tokenizer_dump.c index f6e26aa..4e8e8ec 100644 --- a/tests/tokenizer_dump.c +++ b/tests/tokenizer_dump.c @@ -1,9 +1,14 @@ -/* Parity driver for the BPE tokenizer. Loads vocab.json (merges.txt is read - * from the same directory), encodes each argument through the three encode - * entry points (and cross-checks they agree), then prints "<count> <id> <id> - * ..." per line. check_tokenizer_parity.py builds this with - * `make parity-tokenizer-driver` and compares the ids against the stored - * reference vectors in tests/test-vectors/bpe/. Links libtok. */ +/* BPE tokenizer parity driver. + * Loads vocab.json; merges.txt is read from the same directory. + * Encodes each argument through the three encode entry points and checks that + * they agree. + * + * Prints one line per argument: "<count> <id> <id> ...". + * + * check_tokenizer_parity.py builds this with `make parity-tokenizer-driver` and + * compares ids against reference vectors in tests/test-vectors/bpe/. + */ + #include "bpe.h" #include <stdio.h> diff --git a/tests/wordpiece_dump.c b/tests/wordpiece_dump.c index 5dfb563..952e9e2 100644 --- a/tests/wordpiece_dump.c +++ b/tests/wordpiece_dump.c @@ -1,8 +1,13 @@ -/* Parity driver for the WordPiece tokenizer. Loads a BERT vocab dir, encodes - * each argument through both encode entry points (and cross-checks they agree), - * then prints "<count> <id> <id> ..." per line. check_wordpiece_parity.py builds - * this with `make parity-wordpiece-driver` and compares the ids against the - * stored reference vectors in tests/test-vectors/wordpiece/. Links libtok. */ +/* WordPiece tokenizer parity driver. + * Loads a BERT vocab dir, encodes each argument through both encode entry + * points, and checks that they agree. + * + * Prints one line per argument: "<count> <id> <id> ...". + * + * check_wordpiece_parity.py builds this with `make parity-wordpiece-driver` and + * compares ids against reference vectors in tests/test-vectors/wordpiece/. + */ + #include "wordpiece.h" #include <stdio.h> diff --git a/tools/cli/main.c b/tools/cli/main.c index bb6f2f8..6c84360 100644 --- a/tools/cli/main.c +++ b/tools/cli/main.c @@ -1,5 +1,3 @@ -/* main.c - ffwd-cli command-line tool */ - #include "ffwd.h" #include "internal.h" #include "build.h" @@ -51,9 +49,7 @@ static void print_usage(const char *prog) { prog, prog); } -/* ======================================================================== - * Embed one text: tokenize > forward > return float[dim] - * ======================================================================== */ +/* Embed one text: tokenize > forward > return float[dim] */ typedef struct { ffwd_t *backend; @@ -99,9 +95,7 @@ static void free_tokens(token_buf_t *tokens, int n) { free(tokens[i].ids); } -/* ======================================================================== - * Output helpers - * ======================================================================== */ +/* Output helpers */ static void print_embedding_raw(const float *emb, int dim) { for (int i = 0; i < dim; i++) { @@ -140,9 +134,7 @@ print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t fflush(stdout); } -/* ======================================================================== - * Stdin mode: read lines from stdin, write JSON to stdout - * ======================================================================== */ +/* Stdin mode: read lines from stdin, write JSON to stdout */ static int process_stdin_batch(runner_t *e, char **lines, int n_lines) { if (n_lines <= 0) @@ -275,9 +267,7 @@ static int run_stdin(runner_t *e, int batch_size) { return rc; } -/* ======================================================================== - * Batch mode: embed args or stdin lines, then print similarity or vectors - * ======================================================================== */ +/* Batch mode: embed args or stdin lines, then print similarity or vectors */ static int append_text(char ***texts, int *n_texts, int *cap, const char *s) { if (*n_texts == *cap) { @@ -296,9 +286,12 @@ static int append_text(char ***texts, int *n_texts, int *cap, const char *s) { return 0; } -/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one - * row of space-separated values per line. With print_embs the raw embeddings - * follow, one per line after a blank line (plain-text output only). */ +/* Cosine similarity matrix. + * --json emits a JSON array of rows. + * Otherwise, prints one row of space-separated values per line. + * + * With print_embs, raw embeddings follow after a blank line, one per line. + * This is plain-text output only. */ static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) { if (json) { putchar('['); @@ -434,10 +427,6 @@ oom_texts: return 1; } -/* ======================================================================== - * Main - * ======================================================================== */ - int main(int argc, char *argv[]) { const char *model_dir = NULL; int print_embs = 0; @@ -492,9 +481,7 @@ int main(int argc, char *argv[]) { arg_start++; break; } else if (f[0] == '-' && f[1] == '-') { - /* An unrecognized --option is a hard error, not silently-embedded - * text. Catches typos and stale flags (e.g. the removed --backend) - * loudly instead of running the default path or embedding the flag. */ + /* An unrecognized --option is a hard error. */ fprintf(stderr, "unknown option: %s\n", f); print_usage(prog); return 1; diff --git a/tools/server/base64.h b/tools/server/base64.h index 2e77e2d..41277d1 100644 --- a/tools/server/base64.h +++ b/tools/server/base64.h @@ -5,8 +5,8 @@ /* Standard base64 (with '=' padding). */ -/* Number of base64 characters for n input bytes, excluding any NUL. Aborts - * (die-on-OOM) if n is so large the length would overflow. */ +/* Number of base64 characters for n input bytes, excluding any NUL. + * Aborts (die-on-OOM) if n is so large the length would overflow. */ size_t base64_encoded_len(size_t n); /* Encode n bytes of src as base64 into dst, writing exactly base64_encoded_len(n) diff --git a/tools/server/encode.c b/tools/server/encode.c index 63df965..9b6091f 100644 --- a/tools/server/encode.c +++ b/tools/server/encode.c @@ -1,3 +1,12 @@ +/* Embedding output encoding. + * + * Turns a pooled float vector into the requested wire form. + * + * - OpenAI-compatible models emit true float32 arrays by default. + * - Perplexity-compatible models emit tanh-quantized int8 by default, + * and their "float" view is the decoded int8 value for SDK compatibility. + */ + #include "server_internal.h" #include "util.h" #include "sbuf.h" @@ -8,15 +17,6 @@ #include <stdlib.h> #include <string.h> -/* ======================================================================== - * Embedding output encoding - * - * Turns a pooled float vector into the requested wire form. OpenAI-compatible - * models emit true float32 arrays by default. Perplexity-compatible models - * emit tanh-quantized int8 by default, and their "float" view is the decoded - * int8 value for SDK compatibility. - * ======================================================================== */ - signed char quantize_int8_tanh(float x) { int v = (int)lrintf(tanhf(x) * 127.0f); if (v > 127) @@ -26,9 +26,10 @@ signed char quantize_int8_tanh(float x) { return (signed char)v; } -/* The bytes to base64 for `encoding`. "base64" aliases the float vector - * (*owned = 0); "base64_int8" and "base64_binary" return a freshly allocated - * buffer the caller frees (*owned = 1). */ +/* Bytes to base64-encode. + * "base64" aliases the float vector (*owned = 0). + * "base64_int8" and "base64_binary" return a new buffer for the caller to free + * (*owned = 1). */ static const unsigned char * embedding_bytes(const float *emb, int dims, const char *encoding, size_t *n_out, int *owned) { if (!strcmp(encoding, "base64")) { @@ -71,9 +72,8 @@ static void sbuf_append_base64(sbuf *b, const unsigned char *src, size_t n) { void append_embedding_value( sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) { if (!strcmp(encoding, "float")) { - /* Floats are hand-formatted with %.9g (the shortest form that - * round-trips float32) and streamed into the sbuf, never serialized - * through yyjson's writer. */ + /* Floats are hand-formatted with %.9g (the shortest form that round-trips float32) + * and streamed into the sbuf, never serialized through yyjson's writer. */ sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index); for (int i = 0; i < dims; i++) { if (i) diff --git a/tools/server/handlers.c b/tools/server/handlers.c index 2c98077..204f29e 100644 --- a/tools/server/handlers.c +++ b/tools/server/handlers.c @@ -1,3 +1,17 @@ +/* Endpoint handlers: prepare, batch, execute, render. + * + * Each endpoint has one prepare_* function that validates request JSON and + * tokenizes inputs into a *_request. + * + * The worker in schedule.c groups compatible ready requests and calls the + * matching execute_*_request_list. + * + * execute_*_request_list runs the batched forward pass through model_ffwd_* and + * renders each response through encode.c. + * + * Rerank scores late-interaction token vectors with MaxSim. + */ + #include "server_internal.h" #include "util.h" #include "ffwd.h" @@ -7,16 +21,6 @@ #include <stdlib.h> #include <string.h> -/* ======================================================================== - * Endpoint handlers: prepare, batch, execute, render - * - * One prepare_* per endpoint validates the request JSON and tokenizes its - * inputs into a *_request; the worker (schedule.c) groups compatible - * ready requests and calls the matching execute_*_request_list, which runs the - * batched forward pass through model_ffwd_* and renders each response via - * encode.c. Rerank runs MaxSim over late-interaction token vectors. - * ======================================================================== */ - int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { return a->ready && b->ready && a->model == b->model && a->dims == b->dims && !strcmp(a->encoding, b->encoding); @@ -27,10 +31,11 @@ typedef struct { int output_index; } embedding_batch_item; -/* Hand JSON serialization to the renderer thread only when another batch is - * already queued, so the worker can start that GPU launch while these results - * serialize. With nothing else queued the worker renders inline, which avoids a - * thread hand-off that buys no overlap and adds latency to the reply. */ +/* Send JSON serialization to the renderer thread only when another batch is + * already queued, letting the worker start that GPU launch while these results + * serialize. + * If nothing else is queued, the worker renders inline to avoid a thread handoff + * that adds reply latency without buying overlap. */ static int render_should_defer(http_server *s) { return worker_has_pending_jobs(s); } static int embedding_batch_item_cmp(const void *a, const void *b) { @@ -513,9 +518,9 @@ void prepare_contextual_request(job *j, for (int k = 0; k < n_chunks; k++) { doc->spans[k].start = pos; doc->spans[k].n_tokens = chunk_tokens[k].n_tokens; - /* n_tokens > 0 implies ids != NULL (tokenize_one sets both - * together), and the guard avoids a memcpy(dst, NULL, 0) for - * an empty chunk that the static analyzer reports as UB. */ + /* n_tokens > 0 means ids != NULL because tokenize_one sets both together. + * The guard also avoids memcpy(dst, NULL, 0) for empty chunks, which static + * analyzers report as UB. */ if (chunk_tokens[k].n_tokens > 0) memcpy(doc->ids + pos, chunk_tokens[k].ids, (size_t)chunk_tokens[k].n_tokens * sizeof(*doc->ids)); @@ -585,9 +590,9 @@ void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank validate_rerank_model(root, detail, s, &out->model); yyjson_val *query = yyjson_obj_get(root, "query"); yyjson_val *documents = yyjson_obj_get(root, "documents"); - /* query_str is set only once query passes every check; the tokenize step - * below runs only when detail is empty, so it is non-NULL there. Hoisting it - * also keeps the validated value explicit for the static analyzer. */ + /* query_str is set only once query passes every check; the tokenize step below runs only + * when detail is empty, so it is non-NULL there. + * Hoisting it also keeps the validated value explicit for the static analyzer. */ const char *query_str = NULL; if (!query) { ve_add(detail, "[\"body\",\"query\"]", "field required", "missing"); diff --git a/tools/server/http.c b/tools/server/http.c index a1d347f..6fc21f0 100644 --- a/tools/server/http.c +++ b/tools/server/http.c @@ -1,3 +1,16 @@ +/* HTTP/1.1 connection lifecycle, header parsing, and response framing. + * + * - One refcounted client per connection. + * - The event loop holds a reference while the socket is open; the worker thread + * holds one while a job from that connection is in flight. + * - If a request completes after the peer disconnects, the client is still freed + * safely. + * - read_cb accumulates bytes until a full request is framed, then hands it to + * dispatch_request in server.c. + * - write_cb drains the response, then either closes the connection or arms the + * next pipelined request. + */ + #include "server_internal.h" #include "util.h" @@ -11,17 +24,6 @@ #include <sys/types.h> #include <unistd.h> -/* ======================================================================== - * HTTP/1.1 connection lifecycle, header parsing, and response framing - * - * One client per connection, refcounted: the event loop holds a reference - * while the socket is open and the worker thread holds one while a job derived - * from the connection is in flight, so a request that completes after the peer - * disconnects frees the client safely. read_cb accumulates bytes until a full - * request is framed, then hands off to dispatch_request (server.c); write_cb - * drains the response and either closes or arms the next pipelined request. - * ======================================================================== */ - int set_nonblock(int fd) { int flags = fcntl(fd, F_GETFL, 0); if (flags < 0) diff --git a/tools/server/json.c b/tools/server/json.c index 8f9aec2..43200a8 100644 --- a/tools/server/json.c +++ b/tools/server/json.c @@ -1,3 +1,15 @@ +/* Request validation and error bodies. + * + * This code builds two JSON error shapes: + * - OpenAI-style {"error":{message,type}} for transport/auth failures, via + * append_json_error, json_error_body, and job_set_error. + * - Perplexity-style {"detail":[...]} 422 responses for field validation, built + * from ve_add entries. + * + * parse_* and *_from_root helpers read and validate the request object. + * append_json_string escapes a value for use in a response. + */ + #include "server_internal.h" #include "util.h" @@ -6,16 +18,6 @@ #include <stdlib.h> #include <string.h> -/* ======================================================================== - * Request validation and error bodies - * - * Two JSON shapes are produced here: the OpenAI-style {"error":{message,type}} - * body (append_json_error / json_error_body / job_set_error) for transport and - * auth failures, and the Perplexity-style {"detail":[...]} 422 body built from - * ve_add entries for field validation. The parse_* / *_from_root helpers read - * and validate the request object; append_json_string escapes a value for - * inclusion in a response. */ - static void append_json_error(sbuf *b, const char *message, const char *type) { sbuf_puts(b, "{\"error\":{\"message\":"); append_json_string(b, message ? message : ""); @@ -182,7 +184,7 @@ const char *encoding_from_root(yyjson_val *root, yyjson_mut_doc *detail, embeddi return v; } -/* Parse the `text_type` hint. Returns 1 when the model's query instruction +/* Parse the text_type hint. Returns 1 when the model's query instruction * should be prepended, 0 otherwise (document / absent). Models without a * published query instruction reject the field. */ int text_type_is_query(yyjson_val *root, yyjson_mut_doc *detail, const char *query_instruct) { diff --git a/tools/server/main.c b/tools/server/main.c index b94d438..baa4e39 100644 --- a/tools/server/main.c +++ b/tools/server/main.c @@ -1,10 +1,13 @@ -/* main.c - ffwd-server entry point. +/* ffwd HTTP frontend. * - * The frontend for the ffwd-server binary: argument parsing, model-spec - * handling, and main(). The server logic lives in server.c; this file only - * builds the ffwd_server_config_t and calls ffwd_run_server. It is split - * from server.c so the test suite can include server.c (for its internal - * functions) without a conflicting main() and without a compile-out guard. */ + * Handles argument parsing, model specs, and main(). + * Server logic lives in server.c; this file only builds ffwd_server_config_t + * and calls ffwd_run_server. + * + * Split from server.c so tests can include server.c for internal functions + * without a conflicting main() or compile-out guard. + */ + #include "build.h" #include "server.h" #include "server_internal.h" diff --git a/tools/server/models.c b/tools/server/models.c index 91aee18..2cb35a6 100644 --- a/tools/server/models.c +++ b/tools/server/models.c @@ -1,3 +1,16 @@ +/* Model load/lifecycle, plus tokenization and inference dispatch. + * + * Each loaded model owns resolved serving metadata for the operator-chosen + * label. + * + * Transformer behavior comes from model files through ffwd_config. + * Server-only facts missing from those files, such as output API family and + * Matryoshka floor, come from the load spec. + * + * tokenize_* and model_ffwd_* helpers are the request handlers' single dispatch + * point, hiding the compiled backend choice behind one signature. + */ + #include "server_internal.h" #include "util.h" @@ -8,17 +21,6 @@ #include <string.h> #include <unistd.h> -/* ======================================================================== - * Model load/lifecycle, and tokenize/inference dispatch - * - * Each loaded model owns resolved serving metadata for the operator-chosen - * label. Transformer behavior comes from the model files through ffwd_config; - * server-only facts absent from those files, such as output API family and - * Matryoshka floor, come from the load spec. The tokenize_* and model_ffwd_* - * helpers are the single dispatch point the request handlers call, hiding the - * compiled backend choice behind one signature. - * ======================================================================== */ - loaded_model *loaded_model_for_label(http_server *s, const char *label) { if (!label) return NULL; @@ -195,9 +197,8 @@ void rerank_request_free(rerank_request *r) { memset(r, 0, sizeof(*r)); } -/* Tokenization itself lives in libffwd (ffwd_tok). These thin wrappers only - * add the server's per-stage timing onto the job; the request handlers call - * them unchanged. */ +/* Thin wrappers that add server per-stage timing to the tokenization job. + * Request handlers call them unchanged. */ int tokenize_input( loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) { uint64_t t0 = nstime(); diff --git a/tools/server/sbuf.h b/tools/server/sbuf.h index 3116090..8053609 100644 --- a/tools/server/sbuf.h +++ b/tools/server/sbuf.h @@ -3,9 +3,9 @@ #include <stddef.h> -/* Growable, always-NUL-terminated byte buffer used to build HTTP responses and - * JSON bodies. Growth is die-on-OOM (see server_util.h), so the mutators never - * fail; callers do not check return values. */ +/* Growable, always-NUL-terminated byte buffer for HTTP responses and JSON bodies. + * Growth is die-on-OOM; see server_util.h. + * Mutators never fail, so callers do not check return values. */ typedef struct { char *ptr; diff --git a/tools/server/schedule.c b/tools/server/schedule.c index d57fbed..8254739 100644 --- a/tools/server/schedule.c +++ b/tools/server/schedule.c @@ -1,3 +1,21 @@ +/* Job queue, micro-batching, and completion. + * + * dispatch_request in server.c enqueues one job per framed request. + * + * The inference worker thread, worker_main in server.c, drains jobs with + * collect_job_batch. A short first-arrival wait groups concurrent arrivals. + * + * process_job_group routes each job to its endpoint handler and merges all + * compatible requests into one pool. execute_*_request_list then repacks that + * pool into token-budget batches. + * + * Finished jobs go to the done queue and wake the event loop through the + * completion pipe. + * + * completion_cb writes each response on the loop thread, so sockets are only + * touched there. + */ + #include "server_internal.h" #include "util.h" @@ -9,19 +27,6 @@ #include <sys/time.h> #include <unistd.h> -/* ======================================================================== - * Job queue, micro-batching, and completion - * - * dispatch_request (server.c) enqueues one job per framed request; the - * inference worker thread (worker_main, server.c) drains them with - * collect_job_batch (a short first-arrival wait groups concurrent arrivals) and - * runs process_job_group, which routes each job to its endpoint handler and - * merges all compatible requests into one pool that execute_*_request_list - * re-packs into token-budget batches. Finished jobs go on the done queue and - * wake the event loop through the completion pipe; completion_cb writes each - * response back on the loop thread, so sockets are only ever touched there. - * ======================================================================== */ - static void enqueue_done(job *j) { http_server *s = j->srv; pthread_mutex_lock(&s->mu); @@ -324,12 +329,17 @@ static void process_unknown_job(job *j) { yyjson_doc_free(root); } -/* Parse and tokenize one job into a heap request attached to j->prep, which the - * worker then moves into its request array. Runs on the tokenizer thread, or on - * the worker itself for a request that reached the job queue untokenized. An - * unknown endpoint leaves prep_kind 0 (process_unknown_job handles it); a parse - * failure leaves prep NULL with the 422 already set, so the worker finishes it - * as not-ready. The tokenized flag guards against tokenizing twice. */ +/* Parse and tokenize one job into a heap request attached to j->prep. + * The worker later moves it into its request array. + * + * Runs on the tokenizer thread, or on the worker for requests that reached the + * job queue untokenized. + * + * Unknown endpoints leave prep_kind as 0; process_unknown_job handles them. + * Parse failures leave prep NULL with the 422 already set, so the worker + * finishes them as not-ready. + * + * tokenized prevents tokenizing twice. */ void tokenize_job(http_server *s, job *j) { j->tokenized = 1; if (!j->started_ns) diff --git a/tools/server/server.h b/tools/server/server.h index b8ec7e4..2add3f0 100644 --- a/tools/server/server.h +++ b/tools/server/server.h @@ -1,9 +1,6 @@ #ifndef FFWD_SERVER_H #define FFWD_SERVER_H -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying FFWD_API are exported - * from the shared library; everything else stays internal. */ #ifndef FFWD_API # if defined(__GNUC__) # define FFWD_API __attribute__((visibility("default"))) diff --git a/tools/server/server_internal.h b/tools/server/server_internal.h index 6f1f997..2369031 100644 --- a/tools/server/server_internal.h +++ b/tools/server/server_internal.h @@ -254,7 +254,7 @@ typedef struct { int ready; } rerank_request; -/* ---- server_json.c: request validation and error bodies ---- */ +/* json.c: request validation and error bodies */ void append_json_string(sbuf *b, const char *s); char *json_error_body(const char *message, const char *type, size_t *len); void job_set_error(job *j, int status, const char *message, const char *type); @@ -273,7 +273,7 @@ int text_type_is_query(yyjson_val *root, yyjson_mut_doc *detail, const char *que int dimensions_from_root( yyjson_val *root, yyjson_mut_doc *detail, int min_dim, int max_dim, const char *encoding); -/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */ +/* http.c: connection lifecycle, header parse, response framing */ int set_nonblock(int fd); void client_incref(client *c); void client_decref_n(client *c, int n); @@ -293,7 +293,7 @@ void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask); /* Defined in server.c (request routing): framed requests are handed here. */ void dispatch_request(client *c); -/* ---- server_encode.c: embedding output encoding ---- */ +/* encode.c: embedding output encoding */ signed char quantize_int8_tanh(float x); void append_embedding_value( sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api); @@ -304,14 +304,14 @@ void job_render_free(job *j); void job_set_embedding_render(job *j, const embedding_request *r, const float *embs); int render_job_response(job *j); -/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */ +/* models.c: model load/lifecycle, tokenize/inference dispatch */ loaded_model *loaded_model_for_label(http_server *s, const char *label); void free_token_bufs(token_buf *t, int n); void embedding_request_free(embedding_request *r); void contextual_request_free(contextual_request *r); void late_tokens_free(late_tokens *t); void rerank_request_free(rerank_request *r); -/* Thin timing wrappers over libffwd's ffwd_tokenize / ffwd_tokenize_late. */ +/* Thin timing wrappers over libffwd's ffwd_tokenize/ffwd_tokenize_late. */ int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out); int tokenize_input( loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out); @@ -328,7 +328,7 @@ int finalize_loaded_model(loaded_model *m); int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec); void free_models(http_server *s); -/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ +/* handlers.c: per-endpoint prepare/batch-group/execute */ void prepare_embedding_request(job *j, yyjson_doc *root_doc, http_server *s, embedding_request *out); int embedding_request_compatible(const embedding_request *a, const embedding_request *b); void execute_embedding_request_list(embedding_request **reqs, int n_reqs); @@ -341,7 +341,7 @@ void execute_contextual_request_list(contextual_request **reqs, int n_reqs); void prepare_rerank_request(job *j, yyjson_doc *root_doc, http_server *s, rerank_request *out); void execute_rerank_request(rerank_request *r); -/* ---- server_schedule.c: job queue, micro-batching, completion ---- */ +/* schedule.c: job queue, micro-batching, completion */ void enqueue_job(job *j); int worker_has_pending_jobs(http_server *s); void enqueue_raw_job(job *j); diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c index f1959e4..eaadc9c 100644 --- a/tools/server/test_encode.c +++ b/tools/server/test_encode.c @@ -20,7 +20,7 @@ static void test_quantize_int8_tanh(void) { TEST_ASSERT(quantize_int8_tanh(0.5f) == 59); } -/* Assert that rendering emb under `encoding` puts the base64 of `expect` in the +/* Assert that rendering emb under encoding puts the base64 of expect in the * embedding field of the response object. */ static void assert_base64_field(const float *emb, int dims, diff --git a/tools/server/test_util.h b/tools/server/test_util.h index c6b6a13..551e0fb 100644 --- a/tools/server/test_util.h +++ b/tools/server/test_util.h @@ -4,7 +4,7 @@ /* Shared harness for the ffwd-server unit tests. TEST_ASSERT records failures * (rather than aborting) so a single run reports every failure, not just the * first. Each test file includes this, runs its checks, and ends main with - * `return TEST_REPORT("label");`, which prints a summary and yields the exit + * return TEST_REPORT("label");, which prints a summary and yields the exit * code (0 when clean, 1 otherwise). */ #include <stdio.h> diff --git a/tools/server/util.h b/tools/server/util.h index 3175cd0..97b1c48 100644 --- a/tools/server/util.h +++ b/tools/server/util.h @@ -5,10 +5,13 @@ #include <stddef.h> /* Die-on-OOM allocation, monotonic clocks, and timestamped logging shared by - * the ffwd-server translation units. The x* allocators abort the process on - * failure rather than returning NULL, so callers never null-check them; the - * attributes below carry that contract across translation-unit boundaries so - * the static analyzer does not flag the (correctly) unchecked uses. */ + * ffwd-server translation units. + * + * x* allocators abort on failure instead of returning NULL, so callers never + * null-check them. + * + * The attributes below carry that contract across translation-unit boundaries, + * preventing static-analyzer warnings for correctly unchecked uses. */ #if defined(__GNUC__) || defined(__clang__) # define SERVER_NORETURN __attribute__((noreturn)) |
