diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
| -rw-r--r-- | Makefile | 499 | ||||
| -rw-r--r-- | build.mk | 230 | ||||
| -rw-r--r-- | libffwd/Makefile | 61 | ||||
| -rw-r--r-- | libffwd/act.c (renamed from src/kernels_act.c) | 32 | ||||
| -rw-r--r-- | libffwd/alloc.c (renamed from src/alloc.c) | 12 | ||||
| -rw-r--r-- | libffwd/alloc.h (renamed from src/alloc.h) | 6 | ||||
| -rw-r--r-- | libffwd/attn.c | 464 | ||||
| -rw-r--r-- | libffwd/avx.c (renamed from src/kernels_avx.c) | 186 | ||||
| -rw-r--r-- | libffwd/backend_blas.c | 168 | ||||
| -rw-r--r-- | libffwd/backend_cuda.c | 172 | ||||
| -rw-r--r-- | libffwd/backend_mlx.c | 228 | ||||
| -rw-r--r-- | libffwd/bpe.c (renamed from src/tokenizer_bpe.c) | 124 | ||||
| -rw-r--r-- | libffwd/bpe.h | 74 | ||||
| -rw-r--r-- | libffwd/build.h | 49 | ||||
| -rw-r--r-- | libffwd/config.c (renamed from src/config.c) | 87 | ||||
| -rw-r--r-- | libffwd/config.h | 8 | ||||
| -rw-r--r-- | libffwd/cuda.cu (renamed from src/cuda.cu) | 407 | ||||
| -rw-r--r-- | libffwd/cuda.h (renamed from src/cuda.h) | 64 | ||||
| -rw-r--r-- | libffwd/dtype.h (renamed from src/dtype.h) | 12 | ||||
| -rw-r--r-- | libffwd/ffwd.h | 280 | ||||
| -rw-r--r-- | libffwd/forward.c (renamed from src/forward.c) | 309 | ||||
| -rw-r--r-- | libffwd/gemm.c (renamed from src/kernels_gemm.c) | 101 | ||||
| -rw-r--r-- | libffwd/generic.c (renamed from src/kernels_generic.c) | 14 | ||||
| -rw-r--r-- | libffwd/impl.h | 279 | ||||
| -rw-r--r-- | libffwd/internal.h | 134 | ||||
| -rw-r--r-- | libffwd/kernels.h (renamed from src/kernels.h) | 76 | ||||
| -rw-r--r-- | libffwd/late.c | 218 | ||||
| -rw-r--r-- | libffwd/late_blas.c | 173 | ||||
| -rw-r--r-- | libffwd/mlx.c (renamed from src/mlx.c) | 360 | ||||
| -rw-r--r-- | libffwd/mlx.h (renamed from src/mlx.h) | 117 | ||||
| -rw-r--r-- | libffwd/model.c (renamed from src/model.c) | 352 | ||||
| -rw-r--r-- | libffwd/model_blas.c | 201 | ||||
| -rw-r--r-- | libffwd/model_internal.h (renamed from src/model_internal.h) | 40 | ||||
| -rw-r--r-- | libffwd/model_types.h (renamed from src/internal.h) | 63 | ||||
| -rw-r--r-- | libffwd/neon.c (renamed from src/kernels_neon.c) | 71 | ||||
| -rw-r--r-- | libffwd/norm.c | 142 | ||||
| -rw-r--r-- | libffwd/platform.h | 27 | ||||
| -rw-r--r-- | libffwd/rope.c (renamed from src/kernels_rope.c) | 26 | ||||
| -rw-r--r-- | libffwd/safetensors.c (renamed from src/safetensors.c) | 50 | ||||
| -rw-r--r-- | libffwd/safetensors.h (renamed from src/safetensors.h) | 20 | ||||
| -rw-r--r-- | libffwd/sentencepiece.c (renamed from src/tokenizer_sentencepiece.c) | 144 | ||||
| -rw-r--r-- | libffwd/sentencepiece.h | 88 | ||||
| -rw-r--r-- | libffwd/threadpool.c (renamed from src/threadpool.c) | 59 | ||||
| -rw-r--r-- | libffwd/threadpool.h (renamed from src/threadpool.h) | 22 | ||||
| -rw-r--r-- | libffwd/tokenize.c | 350 | ||||
| -rw-r--r-- | libffwd/unicode.h (renamed from src/tokenizer_unicode.h) | 95 | ||||
| -rw-r--r-- | libffwd/vec.c (renamed from src/vec.c) | 10 | ||||
| -rw-r--r-- | libffwd/wordpiece.c (renamed from src/tokenizer_wordpiece.c) | 80 | ||||
| -rw-r--r-- | libffwd/wordpiece.h | 70 | ||||
| -rw-r--r-- | src/build.h | 64 | ||||
| -rw-r--r-- | src/cli.c | 911 | ||||
| -rw-r--r-- | src/config.h | 8 | ||||
| -rw-r--r-- | src/embed.h | 357 | ||||
| -rw-r--r-- | src/kernels_attn.c | 469 | ||||
| -rw-r--r-- | src/kernels_impl.h | 53 | ||||
| -rw-r--r-- | src/kernels_norm.c | 245 | ||||
| -rw-r--r-- | src/late.c | 383 | ||||
| -rw-r--r-- | src/server.c | 1042 | ||||
| -rw-r--r-- | src/server.h | 58 | ||||
| -rw-r--r-- | src/server_models.c | 627 | ||||
| -rw-r--r-- | src/tokenizer_bpe.h | 86 | ||||
| -rw-r--r-- | src/tokenizer_sentencepiece.h | 99 | ||||
| -rw-r--r-- | src/tokenizer_wordpiece.h | 79 | ||||
| -rw-r--r-- | tools/cli/Makefile | 22 | ||||
| -rw-r--r-- | tools/cli/main.c | 556 | ||||
| -rw-r--r-- | tools/server/Makefile | 49 | ||||
| -rw-r--r-- | tools/server/base64.c (renamed from src/base64.c) | 2 | ||||
| -rw-r--r-- | tools/server/base64.h (renamed from src/base64.h) | 6 | ||||
| -rw-r--r-- | tools/server/encode.c (renamed from src/server_encode.c) | 8 | ||||
| -rw-r--r-- | tools/server/handlers.c (renamed from src/server_handlers.c) | 221 | ||||
| -rw-r--r-- | tools/server/http.c (renamed from src/server_http.c) | 10 | ||||
| -rw-r--r-- | tools/server/json.c (renamed from src/server_json.c) | 20 | ||||
| -rw-r--r-- | tools/server/main.c | 342 | ||||
| -rw-r--r-- | tools/server/models.c | 342 | ||||
| -rw-r--r-- | tools/server/sbuf.c (renamed from src/sbuf.c) | 2 | ||||
| -rw-r--r-- | tools/server/sbuf.h (renamed from src/sbuf.h) | 6 | ||||
| -rw-r--r-- | tools/server/schedule.c (renamed from src/server_schedule.c) | 4 | ||||
| -rw-r--r-- | tools/server/server.c | 414 | ||||
| -rw-r--r-- | tools/server/server.h | 57 | ||||
| -rw-r--r-- | tools/server/server_internal.h (renamed from src/server_internal.h) | 153 | ||||
| -rw-r--r-- | tools/server/test_base64.c | 51 | ||||
| -rw-r--r-- | tools/server/test_encode.c | 90 | ||||
| -rw-r--r-- | tools/server/test_json.c | 143 | ||||
| -rw-r--r-- | tools/server/test_models.c | 26 | ||||
| -rw-r--r-- | tools/server/test_sbuf.c | 35 | ||||
| -rw-r--r-- | tools/server/test_schedule.c | 73 | ||||
| -rw-r--r-- | tools/server/test_util.h | 26 | ||||
| -rw-r--r-- | tools/server/util.c (renamed from src/server_util.c) | 4 | ||||
| -rw-r--r-- | tools/server/util.h (renamed from src/server_util.h) | 16 |
89 files changed, 7044 insertions, 6969 deletions
@@ -1,480 +1,65 @@ -CC ?= gcc -AR ?= ar -ARCH_FLAGS ?= -march=native +include build.mk -BUILD_DATE ?= $(shell date -u '+%Y-%m-%dT%H:%M:%SZ') -BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) -BUILD_OS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]') -BUILD_ARCH ?= $(shell uname -m) +# Top-level build: +# Root owns BACKEND/MODE and exports the resolved flags through build.mk. -# Version and build identity compiled into the embed and embed-server binaries. -# Applied on the cli.o and server.o recipes. VERSION is set for release builds; -# otherwise the C fallbacks apply. -VERSION_CFLAGS = \ - -DEMBED_BUILD_DATE=\"$(BUILD_DATE)\" \ - -DEMBED_BUILD_COMMIT=\"$(BUILD_COMMIT)\" \ - -DEMBED_BUILD_OS=\"$(BUILD_OS)\" \ - -DEMBED_BUILD_ARCH=\"$(BUILD_ARCH)\" -ifneq ($(strip $(VERSION)),) -VERSION_CFLAGS += -DEMBED_VERSION=\"$(VERSION)\" -endif - -# Allow FP reassociation and contraction so reduction loops vectorize, but do -# not assume finite math or approximate libm calls: full -ffast-math produced -# NaN embeddings on GCC/OpenBLAS Linux builds (-ffinite-math-only and -# -fapprox-func are the unsafe parts and stay off). -# -fvisibility=hidden pairs with EMBED_API in the public headers: a new -# public function without the annotation will be missing from the shared lib. -CFLAGS_BASE = -Wall -Wextra -O3 $(ARCH_FLAGS) -fPIC -fvisibility=hidden \ - -fno-math-errno -ffp-contract=fast -fno-trapping-math \ - -fno-signed-zeros -fassociative-math -freciprocal-math -CJSON_CFLAGS ?= $(shell sh -c 'pkg-config --cflags libcjson 2>/dev/null') -CJSON_LDFLAGS ?= $(shell sh -c 'pkg-config --libs libcjson 2>/dev/null') -ifeq ($(strip $(CJSON_LDFLAGS)),) - CJSON_LDFLAGS = -lcjson -endif - -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Darwin) -ifeq ($(strip $(CJSON_CFLAGS)),) - CJSON_CFLAGS = -I/opt/homebrew/include -I/usr/local/include -endif -ifeq ($(strip $(CJSON_LDFLAGS)),-lcjson) - CJSON_LDFLAGS = -L/opt/homebrew/lib -L/usr/local/lib -lcjson -endif -endif +LIBS := libffwd +TOOLS := tools/cli tools/server -# OpenBLAS lives in different places across distros (/usr/include/openblas on -# Debian/Ubuntu, plain /usr/include on Arch/Alpine, openblas-openmp variants). -# Prefer pkg-config and fall back to the Debian layout so existing setups keep -# working. Used by the Linux/CUDA backends and the test/bench BLAS flags below. -OPENBLAS_CFLAGS ?= $(shell sh -c 'pkg-config --cflags openblas 2>/dev/null') -OPENBLAS_LDFLAGS ?= $(shell sh -c 'pkg-config --libs openblas 2>/dev/null') -ifeq ($(strip $(OPENBLAS_CFLAGS)),) - OPENBLAS_CFLAGS = -I/usr/include/openblas -endif -ifeq ($(strip $(OPENBLAS_LDFLAGS)),) - OPENBLAS_LDFLAGS = -lopenblas -endif - -BACKEND ?= cpu -MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null) -MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null) -CUDA_HOME ?= /usr/local/cuda -NVCC ?= $(CUDA_HOME)/bin/nvcc - -ifeq ($(BACKEND),mlx) -BACKEND_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX -I$(MLXC_PREFIX)/include -BACKEND_LDFLAGS = -framework Accelerate -framework Metal -framework Foundation \ - -L$(MLXC_PREFIX)/lib -lmlxc \ - -L$(MLX_PREFIX)/lib -lmlx \ - -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib -EXTRA_OBJS = src/mlx.o -else ifeq ($(BACKEND),cuda) -BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA $(OPENBLAS_CFLAGS) -I$(CUDA_HOME)/include -BACKEND_LDFLAGS = -L$(CUDA_HOME)/lib64 $(OPENBLAS_LDFLAGS) -lcudart -lcublas -EXTRA_OBJS = src/cuda.o -else ifeq ($(UNAME_S),Darwin) -BACKEND_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK -BACKEND_LDFLAGS = -framework Accelerate -else -BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS $(OPENBLAS_CFLAGS) -BACKEND_LDFLAGS = $(OPENBLAS_LDFLAGS) -endif +.PHONY: all libs tools test clean debug profile profiling release blas cpu mlx cuda gpu \ + bench bench-model bench-tokenizer bench-server-utils $(LIBS) $(TOOLS) -# -Isrc so frontends, tests, and benches resolve the public/internal headers -# now that every source lives under src/. EXTRA_CFLAGS/EXTRA_LDFLAGS are the -# append points the debug and prof targets use to layer on sanitizers and -# debug info without dropping the architecture, visibility, or backend flags. -# -MMD -MP make the compiler emit a .d sidecar per object recording the headers -# it included (-MP adds phony targets so a deleted header is not a hard error); -# these are -include'd at the foot of the file so a header edit rebuilds only -# the objects that use it. They live here, not in CFLAGS_BASE, so the one-shot -# test/bench compile+link commands (which reuse CFLAGS_BASE) do not emit stray -# .d files next to their output binaries. -CFLAGS = $(CFLAGS_BASE) $(BACKEND_CFLAGS) -Isrc -MMD -MP $(EXTRA_CFLAGS) -LDFLAGS = -lm -lpthread $(BACKEND_LDFLAGS) $(EXTRA_LDFLAGS) +all: libs tools + @ln -sf $(BUILDREL)/ffwd-cli ./ffwd-cli + @ln -sf $(BUILDREL)/ffwd-server ./ffwd-server -# Source groups. CORE_SRCS is the engine the tests and benches reuse; keeping it -# in one variable stops the per-test source lists from drifting apart. -KERNEL_SRCS = src/kernels_gemm.c src/kernels_norm.c src/kernels_act.c src/kernels_attn.c \ - src/kernels_rope.c src/threadpool.c \ - src/kernels_generic.c src/kernels_neon.c src/kernels_avx.c -TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c src/tokenizer_sentencepiece.c -CORE_SRCS = src/model.c src/forward.c src/late.c src/vec.c src/alloc.c \ - src/config.c src/safetensors.c $(KERNEL_SRCS) +libs: $(LIBS) +tools: $(TOOLS) -# Server-only objects: the embed-server binary and the server test link these -# alongside server.o. Not in libembed.a - the CLI and library do not use them. -SERVER_SRCS = src/server_util.c src/sbuf.c src/base64.c src/server_json.c src/server_encode.c src/server_http.c src/server_models.c src/server_handlers.c src/server_schedule.c -SERVER_OBJS = $(SERVER_SRCS:.c=.o) +$(LIBS): + $(MAKE) -C $@ -SRCS = $(CORE_SRCS) $(TOKENIZER_SRCS) \ - deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c +tools/cli: | libffwd + $(MAKE) -C $@ -MLX_SRCS = src/mlx.c -CUDA_SRCS = src/cuda.cu +tools/server: | libffwd + $(MAKE) -C $@ -OBJS = $(SRCS:.c=.o) -MLX_OBJS = $(MLX_SRCS:.c=.o) -HEADERS = $(wildcard src/*.h) -TARGET = embed -SERVER_TARGET = embed-server -LIB = libembed.a - -ifeq ($(shell uname -s),Darwin) -SHARED_LIB = libembed.dylib -SHARED_FLAGS = -dynamiclib -else -SHARED_LIB = libembed.so -SHARED_FLAGS = -shared -endif +# The current unit suite is CPU-internal; GPU checks are explicit driver targets. +test: + $(MAKE) -C tests BACKEND=blas test + $(MAKE) -C tools/server BACKEND=blas test -.PHONY: all cpu mlx cuda gpu prof test test-bf16 test-safetensors coverage bench \ - bench-model bench-tokenizer bench-server-utils debug clean help force +clean: + $(RM) -r build + @for d in $(TOOLS) tests bench; do $(MAKE) -C $$d clean; done + $(RM) -f ./ffwd-cli ./ffwd-server ffwd.exports ffwd.map -all: help +blas: + $(MAKE) BACKEND=blas MODE=$(MODE) all -help: - @echo "Targets:" - @echo " make cpu CPU-only build" - @echo " make gpu Build with GPU backend support (mlx on macOS, cuda on Linux)" - @echo " make mlx Apple Silicon GPU via MLX" - @echo " make cuda NVIDIA GPU via CUDA" - @echo " make test Build and run the test suite (no model files needed)" - @echo " make coverage Test-suite line coverage report (clang/llvm-cov)" - @echo " make bench Kernel microbenchmarks; records bench/results/*.json" - @echo " make bench-model MODEL_DIR=/path/to/model" - @echo " End-to-end embedding benchmark" - @echo " make bench-server-utils" - @echo " Base64 and server string-buffer microbenchmarks" - @echo " make debug Debug build" - @echo " make prof Profiling build" - @echo " make clean Remove build artifacts" +cpu: blas -# ============================================================================= -# Backend builds: build both binaries and the shared library for one backend. -# ============================================================================= -# Switching BACKEND changes CFLAGS but not the source mtimes, so Make alone -# would relink stale objects on a backend switch. The .backend stamp below -# tracks the active backend so these targets keep incremental builds within a -# backend and still rebuild correctly across one. -cpu mlx cuda: - $(MAKE) BACKEND=$@ $(TARGET) $(SERVER_TARGET) $(SHARED_LIB) +mlx: + $(MAKE) BACKEND=mlx MODE=$(MODE) all -# Stamp recording the backend the current objects were built for. `force` makes -# the recipe run every build, but it rewrites the file (bumping its mtime) only -# when BACKEND actually changed - so the engine objects, which list it as a -# prerequisite, rebuild on a switch and stay untouched otherwise. -.backend: force - @cur=`cat $@ 2>/dev/null || true`; \ - [ "$$cur" = "$(BACKEND)" ] || { echo "$(BACKEND)" > $@; } -force: +cuda: + $(MAKE) BACKEND=cuda MODE=$(MODE) all -ifeq ($(UNAME_S),Darwin) +ifeq ($(OS),Darwin) gpu: mlx else gpu: cuda endif -# ============================================================================= -# Compile rules -# ============================================================================= -# Generic rule for the plain core/kernel/tokenizer/safetensors/mlx objects. -# Sibling headers in src/ resolve via the quote-include search; -Isrc (in -# CFLAGS) makes it explicit. Header prerequisites come from the -MMD-generated -# .d files (-include'd at the foot of the file), which track them per object. -# The .backend prerequisite is attached in bulk further down. -src/%.o: src/%.c - $(CC) $(CFLAGS) -c -o $@ $< - -# -fPIC so the same object links into both binaries and the shared library. -src/cuda.o: src/cuda.cu src/cuda.h src/internal.h src/embed.h - $(NVCC) -O3 -std=c++17 -arch=native -Xcompiler -fPIC,-fvisibility=hidden -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -Isrc $(OPENBLAS_CFLAGS) -I$(CUDA_HOME)/include -x cu -c -o $@ $< - -# server.c reaches deps/ae via "deps/ae/ae.h", so it needs -I. (repo root). -src/server.o: src/server.c - $(CC) $(CFLAGS) $(VERSION_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae -c -o $@ $< - -# Server concern objects: server_internal.h pulls deps/ae/ae.h and cJSON, so -# they need the same -I. and cJSON flags as server.o (the leaf util/sbuf/base64 -# objects do not include it but are unharmed by the extra search paths). -$(SERVER_OBJS): src/%.o: src/%.c - $(CC) $(CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae -c -o $@ $< - -src/cli.o: src/cli.c - $(CC) $(CFLAGS) $(VERSION_CFLAGS) -c -o $@ $< - -deps/ae/%.o: deps/ae/%.c - $(CC) $(CFLAGS) -Ideps/ae -c -o $@ $< - -# Every engine object is built for one backend; attach the .backend stamp to -# all of them at once (the recipes come from the pattern rules above) so a -# backend switch invalidates them. cli.o/server.o are listed too because they -# inline backend-specific paths through the shared headers. -$(OBJS) $(EXTRA_OBJS) $(SERVER_OBJS) src/cli.o src/server.o: .backend - -# ============================================================================= -# Shared library (built by every backend target from the same objects) -# ============================================================================= -$(SHARED_LIB): $(OBJS) $(EXTRA_OBJS) - $(CC) $(CFLAGS) $(SHARED_FLAGS) -o $@ $^ $(LDFLAGS) - -# ============================================================================= -# Static library and binary link -# ============================================================================= -$(LIB): $(OBJS) $(EXTRA_OBJS) - $(AR) rcs $@ $^ - -$(TARGET): $(LIB) src/cli.o - $(CC) $(CFLAGS) -o $@ src/cli.o $(LIB) $(LDFLAGS) - -$(SERVER_TARGET): $(LIB) src/server.o $(SERVER_OBJS) - $(CC) $(CFLAGS) -o $@ src/server.o $(SERVER_OBJS) $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) - -# ============================================================================= -# Tests (no model files required) -# ============================================================================= -ifeq ($(UNAME_S),Darwin) -TEST_BLAS_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK -TEST_BLAS_LDFLAGS = -framework Accelerate -else -TEST_BLAS_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS $(OPENBLAS_CFLAGS) -TEST_BLAS_LDFLAGS = $(OPENBLAS_LDFLAGS) -endif - -# Source lists shared by test and coverage targets so the two cannot drift. -TEST_CC_FLAGS = -Wall -Wextra -O2 -Isrc -TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS) -TEST_TOKENIZER_SRCS = tests/test_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS) -TEST_WORDPIECE_SRCS = tests/test_wordpiece.c src/tokenizer_wordpiece.c -TEST_SENTENCEPIECE_SRCS = tests/test_sentencepiece.c src/tokenizer_sentencepiece.c -TEST_SAFETENSORS_SRCS = tests/test_safetensors.c src/safetensors.c -TEST_BF16_SRCS = tests/test_bf16_model.c $(CORE_SRCS) -TEST_QWEN3_SRCS = tests/test_qwen3.c $(CORE_SRCS) -TEST_QWEN2_SRCS = tests/test_qwen2.c $(CORE_SRCS) -TEST_CLS_SRCS = tests/test_cls.c $(CORE_SRCS) -TEST_XLM_ROBERTA_SRCS = tests/test_xlm_roberta.c $(CORE_SRCS) -TEST_WORKSPACE_SRCS = tests/test_workspace.c $(CORE_SRCS) src/tokenizer_bpe.c -TEST_LATE_SRCS = tests/test_late.c $(CORE_SRCS) src/tokenizer_bpe.c -# test_server.c #includes ../src/server.c, so server is not a separate object -# here; it needs -I. for server.c's "deps/ae/ae.h" plus the ae sources. -# test_server.c #includes ../src/server.c, so server.c itself is not listed; -# the carved-out server objects (util/sbuf/base64) are separate TUs and must be. -TEST_SERVER_SRCS = tests/test_server.c $(CORE_SRCS) $(TOKENIZER_SRCS) \ - $(SERVER_SRCS) \ - deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c -# The CLI check builds the real CLI (CPU backend) plus a driver that runs it. -TEST_CLI_BIN_SRCS = src/cli.c $(CORE_SRCS) $(TOKENIZER_SRCS) +release: + $(MAKE) MODE=release BACKEND=$(BACKEND) all -test: - $(CC) $(TEST_CC_FLAGS) -o tests/test_kernels_generic \ - $(TEST_KERNELS_SRCS) -lm -lpthread - ./tests/test_kernels_generic - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_kernels_blas \ - $(TEST_KERNELS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - ./tests/test_kernels_blas - $(CC) $(TEST_CC_FLAGS) -o tests/test_tokenizer \ - $(TEST_TOKENIZER_SRCS) -lm -lpthread - ./tests/test_tokenizer - $(CC) $(TEST_CC_FLAGS) -o tests/test_wordpiece $(TEST_WORDPIECE_SRCS) - ./tests/test_wordpiece - $(CC) $(TEST_CC_FLAGS) -o tests/test_sentencepiece $(TEST_SENTENCEPIECE_SRCS) - ./tests/test_sentencepiece - $(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS) - ./tests/test_safetensors - $(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \ - $(TEST_BF16_SRCS) -lm -lpthread - ./tests/test_bf16_model - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \ - $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - ./tests/test_qwen3 - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \ - $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - ./tests/test_qwen2 - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \ - $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - ./tests/test_cls - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \ - $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - ./tests/test_xlm_roberta - $(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \ - $(TEST_WORKSPACE_SRCS) -lm -lpthread - ./tests/test_workspace - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_late \ - $(TEST_LATE_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - ./tests/test_late - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/cli_under_test \ - $(TEST_CLI_BIN_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - $(CC) $(TEST_CC_FLAGS) -o tests/test_cli tests/test_cli.c -lm - ./tests/test_cli ./tests/cli_under_test - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae \ - -o tests/test_server $(TEST_SERVER_SRCS) \ - -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CJSON_LDFLAGS) - ./tests/test_server - -test-bf16: - $(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \ - $(TEST_BF16_SRCS) -lm -lpthread - ./tests/test_bf16_model - -test-safetensors: - $(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors \ - $(TEST_SAFETENSORS_SRCS) -lm -lpthread - ./tests/test_safetensors +debug: + $(MAKE) MODE=debug BACKEND=$(BACKEND) all -# Test-suite line coverage (requires clang + llvm-cov/llvm-profdata. Writes a -# per-file text summary to stdout and an HTML report to coverage/html/index.html.) -COV_DIR = coverage -# Source-based coverage; -O0 so no line is folded away by the optimizer. -COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \ - -fcoverage-compilation-dir=. -COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \ - tests/test_tokenizer tests/test_sentencepiece tests/test_safetensors \ - tests/test_bf16_model tests/test_qwen3 tests/test_qwen2 tests/test_workspace \ - tests/test_cls tests/test_xlm_roberta tests/test_late \ - tests/cli_under_test tests/test_server -# Report on project sources only - not the harnesses, vendored deps, or -# system headers (cJSON lands under /opt or /usr otherwise). -COV_IGNORE = -ignore-filename-regex='deps/|tests/|/opt/|/usr/' - -ifeq ($(UNAME_S),Darwin) -LLVM_PROFDATA = xcrun llvm-profdata -LLVM_COV = xcrun llvm-cov -else -LLVM_PROFDATA = llvm-profdata -LLVM_COV = llvm-cov -endif - -coverage: - rm -rf $(COV_DIR) && mkdir -p $(COV_DIR) - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_kernels_generic \ - $(TEST_KERNELS_SRCS) -lm -lpthread - LLVM_PROFILE_FILE=$(COV_DIR)/kernels_generic.profraw ./tests/test_kernels_generic - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_kernels_blas \ - $(TEST_KERNELS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - LLVM_PROFILE_FILE=$(COV_DIR)/kernels_blas.profraw ./tests/test_kernels_blas - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_tokenizer \ - $(TEST_TOKENIZER_SRCS) -lm -lpthread - LLVM_PROFILE_FILE=$(COV_DIR)/tokenizer.profraw ./tests/test_tokenizer - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_sentencepiece \ - $(TEST_SENTENCEPIECE_SRCS) - LLVM_PROFILE_FILE=$(COV_DIR)/sentencepiece.profraw ./tests/test_sentencepiece - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS) - LLVM_PROFILE_FILE=$(COV_DIR)/safetensors.profraw ./tests/test_safetensors - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_bf16_model \ - $(TEST_BF16_SRCS) -lm -lpthread - LLVM_PROFILE_FILE=$(COV_DIR)/bf16_model.profraw ./tests/test_bf16_model - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \ - $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - LLVM_PROFILE_FILE=$(COV_DIR)/qwen3.profraw ./tests/test_qwen3 - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \ - $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - LLVM_PROFILE_FILE=$(COV_DIR)/qwen2.profraw ./tests/test_qwen2 - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \ - $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - LLVM_PROFILE_FILE=$(COV_DIR)/cls.profraw ./tests/test_cls - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \ - $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - LLVM_PROFILE_FILE=$(COV_DIR)/xlm_roberta.profraw ./tests/test_xlm_roberta - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \ - $(TEST_WORKSPACE_SRCS) -lm -lpthread - LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_late \ - $(TEST_LATE_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - LLVM_PROFILE_FILE=$(COV_DIR)/late.profraw ./tests/test_late - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/cli_under_test \ - $(TEST_CLI_BIN_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - $(CC) $(TEST_CC_FLAGS) -o tests/test_cli tests/test_cli.c -lm - LLVM_PROFILE_FILE=$(COV_DIR)/cli_%p.profraw ./tests/test_cli ./tests/cli_under_test - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae \ - -o tests/test_server $(TEST_SERVER_SRCS) \ - -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CJSON_LDFLAGS) - LLVM_PROFILE_FILE=$(COV_DIR)/server.profraw ./tests/test_server - $(LLVM_PROFDATA) merge -sparse $(COV_DIR)/*.profraw -o $(COV_DIR)/tests.profdata - $(LLVM_COV) report -instr-profile=$(COV_DIR)/tests.profdata $(COV_IGNORE) \ - $(addprefix -object ,$(COV_BINS)) - $(LLVM_COV) show \ - -format=html \ - -output-dir=$(COV_DIR)/html \ - -instr-profile=$(COV_DIR)/tests.profdata \ - -show-line-counts-or-regions \ - -show-branches=count \ - -show-expansions \ - -show-instantiations \ - $(COV_IGNORE) \ - $(addprefix -object ,$(COV_BINS)) - @echo "HTML report: $(COV_DIR)/html/index.html" - -bench-tokenizer: - $(CC) -Wall -Wextra -O3 -march=native -Isrc -o bench/bench_tokenizer \ - bench/bench_tokenizer.c $(TOKENIZER_SRCS) $(KERNEL_SRCS) -lm -lpthread - @echo "run: bench/bench_tokenizer <model_dir-or-vocab.json> [runs] TEXT..." - -# Regression microbenchmarks (see bench/bench.h). Each run records a JSON -# sample set under bench/results/, keyed by commit; compare two records with -# tools/benchstat.py to see if a change helped or hurt. -BENCH_STAMP = $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet -- ':!.gitignore' 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S) -BENCH_CC_FLAGS = -Wall -Wextra $(CFLAGS_BASE) $(TEST_BLAS_CFLAGS) -Isrc -# In-tree alongside the other bench binaries (cleaned by `make clean`) so the -# default works on any OS; the old /private/tmp default existed only on macOS. -BENCH_SERVER_UTILS_BIN ?= bench/bench_server_utils -BENCH_ARGS ?= - -bench: - $(CC) $(BENCH_CC_FLAGS) -o bench/bench_kernels \ - bench/bench_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) - @mkdir -p bench/results - ./bench/bench_kernels --json bench/results/kernels-$(BENCH_STAMP).json - @echo "compare: ./tools/benchstat.py bench/results/OLD.json bench/results/NEW.json" - -bench-model: - @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; } - $(CC) $(BENCH_CC_FLAGS) -o bench/bench_model \ - bench/bench_model.c $(CORE_SRCS) \ - -lm -lpthread $(TEST_BLAS_LDFLAGS) - @mkdir -p bench/results - ./bench/bench_model "$(MODEL_DIR)" --json bench/results/model-$(BENCH_STAMP).json - -bench-server-utils: - $(CC) $(BENCH_CC_FLAGS) -Ibench -o $(BENCH_SERVER_UTILS_BIN) \ - bench/bench_server_utils.c src/base64.c src/sbuf.c src/server_util.c \ - -lm -lpthread $(TEST_BLAS_LDFLAGS) - $(BENCH_SERVER_UTILS_BIN) $(BENCH_ARGS) - -# ============================================================================= -# Debug and profiling builds -# ============================================================================= -# Both layer extra flags onto the normal build through EXTRA_CFLAGS/EXTRA_LDFLAGS. -# Backend defaults to cpu; override with BACKEND=mlx or BACKEND=cuda. -SANITIZE ?= address,undefined -DEBUG_CFLAGS = -pedantic -fno-omit-frame-pointer -g -O0 -DDEBUG -fsanitize=$(SANITIZE) -DEBUG_LDFLAGS = -fsanitize=$(SANITIZE) - -debug: clean - $(MAKE) BACKEND=$(BACKEND) \ - EXTRA_CFLAGS="$(DEBUG_CFLAGS)" EXTRA_LDFLAGS="$(DEBUG_LDFLAGS)" \ - $(TARGET) $(SERVER_TARGET) $(SHARED_LIB) - -prof: clean - $(MAKE) BACKEND=$(BACKEND) \ - EXTRA_CFLAGS="-g -fno-omit-frame-pointer" \ - $(TARGET) $(SERVER_TARGET) $(SHARED_LIB) - - -# ============================================================================= -clean: - rm -f src/*.o src/*.d deps/ae/*.o deps/ae/*.d .backend \ - $(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \ - tests/test_kernels_generic tests/test_kernels_blas \ - tests/test_safetensors tests/test_bf16_model tests/test_server \ - tests/test_qwen3 tests/test_qwen2 tests/test_cls tests/test_xlm_roberta \ - tests/test_tokenizer tests/test_wordpiece tests/test_sentencepiece \ - tests/test_workspace tests/test_cli tests/test_late \ - tests/cli_under_test bench/bench_tokenizer bench/bench_kernels \ - bench/bench_model bench/bench_server_utils - rm -rf $(COV_DIR) +profile profiling: + $(MAKE) MODE=profile BACKEND=$(BACKEND) all -# Per-object header dependencies emitted by -MMD -MP. Missing on a clean tree; -# -include ignores them until the first build creates them. --include $(OBJS:.o=.d) $(EXTRA_OBJS:.o=.d) $(SERVER_OBJS:.o=.d) src/cli.d src/server.d +bench bench-model bench-tokenizer bench-server-utils: + $(MAKE) -C bench $@ diff --git a/build.mk b/build.mk new file mode 100644 index 0000000..62e7f45 --- /dev/null +++ b/build.mk @@ -0,0 +1,230 @@ +OS := $(shell uname -s) +ARCH := $(shell uname -m) + +CC ?= cc +AR ?= ar +RM ?= rm -f + +ARCH_FLAGS ?= -march=native +BACKEND ?= blas +MODE ?= release +prefix ?= /usr/local + +# Per-configuration build tree under the repo root. Each MODE/BACKEND pair gets +# its own objects and artifacts, so switching configs never reuses another's +# output and needs no clean in between. A consumer sets ROOT (path to the repo +# root) and COMPONENT (a unique name) before including this file; objects land in +# $(OBJDIR), final artifacts in $(OUTDIR). +ROOT ?= . +COMPONENT ?= misc +BUILDREL := build/$(MODE)/$(BACKEND) +OUTDIR := $(ROOT)/$(BUILDREL) +OBJDIR := $(OUTDIR)/obj/$(COMPONENT) + +CUDA_HOME ?= /usr/local/cuda +NVCC ?= $(CUDA_HOME)/bin/nvcc +CUDAFLAGS ?= -O3 -std=c++17 -arch=native -Xcompiler -fPIC,-fvisibility=hidden + +OPENBLAS_CFLAGS ?= $(shell sh -c 'pkg-config --cflags openblas 2>/dev/null') +OPENBLAS_LDFLAGS ?= $(shell sh -c 'pkg-config --libs openblas 2>/dev/null') +ifeq ($(strip $(OPENBLAS_CFLAGS)),) + OPENBLAS_CFLAGS = -I/usr/include/openblas +endif +ifeq ($(strip $(OPENBLAS_LDFLAGS)),) + OPENBLAS_LDFLAGS = -lopenblas +endif + +CJSON_CFLAGS ?= $(shell sh -c 'pkg-config --cflags libcjson 2>/dev/null') +CJSON_LDFLAGS ?= $(shell sh -c 'pkg-config --libs libcjson 2>/dev/null') +ifeq ($(OS),Darwin) +ifeq ($(strip $(CJSON_CFLAGS)),) + CJSON_CFLAGS = -I/opt/homebrew/include -I/usr/local/include +endif +ifeq ($(strip $(CJSON_LDFLAGS)),) + CJSON_LDFLAGS = -L/opt/homebrew/lib -L/usr/local/lib -lcjson +endif +endif +ifeq ($(strip $(CJSON_LDFLAGS)),) + CJSON_LDFLAGS = -lcjson +endif + +MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null) +MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null) + +# Allow FP reassociation and contraction so reduction loops vectorize, but do +# not assume finite math or approximate libm calls: full -ffast-math produced +# NaN embeddings on GCC/OpenBLAS Linux builds (-ffinite-math-only and +# -fapprox-func are the unsafe parts and stay off). +# -fvisibility=hidden pairs with FFWD_API in the public headers: a new +# public function without the annotation will be missing from the shared lib. +CFLAGS_BASE = -Wall -Wextra -O3 $(ARCH_FLAGS) -fPIC -fvisibility=hidden \ + -fno-math-errno -ffp-contract=fast -fno-trapping-math \ + -fno-signed-zeros -fassociative-math -freciprocal-math + +SANITIZE ?= address,undefined +ifeq ($(MODE),debug) + OPT ?= $(CFLAGS_BASE) -pedantic -fno-omit-frame-pointer -g -O0 -DDEBUG -fsanitize=$(SANITIZE) + LDFLAGS_OPT ?= -fsanitize=$(SANITIZE) +else ifeq ($(MODE),profile) + OPT ?= $(CFLAGS_BASE) -g -fno-omit-frame-pointer + LDFLAGS_OPT ?= +else ifeq ($(MODE),release) + OPT ?= $(CFLAGS_BASE) + LDFLAGS_OPT ?= +else + $(error MODE must be release, debug, or profile) +endif + +# The shared library exports exactly the public ABI listed in the export map +# (ffwd.exports on Mach-O, ffwd.map on ELF) and hides everything else, so +# the exported set is identical on every backend and no internal symbol leaks. +# This pairs with -fvisibility=hidden. Only the libffwd shared library applies +# EXPORT_FLAGS; the static lib and the binaries link all objects directly. +ifeq ($(OS),Darwin) + SHARED_EXT = dylib + SHARED_FLAGS = -dynamiclib + EXPORT_FILE = $(ROOT)/ffwd.exports + EXPORT_FLAGS = -Wl,-exported_symbols_list,$(ROOT)/ffwd.exports + HOST_BLAS_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK + HOST_BLAS_LDFLAGS = -framework Accelerate +else + SHARED_EXT = so + SHARED_FLAGS = -shared + EXPORT_FILE = $(ROOT)/ffwd.map + EXPORT_FLAGS = -Wl,--version-script,$(ROOT)/ffwd.map + HOST_BLAS_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS $(OPENBLAS_CFLAGS) + HOST_BLAS_LDFLAGS = $(OPENBLAS_LDFLAGS) +endif + +# Each backend defines only the flags its own compiled sources need. +# blas: the CPU build; the kernel files call cblas, so USE_BLAS is on. +# mlx: the Apple GPU build runs all math on the GPU and compiles no kernel +# files, so it defines neither USE_BLAS nor links Accelerate. USE_GPU selects +# MLX via platform.h. +# cuda: the NVIDIA GPU build still compiles the CPU kernel files for host-side +# work, so it keeps USE_BLAS (OpenBLAS) alongside the CUDA libraries. +BLAS_BACKEND_CFLAGS = $(HOST_BLAS_CFLAGS) +BLAS_BACKEND_LDFLAGS = $(HOST_BLAS_LDFLAGS) +MLX_BACKEND_CFLAGS = -DUSE_GPU -I$(MLXC_PREFIX)/include +MLX_BACKEND_LDFLAGS = -framework Metal -framework Foundation \ + -L$(MLXC_PREFIX)/lib -lmlxc \ + -L$(MLX_PREFIX)/lib -lmlx \ + -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib +CUDA_BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU $(OPENBLAS_CFLAGS) \ + -I$(CUDA_HOME)/include +CUDA_BACKEND_LDFLAGS = -L$(CUDA_HOME)/lib64 $(OPENBLAS_LDFLAGS) -lcudart -lcublas + +ifeq ($(BACKEND),blas) + BACKEND_CFLAGS = $(BLAS_BACKEND_CFLAGS) + BACKEND_LDFLAGS = $(BLAS_BACKEND_LDFLAGS) +else ifeq ($(BACKEND),mlx) + BACKEND_CFLAGS = $(MLX_BACKEND_CFLAGS) + BACKEND_LDFLAGS = $(MLX_BACKEND_LDFLAGS) +else ifeq ($(BACKEND),cuda) + BACKEND_CFLAGS = $(CUDA_BACKEND_CFLAGS) + BACKEND_LDFLAGS = $(CUDA_BACKEND_LDFLAGS) +else + $(error BACKEND must be blas, mlx, or cuda) +endif + +BUILD_DATE ?= $(shell date -u '+%Y-%m-%dT%H:%M:%SZ') +BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +BUILD_OS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]') +BUILD_ARCH ?= $(shell uname -m) +VERSION_CFLAGS = \ + -DFFWD_BUILD_DATE=\"$(BUILD_DATE)\" \ + -DFFWD_BUILD_COMMIT=\"$(BUILD_COMMIT)\" \ + -DFFWD_BUILD_OS=\"$(BUILD_OS)\" \ + -DFFWD_BUILD_ARCH=\"$(BUILD_ARCH)\" +ifneq ($(strip $(VERSION)),) +VERSION_CFLAGS += -DFFWD_VERSION=\"$(VERSION)\" +endif + +ifdef CFLAGS +else +CFLAGS = $(OPT) +endif +CPPFLAGS ?= +LDLIBS ?= +SHARED_LDLIBS ?= + +objs = $(patsubst %.c,$(OBJDIR)/%.o,$(filter %.c,$1)) $(patsubst %.cu,$(OBJDIR)/%.o,$(filter %.cu,$1)) +deps = $(patsubst %.o,%.d,$1) + +$(OBJDIR)/%.o: %.c + @mkdir -p $(@D) + $(CC) $(CFLAGS) $(CPPFLAGS) -MMD -MP -c -o $@ $< + +$(OBJDIR)/%.o: %.cu + @mkdir -p $(@D) + $(NVCC) $(CUDAFLAGS) $(CUDA_BACKEND_CFLAGS) $(CPPFLAGS) -x cu -c -o $@ $< + +define static_archive +$(1): $(2) $(3) + @mkdir -p $$(@D) + $$(RM) $$@ + $$(AR) rcs $$@ $(2) +endef + +define shared_library +$(1): $(2) $(3) + @mkdir -p $$(@D) + $$(CC) $$(SHARED_FLAGS) $$(SHARED_LDFLAGS) -o $$@ $(2) $(3) $(4) $$(LDFLAGS_OPT) +endef + +define link_program +$(1): $(2) $(3) + @mkdir -p $$(@D) + $$(CC) $$(CFLAGS) -o $$@ $(2) $(3) $(4) $$(LDFLAGS_OPT) +endef + +define program +OBJS := $$(call objs,$$(SRCS)) +DEPS := $$(call deps,$$(OBJS)) +.PHONY: all clean +all: $$(BIN) +$$(eval $$(call link_program,$$(BIN),$$(OBJS),$$(LIBS),$$(LDLIBS))) +clean: + $$(RM) -r $$(OBJS) $$(DEPS) $$(BIN) $$(BIN).dSYM +-include $$(DEPS) +endef + +define simple_library +STATIC := $$(OUTDIR)/lib$$(LIB).a +SHARED := $$(OUTDIR)/lib$$(LIB).$$(SHARED_EXT) +OBJS := $$(call objs,$$(SRCS)) +DEPS := $$(call deps,$$(OBJS)) +.PHONY: all clean install +all: $$(STATIC) $$(SHARED) +$$(eval $$(call static_archive,$$(STATIC),$$(OBJS),)) +$$(eval $$(call shared_library,$$(SHARED),$$(OBJS),,$$(SHARED_LDLIBS))) +install: all + install -d $$(prefix)/lib $$(prefix)/include + install -m 644 $$(STATIC) $$(prefix)/lib/ + install -m 755 $$(SHARED) $$(prefix)/lib/ + install -m 644 $$(HEADERS) $$(prefix)/include/ +clean: + $$(RM) $$(OBJS) $$(DEPS) $$(STATIC) $$(SHARED) +-include $$(DEPS) +endef + +define c_program +$(1): $(2) $(3) + @mkdir -p $$(@D) + $$(CC) $$(CFLAGS) $$(CPPFLAGS) $(4) -o $$@ $(2) $(3) $(5) $$(LDFLAGS_OPT) +endef + +define c_program_deps +$(1): $(2) $(3) + @mkdir -p $$(@D) + $$(CC) $$(CFLAGS) $$(CPPFLAGS) $(5) -o $$@ $(2) $(4) $(6) $$(LDFLAGS_OPT) +endef + +export BACKEND MODE ARCH_FLAGS CFLAGS_BASE OPT LDFLAGS_OPT CUDA_HOME NVCC CUDAFLAGS +export OPENBLAS_CFLAGS OPENBLAS_LDFLAGS CJSON_CFLAGS CJSON_LDFLAGS +export MLX_PREFIX MLXC_PREFIX +export HOST_BLAS_CFLAGS HOST_BLAS_LDFLAGS +export BLAS_BACKEND_CFLAGS BLAS_BACKEND_LDFLAGS +export MLX_BACKEND_CFLAGS MLX_BACKEND_LDFLAGS +export CUDA_BACKEND_CFLAGS CUDA_BACKEND_LDFLAGS +export BACKEND_CFLAGS BACKEND_LDFLAGS VERSION_CFLAGS diff --git a/libffwd/Makefile b/libffwd/Makefile new file mode 100644 index 0000000..0c831a6 --- /dev/null +++ b/libffwd/Makefile @@ -0,0 +1,61 @@ +ROOT := .. +COMPONENT := libffwd +include ../build.mk + +LIB := ffwd +STATIC := $(OUTDIR)/lib$(LIB).a +SHARED := $(OUTDIR)/lib$(LIB).$(SHARED_EXT) + +CFLAGS += -std=c11 -pthread $(BACKEND_CFLAGS) +CPPFLAGS += -I. +SHARED_FLAGS += -pthread +# Restrict the shared library's exported symbols to the public ABI. +SHARED_LDFLAGS += $(EXPORT_FLAGS) + +# Tokenizer, config, safetensors reader, and L2/cosine helpers: every backend. +COMMON_SRCS := config.c vec.c tokenize.c bpe.c sentencepiece.c wordpiece.c safetensors.c + +# CPU math kernels and SIMD dispatch. generic.c is always built; avx.c and +# neon.c each guard their entire body by arch (__AVX2__ / __ARM_NEON), so the +# off-target file compiles to an empty translation unit. +KERNEL_SRCS := act.c gemm.c norm.c rope.c attn.c threadpool.c generic.c avx.c neon.c + +BLAS_SRCS := backend_blas.c model.c model_blas.c forward.c late.c late_blas.c alloc.c \ + $(COMMON_SRCS) $(KERNEL_SRCS) +MLX_SRCS := backend_mlx.c mlx.c $(COMMON_SRCS) +CUDA_SRCS := backend_cuda.c model.c late.c alloc.c $(COMMON_SRCS) $(KERNEL_SRCS) + +ifeq ($(BACKEND),mlx) + SRCS := $(MLX_SRCS) + CU_SRCS := +else ifeq ($(BACKEND),cuda) + SRCS := $(CUDA_SRCS) + CU_SRCS := cuda.cu +else + SRCS := $(BLAS_SRCS) + CU_SRCS := +endif + +OBJS := $(call objs,$(SRCS) $(CU_SRCS)) +DEPS := $(call deps,$(OBJS)) + +.PHONY: all clean + +all: $(STATIC) $(SHARED) + +$(eval $(call static_archive,$(STATIC),$(OBJS))) +$(eval $(call shared_library,$(SHARED),$(OBJS),,$(BACKEND_LDFLAGS) -lm)) +# The export map is generated from the public ABI (the FFWD_API declarations in +# ffwd.h) instead of being committed, so it cannot drift from the header. It is +# regenerated when the header or the generator changes, and the shared library +# relinks when it does. +$(EXPORT_FILE): ffwd.h $(ROOT)/devtools/gen_exports.py + python3 $(ROOT)/devtools/gen_exports.py --outdir $(ROOT) +$(SHARED): $(EXPORT_FILE) + +# Removes only this MODE/BACKEND's objects and artifacts; the root `make clean` +# removes the whole build/ tree. +clean: + $(RM) -r $(OBJDIR) $(STATIC) $(SHARED) $(SHARED).dSYM + +-include $(DEPS) diff --git a/src/kernels_act.c b/libffwd/act.c index abbade4..688dd71 100644 --- a/src/kernels_act.c +++ b/libffwd/act.c @@ -1,28 +1,32 @@ -#include "kernels.h" -#include "kernels_impl.h" +/* + * act.c - activations + */ #include <math.h> #include <stdlib.h> #include <string.h> +#include "impl.h" +#include "kernels.h" + #if (defined(__AVX512F__) || defined(__AVX2__)) && \ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) -#include <immintrin.h> +# include <immintrin.h> #endif #if defined(__ARM_NEON) || defined(__ARM_NEON__) -#include <arm_neon.h> +# include <arm_neon.h> #endif #ifdef USE_BLAS -#ifdef __APPLE__ -#include <Accelerate/Accelerate.h> -#else -#include <cblas.h> -#endif +# ifdef __APPLE__ +# include <Accelerate/Accelerate.h> +# else +# include <cblas.h> +# endif #endif #ifndef M_PI -#define M_PI 3.14159265358979323846 +# define M_PI 3.14159265358979323846 #endif /* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU @@ -32,7 +36,7 @@ * Activation Functions * ======================================================================== */ -void embed_silu_mul_inplace(float *gate, const float *up, int n) { +void silu_mul_inplace(float *gate, const float *up, int n) { #if defined(__APPLE__) && defined(USE_BLAS) enum { CHUNK = 4096 }; float exp_neg[CHUNK]; @@ -59,13 +63,13 @@ void embed_silu_mul_inplace(float *gate, const float *up, int n) { #endif } -void embed_gelu_inplace(float *x, int n) { +void gelu_inplace(float *x, int n) { /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ for (int i = 0; i < n; i++) x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); } -void embed_gelu_tanh_inplace(float *x, int n) { +void gelu_tanh_inplace(float *x, int n) { /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */ for (int i = 0; i < n; i++) { float v = x[i]; @@ -74,7 +78,7 @@ void embed_gelu_tanh_inplace(float *x, int n) { } } -void embed_softmax(float *x, int rows, int cols) { +void softmax(float *x, int rows, int cols) { for (int r = 0; r < rows; r++) { float *row = x + r * cols; float max_val = row[0]; diff --git a/src/alloc.c b/libffwd/alloc.c index 41a59a4..6af854a 100644 --- a/src/alloc.c +++ b/libffwd/alloc.c @@ -8,8 +8,8 @@ * growth, never allocate fewer than the minimum, and round up to the * granularity so small length changes do not trigger a realloc on every call. * Existing larger capacity is preserved. */ -#define EMBED_MIN_WORKSPACE_SEQ_CAP 16 -#define EMBED_WORKSPACE_SEQ_GRANULARITY 16 +#define FFWD_MIN_WORKSPACE_SEQ_CAP 16 +#define FFWD_WORKSPACE_SEQ_GRANULARITY 16 int mul_size(size_t a, size_t b, size_t *out) { if (a != 0 && b > SIZE_MAX / a) @@ -30,12 +30,12 @@ int grow_cap(int current, int needed, int *out) { return -1; int cap = needed; - if (cap < EMBED_MIN_WORKSPACE_SEQ_CAP) - cap = EMBED_MIN_WORKSPACE_SEQ_CAP; + if (cap < FFWD_MIN_WORKSPACE_SEQ_CAP) + cap = FFWD_MIN_WORKSPACE_SEQ_CAP; - int rem = cap % EMBED_WORKSPACE_SEQ_GRANULARITY; + int rem = cap % FFWD_WORKSPACE_SEQ_GRANULARITY; if (rem != 0) { - int pad = EMBED_WORKSPACE_SEQ_GRANULARITY - rem; + int pad = FFWD_WORKSPACE_SEQ_GRANULARITY - rem; if (cap > INT_MAX - pad) return -1; cap += pad; diff --git a/src/alloc.h b/libffwd/alloc.h index bc40e8b..0712e21 100644 --- a/src/alloc.h +++ b/libffwd/alloc.h @@ -1,5 +1,5 @@ -#ifndef EMBED_ALLOC_H -#define EMBED_ALLOC_H +#ifndef FFWD_ALLOC_H +#define FFWD_ALLOC_H #include <stddef.h> @@ -31,4 +31,4 @@ int realloc_floats_2d(float **ptr, int rows, int cols); /* realloc *ptr to hold `count` ints; *ptr unchanged on failure. */ int realloc_ints(int **ptr, size_t count); -#endif /* EMBED_ALLOC_H */ +#endif /* FFWD_ALLOC_H */ diff --git a/libffwd/attn.c b/libffwd/attn.c new file mode 100644 index 0000000..6134064 --- /dev/null +++ b/libffwd/attn.c @@ -0,0 +1,464 @@ +/* + * attn.c - attention + */ + +#include "kernels.h" +#include "impl.h" +#include "threadpool.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#ifdef USE_BLAS +# ifdef __APPLE__ +# include <Accelerate/Accelerate.h> +# else +# include <cblas.h> +# endif +#endif + +#ifndef M_PI +# define M_PI 3.14159265358979323846 +#endif + +/* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE) + * GQA, each with an online-softmax and a BLAS-GEMM path. */ + +/* ======================================================================== + * Attention Operations + * ======================================================================== */ + +float dot_f32(const float *a, const float *b, int n) { return dot_f32_impl(a, b, n); } + +static inline float dot_f32_fast(const float *a, const float *b, int n) { + return dot_f32_impl(a, b, n); +} + +/* dst = dst * scale */ +static inline void vec_scale_inplace(float *dst, float scale, int n) { + vec_scale_inplace_impl(dst, scale, n); +} + +/* dst += alpha * src */ +static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { + vec_axpy_inplace_impl(dst, src, alpha, n); +} + +/* dst = dst * correction + src */ +static inline void vec_scale_add(float *dst, const float *src, float correction, int n) { + vec_scale_add_impl(dst, src, correction, n); +} + +#define PACKED_ATTN_QUERY_TILE 32 +#define PACKED_ATTN_BLAS_MIN_SEQ 128 + +static void bidirectional_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; + + for (int j = start; j < end; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + + float score = dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) { + float inv_sum = 1.0f / sum_exp; + vec_scale_inplace(o_row, inv_sum, head_dim); + } + } +} + +#ifdef USE_BLAS +static void bidirectional_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int end, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, + kv_hidden, 0.0f, scores, keys); + softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); +} +#endif + +typedef struct { + float *out; + const float *Q; + const float *K; + const float *V; + const int *offsets; + int batch; + int n_heads, n_kv_heads; + int head_dim; + float scale; + float *scores; + int max_seq; +} packed_gqa_attn_task_t; + +static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + PACKED_ATTN_QUERY_TILE; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; +#ifdef USE_BLAS + if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { + bidirectional_gqa_attention_packed_blas_rows( + t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1, scores); + continue; + } +#endif + bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end, + t->n_heads, t->n_kv_heads, t->head_dim, + t->scale, h, q0, q1); + } + } + } +} + +size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { +#ifdef USE_BLAS + if (!offsets || batch <= 0) + return 0; + + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + if (len > max_seq) + max_seq = len; + } + if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ) + return 0; + if ((size_t)max_seq > + SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads())) + return 0; + return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); +#else + (void)offsets; + (void)batch; + return 0; +#endif +} + +void bidirectional_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * len * n_heads; + if (len > max_seq) + max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; + if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { + parallel_for(packed_gqa_attn_worker, &task); + } else { + packed_gqa_attn_worker(0, 1, &task); + } +} + +void bidirectional_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale) { + size_t scratch_bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + bidirectional_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, + head_dim, scale, scores, scratch_bytes); + free(scores); +} + +static void causal_gqa_attention_packed_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + + for (int i = query_start; i < query_end; i++) { + const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; + float *o_row = out + (size_t)i * q_hidden + h * head_dim; + + float max_score = -1e30f; + float sum_exp = 0.0f; + for (int d = 0; d < head_dim; d++) + o_row[d] = 0.0f; + + for (int j = start; j <= i; j++) { + const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; + const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + float score = dot_f32_fast(q_row, k_row, head_dim) * scale; + + if (score > max_score) { + float correction = expf(max_score - score); + sum_exp = sum_exp * correction + 1.0f; + vec_scale_add(o_row, v_row, correction, head_dim); + max_score = score; + } else { + float wt = expf(score - max_score); + sum_exp += wt; + vec_axpy_inplace(o_row, v_row, wt, head_dim); + } + } + + if (sum_exp > 0.0f) + vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); + } +} + +#ifdef USE_BLAS +static void causal_gqa_attention_packed_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int start, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int heads_per_kv = n_heads / n_kv_heads; + int q_hidden = n_heads * head_dim; + int kv_hidden = n_kv_heads * head_dim; + int kv_h = h / heads_per_kv; + int rows = query_end - query_start; + int keys = query_end - start; + + const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; + const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; + const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; + float *o = out + (size_t)query_start * q_hidden + h * head_dim; + + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, + kv_hidden, 0.0f, scores, keys); + for (int r = 0; r < rows; r++) { + int allowed = query_start + r - start + 1; + float *row = scores + (size_t)r * keys; + for (int j = allowed; j < keys; j++) + row[j] = -INFINITY; + } + softmax(scores, rows, keys); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, + v, kv_hidden, 0.0f, o, q_hidden); +} +#endif + +static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { + packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; + int task_id = 0; +#ifdef USE_BLAS + float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; +#endif + + for (int h = 0; h < t->n_heads; h++) { + for (int b = 0; b < t->batch; b++) { + int start = t->offsets[b]; + int end = t->offsets[b + 1]; + for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + PACKED_ATTN_QUERY_TILE; + if (q1 > end) + q1 = end; + if (task_id++ % n_threads != tid) + continue; +#ifdef USE_BLAS + if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { + causal_gqa_attention_packed_blas_rows(t->out, t->Q, t->K, t->V, start, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, + q1, scores); + continue; + } +#endif + causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, t->n_heads, + t->n_kv_heads, t->head_dim, t->scale, h, q0, + q1); + } + } + } +} + +void causal_gqa_attention_packed_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = 0; + int max_seq = 0; + for (int b = 0; b < batch; b++) { + int len = offsets[b + 1] - offsets[b]; + qk_work += (long long)len * (len + 1) / 2 * n_heads; + if (len > max_seq) + max_seq = len; + } + + float *scores = NULL; +#ifdef USE_BLAS + size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (required != 0 && scratch && scratch_bytes >= required) + scores = scratch; +#else + (void)scratch; + (void)scratch_bytes; +#endif + packed_gqa_attn_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .offsets = offsets, + .batch = batch, + .n_heads = n_heads, + .n_kv_heads = n_kv_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores, + .max_seq = max_seq}; + if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) + parallel_for(packed_causal_gqa_attn_worker, &task); + else + packed_causal_gqa_attn_worker(0, 1, &task); +} + +void causal_gqa_attention_packed(float *out, + const float *Q, + const float *K, + const float *V, + const int *offsets, + int batch, + int n_heads, + int n_kv_heads, + int head_dim, + float scale) { + size_t scratch_bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; + causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, + head_dim, scale, scores, scratch_bytes); + free(scores); +} diff --git a/src/kernels_avx.c b/libffwd/avx.c index 920fecf..41259a1 100644 --- a/src/kernels_avx.c +++ b/libffwd/avx.c @@ -1,5 +1,5 @@ /* - * embed_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) + * avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) * * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. * Uses AVX-512F+BW for 16-wide bf16 matvec (dominant for seq_len==1), and @@ -9,20 +9,20 @@ * overhead and improve out-of-order execution on memory-bound workloads. */ -#include "kernels_impl.h" +#include "impl.h" #if defined(__AVX2__) && defined(__FMA__) -#include <immintrin.h> -#include <string.h> +# include <immintrin.h> +# include <string.h> /* ===================================================================== * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise * ===================================================================== */ -#if defined(__AVX512F__) && defined(__AVX512BW__) +# if defined(__AVX512F__) && defined(__AVX512BW__) -void embed_bf16_matvec_fused_avx( +void bf16_matvec_fused_avx( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; @@ -188,7 +188,7 @@ void embed_bf16_matvec_fused_avx( } } -#else /* AVX2 only (or AVX-512F without BW) */ +# else /* AVX2 only (or AVX-512F without BW) */ /* Helper: bf16 to f32 for 8 elements */ static inline __m256 bf16x8_to_f32(__m128i raw) { @@ -201,7 +201,7 @@ static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) { *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1)); } -void embed_bf16_matvec_fused_avx( +void bf16_matvec_fused_avx( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; @@ -266,174 +266,6 @@ void embed_bf16_matvec_fused_avx( } } -#endif /* AVX-512F+BW vs AVX2 for bf16 */ - -/* ===================================================================== - * f32 attention helpers - AVX2+FMA, with AVX-512F when available - * (operates on L1-resident head vectors) - * ===================================================================== */ - -float embed_dot_f32_avx(const float *a, const float *b, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 acc0 = _mm512_setzero_ps(); - __m512 acc1 = _mm512_setzero_ps(); - __m512 acc2 = _mm512_setzero_ps(); - __m512 acc3 = _mm512_setzero_ps(); - for (; i + 64 <= n; i += 64) { - acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); - acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); - acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); - acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); - } - __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); - for (; i + 16 <= n; i += 16) { - acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); - } - float sum = _mm512_reduce_add_ps(acc); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -#else - int i = 0; - __m256 acc0 = _mm256_setzero_ps(); - __m256 acc1 = _mm256_setzero_ps(); - __m256 acc2 = _mm256_setzero_ps(); - __m256 acc3 = _mm256_setzero_ps(); - for (; i + 32 <= n; i += 32) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); - acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1); - acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2); - acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3); - } - acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); - for (; i + 8 <= n; i += 8) { - acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); - } - __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); - r = _mm_hadd_ps(r, r); - r = _mm_hadd_ps(r, r); - float sum = _mm_cvtss_f32(r); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -#endif -} - -void embed_vec_scale_inplace_avx(float *dst, float scale, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 s = _mm512_set1_ps(scale); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); - _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); - _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); - } - for (; i < n; i++) - dst[i] *= scale; -#else - int i = 0; - __m256 s = _mm256_set1_ps(scale); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); - _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s)); - _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s)); - _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s)); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); - } - for (; i < n; i++) - dst[i] *= scale; -#endif -} - -void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 a = _mm512_set1_ps(alpha); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, - _mm512_loadu_ps(dst + i + 16))); - _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, - _mm512_loadu_ps(dst + i + 32))); - _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, - _mm512_loadu_ps(dst + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -#else - int i = 0; - __m256 a = _mm256_set1_ps(alpha); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); - _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a, - _mm256_loadu_ps(dst + i + 8))); - _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a, - _mm256_loadu_ps(dst + i + 16))); - _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a, - _mm256_loadu_ps(dst + i + 24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -#endif -} - -void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n) { -#if defined(__AVX512F__) - int i = 0; - __m512 c = _mm512_set1_ps(correction); - for (; i + 64 <= n; i += 64) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, - _mm512_loadu_ps(src + i + 16))); - _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, - _mm512_loadu_ps(src + i + 32))); - _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, - _mm512_loadu_ps(src + i + 48))); - } - for (; i + 16 <= n; i += 16) { - _mm512_storeu_ps(dst + i, - _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -#else - int i = 0; - __m256 c = _mm256_set1_ps(correction); - for (; i + 32 <= n; i += 32) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); - _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c, - _mm256_loadu_ps(src + i + 8))); - _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c, - _mm256_loadu_ps(src + i + 16))); - _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c, - _mm256_loadu_ps(src + i + 24))); - } - for (; i + 8 <= n; i += 8) { - _mm256_storeu_ps(dst + i, - _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -#endif -} +# endif /* AVX-512F+BW vs AVX2 for bf16 */ #endif /* __AVX2__ && __FMA__ */ diff --git a/libffwd/backend_blas.c b/libffwd/backend_blas.c new file mode 100644 index 0000000..edf54bb --- /dev/null +++ b/libffwd/backend_blas.c @@ -0,0 +1,168 @@ +/* backend_blas.c - CPU backend (BLAS), implements the engine API in ffwd.h. + * Linked only into CPU builds. The CPU model loads on the caller's thread, so + * ffwd_open does the full load and activate/worker_free are no-ops. */ +#include "internal.h" +#include "threadpool.h" +#include "platform.h" + +#include <stdio.h> +#include <stdlib.h> + +struct embed { + int is_late; + ffwd_model_t *model; + ffwd_workspace_t *ws; + ffwd_late_model_t *late; + ffwd_late_workspace_t *late_ws; +}; + +const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; } + +void ffwd_cli_help(FILE *f) { + fputs(" -t, --threads N\n" + " CPU threads (default: available cores, cgroup-aware)\n", + f); +} + +void ffwd_server_help(FILE *f) { + fputs(" -t, --threads N CPU threads (default: available cores, cgroup-aware)\n", f); +} + +int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { + (void)err; + (void)errlen; + int n = opts->n_threads > 0 ? opts->n_threads : tp_get_num_cpus(); + tp_set_threads(n); + if (ffwd_verbose >= 1) + fprintf(stderr, "Using %d CPU thread(s)\n", n); + return 0; +} + +ffwd_t *ffwd_open( + const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { + (void)opts; + ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b)); + if (!b) { + snprintf(err, errlen, "out of memory"); + return NULL; + } + b->is_late = is_late; + if (is_late) { + b->late = ffwd_late_model_load(model_dir); + if (!b->late) { + snprintf(err, errlen, "failed to load late model"); + free(b); + return NULL; + } + b->late_ws = ffwd_late_workspace_new(b->late); + if (!b->late_ws) { + snprintf(err, errlen, "failed to allocate late workspace"); + ffwd_late_model_free(b->late); + free(b); + return NULL; + } + } else { + b->model = ffwd_model_load(model_dir); + if (!b->model) { + snprintf(err, errlen, "failed to load model"); + free(b); + return NULL; + } + b->ws = ffwd_workspace_new(b->model); + if (!b->ws) { + snprintf(err, errlen, "failed to allocate workspace"); + ffwd_model_free(b->model); + free(b); + return NULL; + } + } + return b; +} + +int ffwd_activate(ffwd_t *b, char *err, size_t errlen) { + (void)b; + (void)err; + (void)errlen; + return 0; +} + +void ffwd_worker_free(ffwd_t *b) { (void)b; } + +void ffwd_free(ffwd_t *b) { + if (!b) + return; + ffwd_workspace_free(b->ws); + ffwd_model_free(b->model); + ffwd_late_workspace_free(b->late_ws); + ffwd_late_model_free(b->late); + free(b); +} + +const ffwd_config_t *ffwd_config(const ffwd_t *b) { + return b->is_late ? ffwd_late_model_config(b->late) : ffwd_model_config(b->model); +} + +int ffwd_token_dim(const ffwd_t *b) { + return b->is_late ? ffwd_late_model_token_dim(b->late) : 0; +} + +int ffwd_uses_dense_batches(const ffwd_t *b) { + (void)b; + return 0; +} + +int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) { + return ffwd_model_encode_batch(b->model, b->ws, inputs, batch, out); +} + +int ffwd_encode_spans_batch(ffwd_t *b, + const ffwd_context_input_t *inputs, + int batch, + float *out) { + return ffwd_model_encode_spans_batch(b->model, b->ws, inputs, batch, out); +} + +int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) { + int dim = ffwd_late_model_token_dim(b->late); + int total_doc_vecs = 0; + for (int i = 0; i < in->n_docs; i++) + total_doc_vecs += in->doc_n_keep[i]; + + float *query = (float *)malloc((size_t)in->query_n_tokens * dim * sizeof(*query)); + float *docs = (float *)malloc((size_t)total_doc_vecs * dim * sizeof(*docs)); + int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets)); + int rc = (query && docs && offsets) ? 0 : -1; + + /* Encode every candidate in one block-diagonal forward: the encoder packs + * each document's kept token vectors back-to-back and fills offsets. */ + if (rc == 0) + rc = ffwd_late_model_encode_tokens(b->late, b->late_ws, in->query_ids, in->query_n_tokens, + 1, query); + if (rc == 0) + rc = ffwd_late_model_encode_docs(b->late, b->late_ws, in->doc_ids, in->doc_n_tokens, + in->doc_keep, in->doc_n_keep, in->n_docs, 1, docs, + offsets); + if (rc == 0) + rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, + scores); + free(offsets); + free(docs); + free(query); + return rc; +} + +size_t ffwd_scratch_nbytes(const ffwd_t *b) { + return (b && b->ws) ? ffwd_workspace_nbytes(b->ws) : 0; +} + +int ffwd_default_batch_wait_us(void) { return 0; } + +int ffwd_preflight( + const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) { + (void)paths; + (void)n_paths; + (void)opts; + (void)err; + (void)errlen; + return 0; +} diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c new file mode 100644 index 0000000..db39345 --- /dev/null +++ b/libffwd/backend_cuda.c @@ -0,0 +1,172 @@ +/* backend_cuda.c - NVIDIA GPU backend (CUDA), implements the engine API in ffwd.h. + * Linked only into CUDA builds. The context is owned by the inference thread: + * open records what to load, activate creates the context, worker_free destroys + * it. CUDA encodes on the device and scores MaxSim on the host. */ +#include "internal.h" +#include "cuda.h" +#include "platform.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +struct embed { + int is_late; + char *dir; + ffwd_cuda_ctx_t *ctx; + ffwd_cuda_late_ctx_t *late_ctx; +}; + +const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; } + +void ffwd_cli_help(FILE *f) { + fputs(" --gpu-gemm-mode MODE\n" + " GPU GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" + " --gpu-weight-dtype DTYPE\n" + " GPU weight storage: f32 or bf16 (default: f32). bf16 halves\n" + " weight memory and uses BF16 tensor cores\n", + f); +} + +void ffwd_server_help(FILE *f) { + fputs(" --gpu-gemm-mode MODE GPU GEMM compute: f32, tf32, bf16, or 16f\n" + " (default: f32, exact)\n" + " --gpu-weight-dtype DTYPE GPU weight storage: f32 or bf16 (default:\n" + " f32); bf16 halves weight memory\n", + f); +} + +int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { + if (opts->gpu_gemm_mode && ffwd_cuda_set_fast_gemm(opts->gpu_gemm_mode) != 0) { + snprintf(err, errlen, "--gpu-gemm-mode must be f32, tf32, bf16, or 16f"); + return -1; + } + if (opts->gpu_weight_dtype) { + if (!strcmp(opts->gpu_weight_dtype, "bf16")) + ffwd_cuda_set_weights_bf16(1); + else if (!strcmp(opts->gpu_weight_dtype, "f32")) + ffwd_cuda_set_weights_bf16(0); + else { + snprintf(err, errlen, "--gpu-weight-dtype must be f32 or bf16"); + return -1; + } + } + if (ffwd_verbose >= 1) + fprintf(stderr, "Using %s\n", FFWD_BACKEND_LABEL); + return 0; +} + +ffwd_t *ffwd_open( + const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { + (void)opts; + ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b)); + if (!b || !(b->dir = strdup(model_dir))) { + snprintf(err, errlen, "out of memory"); + free(b); + return NULL; + } + b->is_late = is_late; + return b; +} + +int ffwd_activate(ffwd_t *b, char *err, size_t errlen) { + if (b->is_late) + b->late_ctx = ffwd_cuda_late_load(b->dir); + else + b->ctx = ffwd_cuda_load(b->dir); + if (!b->ctx && !b->late_ctx) { + snprintf(err, errlen, "failed to load model"); + return -1; + } + return 0; +} + +void ffwd_worker_free(ffwd_t *b) { + if (!b) + return; + if (b->ctx) { + ffwd_cuda_free(b->ctx); + b->ctx = NULL; + } + if (b->late_ctx) { + ffwd_cuda_late_free(b->late_ctx); + b->late_ctx = NULL; + } +} + +void ffwd_free(ffwd_t *b) { + if (!b) + return; + ffwd_worker_free(b); + free(b->dir); + free(b); +} + +const ffwd_config_t *ffwd_config(const ffwd_t *b) { + return b->is_late ? ffwd_cuda_late_config(b->late_ctx) : ffwd_cuda_config(b->ctx); +} + +int ffwd_token_dim(const ffwd_t *b) { + return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0; +} + +int ffwd_uses_dense_batches(const ffwd_t *b) { + (void)b; + return 0; +} + +int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) { + return ffwd_cuda_encode_batch(b->ctx, inputs, batch, out); +} + +int ffwd_encode_spans_batch(ffwd_t *b, + const ffwd_context_input_t *inputs, + int batch, + float *out) { + return ffwd_cuda_encode_spans_batch(b->ctx, inputs, batch, out); +} + +int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) { + int dim = ffwd_cuda_late_token_dim(b->late_ctx); + int total_doc_vecs = 0; + for (int i = 0; i < in->n_docs; i++) + total_doc_vecs += in->doc_n_keep[i]; + + float *query = (float *)malloc((size_t)in->query_n_tokens * dim * sizeof(*query)); + float *docs = (float *)malloc((size_t)total_doc_vecs * dim * sizeof(*docs)); + int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets)); + int rc = (query && docs && offsets) ? 0 : -1; + + /* GPU encodes the query and every candidate in one packed forward; MaxSim + * runs on the host, where the grouped-GEMM scorer beats a device graph. */ + if (rc == 0) + rc = + ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query); + if (rc == 0) + rc = ffwd_cuda_late_encode_docs(b->late_ctx, in->doc_ids, in->doc_n_tokens, in->doc_keep, + in->doc_n_keep, in->n_docs, 1, docs, offsets); + if (rc == 0) + rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, + scores); + free(offsets); + free(docs); + free(query); + return rc; +} + +size_t ffwd_scratch_nbytes(const ffwd_t *b) { + (void)b; + return 0; +} + +int ffwd_default_batch_wait_us(void) { return 1000; } + +int ffwd_preflight( + const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) { + (void)paths; + (void)n_paths; + (void)opts; + (void)err; + (void)errlen; + return 0; +} diff --git a/libffwd/backend_mlx.c b/libffwd/backend_mlx.c new file mode 100644 index 0000000..80b893c --- /dev/null +++ b/libffwd/backend_mlx.c @@ -0,0 +1,228 @@ +/* backend_mlx.c - Apple GPU backend (MLX), implements the engine API in ffwd.h. + * Linked only into Apple Silicon GPU builds. MLX streams are thread-local, so + * the context is created and destroyed on the inference thread: open only + * records what to load, activate creates the context, worker_free destroys it. */ +#include "internal.h" +#include "mlx.h" +#include "platform.h" +#include "safetensors.h" + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#ifdef __APPLE__ +# include <sys/sysctl.h> +#endif + +/* Peak resident memory is estimated as the on-disk tensor payload times this + * factor; the preflight refuses a model set whose estimate exceeds the budget. */ +#define FFWD_MLX_RESIDENT_MULTIPLIER 2 +#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90 + +struct embed { + int is_late; + char *dir; + ffwd_options_t opts; + ffwd_mlx_ctx_t *ctx; + ffwd_mlx_late_ctx_t *late_ctx; +}; + +const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; } + +void ffwd_cli_help(FILE *f) { + fputs(" --gpu-quant-bits N\n" + " Quantize GPU linear weights to 8 bits at load time\n" + " --gpu-quant-group-size N\n" + " GPU quantization group size (default: 64)\n", + f); +} + +void ffwd_server_help(FILE *f) { + fputs(" --memory-utilization F Fraction of physical memory the GPU model-set\n" + " preflight may plan for (default: 0.90; values\n" + " above 1.0 overcommit)\n" + " --gpu-quant-bits N Quantize GPU linear weights to 8 bits at load\n" + " --gpu-quant-group-size N GPU quantization group size (default: 64)\n", + f); +} + +int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { + if (opts->gpu_quant_bits != 0 && opts->gpu_quant_bits != 8) { + snprintf(err, errlen, "--gpu-quant-bits must be 0 or 8"); + return -1; + } + if (opts->gpu_quant_group_size <= 0) { + snprintf(err, errlen, "--gpu-quant-group-size must be > 0"); + return -1; + } + if (ffwd_verbose >= 1) + fprintf(stderr, "Using %s\n", FFWD_BACKEND_LABEL); + return 0; +} + +ffwd_t *ffwd_open( + const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) { + ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b)); + if (!b || !(b->dir = strdup(model_dir))) { + snprintf(err, errlen, "out of memory"); + free(b); + return NULL; + } + b->is_late = is_late; + b->opts = *opts; + return b; +} + +int ffwd_activate(ffwd_t *b, char *err, size_t errlen) { + ffwd_mlx_options_t mlx_opts = { + .quantize_bits = b->opts.gpu_quant_bits, + .quantize_group_size = b->opts.gpu_quant_group_size, + }; + if (b->is_late) + b->late_ctx = ffwd_mlx_late_load_with_options(b->dir, &mlx_opts); + else + b->ctx = ffwd_mlx_load_with_options(b->dir, &mlx_opts); + if (!b->ctx && !b->late_ctx) { + snprintf(err, errlen, "failed to load model"); + return -1; + } + return 0; +} + +void ffwd_worker_free(ffwd_t *b) { + if (!b) + return; + if (b->ctx) { + ffwd_mlx_free(b->ctx); + b->ctx = NULL; + } + if (b->late_ctx) { + ffwd_mlx_late_free(b->late_ctx); + b->late_ctx = NULL; + } +} + +void ffwd_free(ffwd_t *b) { + if (!b) + return; + /* The inference thread releases the context via ffwd_worker_free; + * by the time the main thread frees the handle both are already NULL. The + * single-threaded CLI loads and frees on one thread, so this also releases + * a context that worker_free never ran on. */ + ffwd_worker_free(b); + free(b->dir); + free(b); +} + +const ffwd_config_t *ffwd_config(const ffwd_t *b) { + return b->is_late ? ffwd_mlx_late_config(b->late_ctx) : ffwd_mlx_config(b->ctx); +} + +int ffwd_token_dim(const ffwd_t *b) { + return b->is_late ? ffwd_mlx_late_token_dim(b->late_ctx) : 0; +} + +int ffwd_uses_dense_batches(const ffwd_t *b) { return b->ctx != NULL; } + +int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) { + return ffwd_mlx_encode_batch(b->ctx, inputs, batch, out); +} + +int ffwd_encode_spans_batch(ffwd_t *b, + const ffwd_context_input_t *inputs, + int batch, + float *out) { + return ffwd_mlx_encode_spans_batch(b->ctx, inputs, batch, out); +} + +int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) { + int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets)); + ffwd_mlx_late_vectors_t *query = + ffwd_mlx_late_encode_tokens_device(b->late_ctx, in->query_ids, in->query_n_tokens, 1); + ffwd_mlx_late_vectors_t *packed = + (query && offsets) + ? ffwd_mlx_late_encode_docs_device(b->late_ctx, in->doc_ids, in->doc_n_tokens, + in->doc_keep, in->doc_n_keep, in->n_docs, 1, + offsets) + : NULL; + int rc = packed ? ffwd_mlx_late_maxsim_batch_device(b->late_ctx, query, packed, offsets, + in->n_docs, scores) + : -1; + ffwd_mlx_late_vectors_free(packed); + ffwd_mlx_late_vectors_free(query); + free(offsets); + return rc; +} + +size_t ffwd_scratch_nbytes(const ffwd_t *b) { + (void)b; + return 0; +} + +int ffwd_default_batch_wait_us(void) { return 0; } + +static uint64_t physical_memory_nbytes(void) { +#ifdef __APPLE__ + uint64_t bytes = 0; + size_t len = sizeof(bytes); + if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0) + return bytes; +#endif + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) + return 0; + return (uint64_t)pages * (uint64_t)page_size; +} + +int ffwd_preflight( + const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) { + uint64_t payload = 0; + for (int i = 0; i < n_paths; i++) { + multi_safetensors_t *ms = multi_safetensors_open(paths[i]); + size_t model_payload = 0; + if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { + multi_safetensors_close(ms); + snprintf(err, errlen, "failed to inspect GPU model: %s", paths[i]); + return -1; + } + multi_safetensors_close(ms); + if ((uint64_t)model_payload > UINT64_MAX - payload) { + snprintf(err, errlen, "GPU model payload size overflow"); + return -1; + } + payload += (uint64_t)model_payload; + } + if (payload > UINT64_MAX / FFWD_MLX_RESIDENT_MULTIPLIER) { + snprintf(err, errlen, "GPU resident-memory estimate overflow"); + return -1; + } + uint64_t estimated = payload * FFWD_MLX_RESIDENT_MULTIPLIER; + uint64_t physical = physical_memory_nbytes(); + if (physical == 0) { + fprintf(stderr, "ffwd-server: warning: could not determine physical memory; " + "skipping GPU memory preflight\n"); + return 0; + } + double utilization = opts->memory_utilization > 0.0 ? opts->memory_utilization + : FFWD_MLX_MEMORY_BUDGET_PERCENT / 100.0; + uint64_t budget = (uint64_t)((double)physical * utilization); + const double gib = 1024.0 * 1024.0 * 1024.0; + fprintf(stderr, + "ffwd-server: GPU memory preflight: %.1f GiB tensors, %.1f GiB estimated " + "resident, %.1f GiB budget (%.2f of physical memory)\n", + (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization); + if (opts->gpu_quant_bits) + fprintf(stderr, + "ffwd-server: GPU %d-bit quantization enabled; preflight uses source " + "payload as a conservative peak estimate\n", + opts->gpu_quant_bits); + if (estimated <= budget) + return 0; + snprintf(err, errlen, + "ffwd-server: GPU model set exceeds the host-memory budget; use BF16 artifacts, " + "load fewer models, or raise --memory-utilization"); + return -1; +} diff --git a/src/tokenizer_bpe.c b/libffwd/bpe.c index 511b29e..c74845f 100644 --- a/src/tokenizer_bpe.c +++ b/libffwd/bpe.c @@ -1,5 +1,4 @@ -#include "tokenizer_bpe.h" -#include "kernels.h" +#include "bpe.h" #include <limits.h> #include <stdint.h> #include <stdio.h> @@ -419,8 +418,7 @@ static uint64_t pair_hash(uint64_t k) { return k; } -static void -pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) { +static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) { int mask = cap - 1; int idx = (int)(pair_hash(key) & (uint64_t)mask); for (int probe = 0; probe < cap; probe++, idx = (idx + 1) & mask) { @@ -437,7 +435,7 @@ pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int me /* Returns the rank and sets *merged_id, or INT_MAX when the pair never * merges. */ -static int pair_merge_lookup(const embed_tokenizer_t *tok, int left, int right, int *merged_id) { +static int pair_merge_lookup(const tok_bpe_t *tok, int left, int right, int *merged_id) { const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges; uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right; int mask = tok->int_merges_cap - 1; @@ -453,7 +451,7 @@ static int pair_merge_lookup(const embed_tokenizer_t *tok, int left, int right, return INT_MAX; } -static int merge_rank(const embed_tokenizer_t *tok, const char *a, const char *b) { +static int merge_rank(const tok_bpe_t *tok, const char *a, const char *b) { if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX; @@ -495,8 +493,7 @@ static char *text_to_bpe_unicode(const char *text) { return out; } -static int -encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n); +static int encode_bpe_word(const tok_bpe_t *tok, const char *mapped, int **out_ids, int *out_n); static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) { @@ -828,12 +825,8 @@ static int match_contraction(const unsigned char *p, const unsigned char *end) { return 0; } -static int append_encoded_piece(const embed_tokenizer_t *tok, - const char *piece, - size_t piece_len, - int **ids, - int *n_ids, - int *cap) { +static int append_encoded_piece( + const tok_bpe_t *tok, const char *piece, size_t piece_len, int **ids, int *n_ids, int *cap) { if (piece_len == 0) return 0; @@ -867,8 +860,7 @@ static int append_encoded_piece(const embed_tokenizer_t *tok, return 0; } -static int -encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) { +static int encode_pretokenized(const tok_bpe_t *tok, const char *text, int **out_ids, int *out_n) { const unsigned char *start = (const unsigned char *)text; const unsigned char *p = start; const unsigned char *end = start + strlen(text); @@ -987,8 +979,8 @@ encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_id } } - if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids, - &cap) != 0) { + if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids, &cap) != + 0) { free(ids); return -1; } @@ -1000,8 +992,7 @@ encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_id } /* Encode one mapped BPE unicode string to token IDs. */ -static int -encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { +static int encode_bpe_word(const tok_bpe_t *tok, const char *mapped, int **out_ids, int *out_n) { *out_ids = NULL; *out_n = 0; if (!mapped || !*mapped) @@ -1078,7 +1069,7 @@ typedef struct { size_t off; } bpe_sym_ref_t; -struct embed_tokenizer_workspace { +struct tok_bpe_workspace { char *normalized; size_t normalized_cap; char *mapped; @@ -1101,7 +1092,7 @@ struct embed_tokenizer_workspace { int bpe_cap; }; -static int reserve_bpe(embed_tokenizer_workspace_t *ws, int need) { +static int reserve_bpe(tok_bpe_workspace_t *ws, int need) { if (ws->bpe_cap >= need) return 0; int new_cap = ws->bpe_cap ? ws->bpe_cap : 64; @@ -1122,7 +1113,7 @@ static int reserve_bpe(embed_tokenizer_workspace_t *ws, int need) { R(bpe_rnk) R(bpe_mrg) #undef R - ws->bpe_cap = new_cap; + ws->bpe_cap = new_cap; return 0; } @@ -1160,7 +1151,7 @@ static int reserve_ints(int **buf, int *cap, int need) { return 0; } -static int reserve_syms(embed_tokenizer_workspace_t *ws, int need) { +static int reserve_syms(tok_bpe_workspace_t *ws, int need) { if (ws->syms_cap >= need) return 0; int new_cap = ws->syms_cap ? ws->syms_cap : 64; @@ -1177,7 +1168,7 @@ static int reserve_syms(embed_tokenizer_workspace_t *ws, int need) { return 0; } -static int arena_append(embed_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) { +static int arena_append(tok_bpe_workspace_t *ws, const char *s, size_t len, size_t *off) { if (ws->arena_len > SIZE_MAX - len - 1) return -1; size_t need = ws->arena_len + len + 1; @@ -1190,12 +1181,11 @@ static int arena_append(embed_tokenizer_workspace_t *ws, const char *s, size_t l return 0; } -static const char *arena_str(const embed_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) { +static const char *arena_str(const tok_bpe_workspace_t *ws, bpe_sym_ref_t sym) { return ws->arena + sym.off; } -static int -normalize_nfc_latin_into(embed_tokenizer_workspace_t *ws, const char *text, const char **out) { +static int normalize_nfc_latin_into(tok_bpe_workspace_t *ws, const char *text, const char **out) { const unsigned char *p = (const unsigned char *)text; const unsigned char *end = p + strlen(text); size_t in_len = (size_t)(end - p); @@ -1235,10 +1225,8 @@ normalize_nfc_latin_into(embed_tokenizer_workspace_t *ws, const char *text, cons return 0; } -static int text_to_bpe_unicode_into(embed_tokenizer_workspace_t *ws, - const char *text, - size_t len, - const char **out) { +static int +text_to_bpe_unicode_into(tok_bpe_workspace_t *ws, const char *text, size_t len, const char **out) { init_gpt2_mapping(); if (len > (SIZE_MAX - 1) / 3) return -1; @@ -1267,8 +1255,8 @@ static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow return 0; } -static int encode_bpe_word_into_slow(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, +static int encode_bpe_word_into_slow(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, const char *mapped, int *out_ids, int out_cap, @@ -1349,8 +1337,8 @@ static int encode_bpe_word_into_slow(const embed_tokenizer_t *tok, * the string-based loop when the int tables are unavailable. Identical * output: initial ids are the byte-level single-char token ids and every * merge result is the vocab id of the concatenated pair. */ -static int encode_bpe_word_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, +static int encode_bpe_word_into(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, const char *mapped, int *out_ids, int out_cap, @@ -1418,8 +1406,8 @@ static int encode_bpe_word_into(const embed_tokenizer_t *tok, if (prv[best] != -1) rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], &mrg[prv[best]]); - rnk[best] = nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) - : INT_MAX; + rnk[best] = + nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) : INT_MAX; } for (int j = 0; j != -1; j = nxt[j]) @@ -1427,8 +1415,8 @@ static int encode_bpe_word_into(const embed_tokenizer_t *tok, return 0; } -static int append_encoded_piece_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, +static int append_encoded_piece_into(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, const char *piece, size_t piece_len, int *out_ids, @@ -1444,8 +1432,8 @@ static int append_encoded_piece_into(const embed_tokenizer_t *tok, return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); } -static int encode_pretokenized_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, +static int encode_pretokenized_into(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, const char *text, int *out_ids, int out_cap, @@ -1622,7 +1610,7 @@ static int parse_merge_pair(char *line, char **a, char **b) { return 0; } -static int load_merges_map(embed_tokenizer_t *tok, const char *merges_path) { +static int load_merges_map(tok_bpe_t *tok, const char *merges_path) { FILE *f = fopen(merges_path, "rb"); if (!f) return -1; @@ -1745,10 +1733,10 @@ static int load_merges_map(embed_tokenizer_t *tok, const char *merges_path) { * Public API * ======================================================================== */ -embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { +tok_bpe_t *tok_bpe_load(const char *vocab_json_path) { FILE *f = fopen(vocab_json_path, "rb"); if (!f) { - fprintf(stderr, "embed_tokenizer_load: cannot open %s\n", vocab_json_path); + fprintf(stderr, "tok_bpe_load: cannot open %s\n", vocab_json_path); return NULL; } @@ -1803,7 +1791,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { } int vocab_size = max_id + 1; - embed_tokenizer_t *tok = (embed_tokenizer_t *)calloc(1, sizeof(embed_tokenizer_t)); + tok_bpe_t *tok = (tok_bpe_t *)calloc(1, sizeof(tok_bpe_t)); if (!tok) { free(json); return NULL; @@ -1812,7 +1800,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *)); tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *)); if (!tok->id_to_text || !tok->id_to_bpe) { - embed_tokenizer_free(tok); + tok_bpe_free(tok); free(json); return NULL; } @@ -1852,7 +1840,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); if (!tok->vocab_map) { - embed_tokenizer_free(tok); + tok_bpe_free(tok); return NULL; } for (int i = 0; i < vocab_size; i++) { @@ -1863,7 +1851,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { char merges_path[1024]; if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { - if (load_merges_map(tok, merges_path) != 0 && embed_verbose >= 2) { + if (load_merges_map(tok, merges_path) != 0) { fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", merges_path); @@ -1873,19 +1861,19 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { return tok; } -const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id) { +const char *tok_bpe_decode(const tok_bpe_t *tok, int token_id) { if (!tok || token_id < 0 || token_id >= tok->vocab_size) return ""; return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; } -int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token) { +int tok_bpe_token_id(const tok_bpe_t *tok, const char *token) { if (!tok || !token || !tok->vocab_map) return -1; return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token); } -int *embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens) { +int *tok_bpe_encode(const tok_bpe_t *tok, const char *text, int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !text || text[0] == '\0') @@ -1909,11 +1897,11 @@ int *embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int return ids; } -embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void) { - return (embed_tokenizer_workspace_t *)calloc(1, sizeof(embed_tokenizer_workspace_t)); +tok_bpe_workspace_t *tok_bpe_workspace_new(void) { + return (tok_bpe_workspace_t *)calloc(1, sizeof(tok_bpe_workspace_t)); } -void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws) { +void tok_bpe_workspace_free(tok_bpe_workspace_t *ws) { if (!ws) return; free(ws->normalized); @@ -1929,12 +1917,12 @@ void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws) { free(ws); } -int embed_tokenizer_encode_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens) { +int tok_bpe_encode_into(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !ws || !text || out_cap < 0) @@ -1953,10 +1941,10 @@ int embed_tokenizer_encode_into(const embed_tokenizer_t *tok, return rc; } -int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens) { +int *tok_bpe_encode_with_workspace(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, + const char *text, + int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !ws || !text || text[0] == '\0') @@ -1972,11 +1960,11 @@ int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, return NULL; int n_ids = 0; - int rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); + int rc = tok_bpe_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); if (rc == -2) { if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) return NULL; - rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); + rc = tok_bpe_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); } if (rc != 0 || n_ids <= 0) return NULL; @@ -1990,7 +1978,7 @@ int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, return ids; } -void embed_tokenizer_free(embed_tokenizer_t *tok) { +void tok_bpe_free(tok_bpe_t *tok) { if (!tok) return; diff --git a/libffwd/bpe.h b/libffwd/bpe.h new file mode 100644 index 0000000..5b16c52 --- /dev/null +++ b/libffwd/bpe.h @@ -0,0 +1,74 @@ +/* + * bpe.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * + * Supports: + * - decode token ID -> UTF-8 text + * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) + */ + +#ifndef TOK_BPE_H +#define TOK_BPE_H + +typedef struct { + char **id_to_text; /* [vocab_size] decoded text strings */ + char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ + int vocab_size; + + /* Internal hash maps (opaque to callers) */ + void *vocab_map; + int vocab_map_cap; + void *merge_map; + int merge_map_cap; + /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id). + * NULL when the vocab could not resolve every merge; encoding then falls + * back to the string-based merge loop. */ + void *int_merges; + int int_merges_cap; + /* Byte-level unicode codepoint -> single-character token id (512 slots), + * or NULL when any byte token is missing from the vocab. */ + int *cp_to_id; +} tok_bpe_t; + +typedef struct tok_bpe_workspace tok_bpe_workspace_t; + +/* Load tokenizer from vocab.json in model directory */ +tok_bpe_t *tok_bpe_load(const char *vocab_json_path); + +/* Decode a single token ID to text. Returns pointer to internal string. */ +const char *tok_bpe_decode(const tok_bpe_t *tok, int token_id); + +/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>"). + * Returns -1 when the token is not in the vocab. */ +int tok_bpe_token_id(const tok_bpe_t *tok, const char *token); + +/* Encode UTF-8 text into token IDs using BPE. + * Returns malloc'd array of token IDs and sets *out_n_tokens. + * Returns NULL on error (and sets *out_n_tokens to 0). */ +int *tok_bpe_encode(const tok_bpe_t *tok, const char *text, int *out_n_tokens); + +/* Reusable scratch for allocation-light repeated encoding. */ +tok_bpe_workspace_t *tok_bpe_workspace_new(void); +void tok_bpe_workspace_free(tok_bpe_workspace_t *ws); + +/* Encode into caller-provided out_ids[0..out_cap-1]. + * Returns 0 on success, -1 on error, -2 if out_cap is too small. + * In all cases *out_n_tokens is set to the number of tokens needed/emitted + * when out_n_tokens is non-NULL. */ +int tok_bpe_encode_into(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); + +/* Same tokenization as tok_bpe_encode(), but reuses ws internally and + * performs only the final returned int[] allocation. */ +int *tok_bpe_encode_with_workspace(const tok_bpe_t *tok, + tok_bpe_workspace_t *ws, + const char *text, + int *out_n_tokens); + +/* Free tokenizer */ +void tok_bpe_free(tok_bpe_t *tok); + +#endif /* TOK_BPE_H */ diff --git a/libffwd/build.h b/libffwd/build.h new file mode 100644 index 0000000..01073d5 --- /dev/null +++ b/libffwd/build.h @@ -0,0 +1,49 @@ +/* ffwd_build.h - build identity and capability information for the ffwd tools + * + * The real values come from the Makefile's VERSION_CFLAGS at compile time; the + * fallbacks below apply when those flags are absent. Front-end only: the + * inference library does not use these. */ +#ifndef FFWD_BUILD_H +#define FFWD_BUILD_H + +#include "ffwd.h" /* ffwd_capability(), declared alongside the rest of the API */ + +#include <stdio.h> +#include <string.h> + +#ifndef FFWD_VERSION +# define FFWD_VERSION "development" +#endif +#ifndef FFWD_BUILD_COMMIT +# define FFWD_BUILD_COMMIT "unknown" +#endif +#ifndef FFWD_BUILD_DATE +# define FFWD_BUILD_DATE "unknown" +#endif +#ifndef FFWD_BUILD_OS +# define FFWD_BUILD_OS "unknown" +#endif +#ifndef FFWD_BUILD_ARCH +# define FFWD_BUILD_ARCH "unknown" +#endif + +/* Program name with any leading directory removed. */ +static inline const char *ffwd_prog_name(const char *argv0) { + const char *slash = strrchr(argv0, '/'); + return slash ? slash + 1 : argv0; +} + +static inline void ffwd_print_version(const char *prog) { + printf("%s %s (%s/%s)\n", prog, FFWD_VERSION, FFWD_BUILD_OS, FFWD_BUILD_ARCH); +} + +static inline void ffwd_print_build_info(const char *prog) { + ffwd_print_version(prog); + printf("backend: %s\n" + "compiler: %s\n" + "built: %s\n" + "commit: %s\n", + ffwd_capability(), __VERSION__, FFWD_BUILD_DATE, FFWD_BUILD_COMMIT); +} + +#endif /* FFWD_BUILD_H */ diff --git a/src/config.c b/libffwd/config.c index a33a135..efc4bcb 100644 --- a/src/config.c +++ b/libffwd/config.c @@ -8,6 +8,10 @@ #include <stdlib.h> #include <string.h> +/* Verbosity flag, read across the codebase for optional stderr diagnostics. + * Defined in config.c because every backend links it. */ +int ffwd_verbose = 0; + typedef enum { JSON_BOOL_MISSING = 0, JSON_BOOL_FALSE = 1, @@ -74,7 +78,7 @@ static int read_optional_file(const char *model_dir, const char *relative, char char path[1024]; int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative); if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "embed_config: model path too long\n"); + fprintf(stderr, "ffwd_config: model path too long\n"); return -1; } @@ -82,7 +86,7 @@ static int read_optional_file(const char *model_dir, const char *relative, char if (!f && errno == ENOENT) return 0; if (!f) { - fprintf(stderr, "embed_config: cannot open %s\n", path); + fprintf(stderr, "ffwd_config: cannot open %s\n", path); return -1; } if (fseek(f, 0, SEEK_END) != 0) { @@ -106,7 +110,7 @@ static int read_optional_file(const char *model_dir, const char *relative, char return 1; } -static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char *model_dir) { +static int parse_sentence_transformers_metadata(ffwd_config_t *cfg, const char *model_dir) { char *buf = NULL; int rc = read_optional_file(model_dir, "1_Pooling/config.json", &buf); if (rc < 0) @@ -119,22 +123,21 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char json_bool_t sqrt_len = json_get_bool(buf, "pooling_mode_mean_sqrt_len_tokens"); json_bool_t weighted = json_get_bool(buf, "pooling_mode_weightedmean_tokens"); if (mean < 0 || last < 0 || cls < 0 || max < 0 || sqrt_len < 0 || weighted < 0) { - fprintf(stderr, "embed_config: invalid pooling metadata\n"); + fprintf(stderr, "ffwd_config: invalid pooling metadata\n"); free(buf); return -1; } - int supported = - (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE) + (cls == JSON_BOOL_TRUE); + int supported = (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE) + (cls == JSON_BOOL_TRUE); int unsupported = max == JSON_BOOL_TRUE || sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE; if (supported != 1 || unsupported) { - fprintf(stderr, "embed_config: unsupported Sentence Transformers pooling mode\n"); + fprintf(stderr, "ffwd_config: unsupported Sentence Transformers pooling mode\n"); free(buf); return -1; } - cfg->pooling_mode = cls == JSON_BOOL_TRUE ? EMBED_POOL_CLS - : last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN - : EMBED_POOL_MEAN; + cfg->pooling_mode = cls == JSON_BOOL_TRUE ? FFWD_POOL_CLS + : last == JSON_BOOL_TRUE ? FFWD_POOL_LAST_TOKEN + : FFWD_POOL_MEAN; free(buf); buf = NULL; } @@ -143,8 +146,7 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char if (rc < 0) return -1; if (rc > 0) { - cfg->normalize_embeddings = - strstr(buf, "\"sentence_transformers.models.Normalize\"") != NULL; + cfg->normalize_embeddings = strstr(buf, "\"sentence_transformers.models.Normalize\"") != NULL; free(buf); buf = NULL; } @@ -155,7 +157,7 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char if (rc > 0) { json_bool_t add_eos = json_get_bool(buf, "add_eos_token"); if (add_eos < 0) { - fprintf(stderr, "embed_config: invalid add_eos_token metadata\n"); + fprintf(stderr, "ffwd_config: invalid add_eos_token metadata\n"); free(buf); return -1; } @@ -166,17 +168,17 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char return 0; } -int embed_config_parse(embed_config_t *cfg, const char *model_dir) { +int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) { char path[1024]; int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "embed_config: model path too long\n"); + fprintf(stderr, "ffwd_config: model path too long\n"); return -1; } FILE *f = fopen(path, "rb"); if (!f) { - fprintf(stderr, "embed_config: cannot open %s\n", path); + fprintf(stderr, "ffwd_config: cannot open %s\n", path); return -1; } @@ -202,46 +204,46 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0); cfg->n_heads = json_get_int(buf, "num_attention_heads", 0); cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0); - cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM); + cfg->head_dim = json_get_int(buf, "head_dim", FFWD_HEAD_DIM); cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0); - cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE); + cfg->vocab_size = json_get_int(buf, "vocab_size", FFWD_VOCAB_SIZE); cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6); cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12); cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0); cfg->position_id_offset = 0; cfg->type_vocab_size = json_get_int(buf, "type_vocab_size", 2); - cfg->family = EMBED_FAMILY_QWEN3; + cfg->family = FFWD_FAMILY_QWEN3; cfg->qk_norm = 1; cfg->qkv_bias = 0; - cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; - cfg->pooling_mode = EMBED_POOL_MEAN; + cfg->attention_mode = FFWD_ATTENTION_BIDIRECTIONAL; + cfg->pooling_mode = FFWD_POOL_MEAN; cfg->normalize_embeddings = 0; cfg->append_terminal_token = 0; if (json_string_equals(buf, "model_type", "qwen3")) { - cfg->attention_mode = EMBED_ATTENTION_CAUSAL; - cfg->pooling_mode = EMBED_POOL_LAST_TOKEN; + cfg->attention_mode = FFWD_ATTENTION_CAUSAL; + cfg->pooling_mode = FFWD_POOL_LAST_TOKEN; cfg->normalize_embeddings = 1; cfg->append_terminal_token = 1; } else if (json_string_equals(buf, "model_type", "qwen2")) { cfg->qk_norm = 0; cfg->qkv_bias = 1; - cfg->attention_mode = EMBED_ATTENTION_CAUSAL; + cfg->attention_mode = FFWD_ATTENTION_CAUSAL; } else if (json_string_equals(buf, "model_type", "bert") || json_string_equals(buf, "model_type", "roberta") || json_string_equals(buf, "model_type", "xlm-roberta")) { - cfg->family = EMBED_FAMILY_BERT; + cfg->family = FFWD_FAMILY_BERT; cfg->qk_norm = 0; cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */ - cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; + cfg->attention_mode = FFWD_ATTENTION_BIDIRECTIONAL; cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */ cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; if (json_string_equals(buf, "model_type", "roberta") || json_string_equals(buf, "model_type", "xlm-roberta")) { int pad_id = json_get_int(buf, "pad_token_id", 1); if (pad_id < 0 || pad_id == INT_MAX) { - fprintf(stderr, "embed_config: invalid RoBERTa pad_token_id\n"); + fprintf(stderr, "ffwd_config: invalid RoBERTa pad_token_id\n"); free(buf); return -1; } @@ -252,13 +254,12 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { * (gelu, absent) is the exact erf GeLU the released encoders use. */ if (json_string_equals(buf, "hidden_act", "gelu_new") || json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) { - cfg->ffn_act = EMBED_ACT_GELU_TANH; + cfg->ffn_act = FFWD_ACT_GELU_TANH; } if (cfg->max_position_embeddings <= 0 || - cfg->position_id_offset >= cfg->max_position_embeddings || - cfg->type_vocab_size <= 0 || !isfinite(cfg->layer_norm_eps) || - cfg->layer_norm_eps <= 0.0f) { - fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n"); + cfg->position_id_offset >= cfg->max_position_embeddings || cfg->type_vocab_size <= 0 || + !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { + fprintf(stderr, "ffwd_config: invalid BERT layer_norm_eps/max_position_embeddings\n"); free(buf); return -1; } @@ -266,13 +267,13 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { json_bool_t is_causal = json_get_bool(buf, "is_causal"); if (is_causal < 0) { - fprintf(stderr, "embed_config: invalid is_causal in %s\n", path); + fprintf(stderr, "ffwd_config: invalid is_causal in %s\n", path); free(buf); return -1; } if (is_causal != JSON_BOOL_MISSING) { cfg->attention_mode = - is_causal == JSON_BOOL_TRUE ? EMBED_ATTENTION_CAUSAL : EMBED_ATTENTION_BIDIRECTIONAL; + is_causal == JSON_BOOL_TRUE ? FFWD_ATTENTION_CAUSAL : FFWD_ATTENTION_BIDIRECTIONAL; } cfg->q_dim = 0; @@ -293,28 +294,28 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) { fprintf(stderr, - "embed_config: invalid config in %s " + "ffwd_config: invalid config in %s " "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n", path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size); return -1; } - if (cfg->n_layers > EMBED_MAX_LAYERS) { - fprintf(stderr, "embed_config: too many layers (%d > %d)\n", cfg->n_layers, - EMBED_MAX_LAYERS); + if (cfg->n_layers > FFWD_MAX_LAYERS) { + fprintf(stderr, "ffwd_config: too many layers (%d > %d)\n", cfg->n_layers, + FFWD_MAX_LAYERS); return -1; } - if (embed_verbose >= 1) + if (ffwd_verbose >= 1) fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, " "inter=%d, head_dim=%d, attention=%s, pooling=%s, qk_norm=%s, qkv_bias=%s\n", cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->intermediate_size, cfg->head_dim, - cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional", - cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" - : cfg->pooling_mode == EMBED_POOL_CLS ? "cls" - : "mean", + cfg->attention_mode == FFWD_ATTENTION_CAUSAL ? "causal" : "bidirectional", + cfg->pooling_mode == FFWD_POOL_LAST_TOKEN ? "last-token" + : cfg->pooling_mode == FFWD_POOL_CLS ? "cls" + : "mean", cfg->qk_norm ? "yes" : "no", cfg->qkv_bias ? "yes" : "no"); return 0; diff --git a/libffwd/config.h b/libffwd/config.h new file mode 100644 index 0000000..b3e2f67 --- /dev/null +++ b/libffwd/config.h @@ -0,0 +1,8 @@ +#ifndef FFWD_CONFIG_H +#define FFWD_CONFIG_H + +#include "ffwd.h" + +int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir); + +#endif /* FFWD_CONFIG_H */ diff --git a/src/cuda.cu b/libffwd/cuda.cu index 872834b..2b329cd 100644 --- a/src/cuda.cu +++ b/libffwd/cuda.cu @@ -1,6 +1,6 @@ extern "C" { #include "cuda.h" -#include "internal.h" +#include "model_types.h" } #include <cuda_runtime.h> @@ -39,13 +39,13 @@ typedef struct { float *ffn_ln_bias; } cuda_layer_t; -struct embed_cuda_ctx { - embed_model_t *cpu; - int own_cpu; // free ctx->cpu in embed_cuda_free only when this context owns it - embed_config_t config; +struct ffwd_cuda_ctx { + ffwd_model_t *cpu; + int own_cpu; // free ctx->cpu in ffwd_cuda_free only when this context owns it + ffwd_config_t config; cudaStream_t stream; cublasHandle_t blas; - cuda_matrix_t embed_tokens; + cuda_matrix_t ffwd_tokens; cuda_layer_t *layers; float *norm; /* BERT family (NULL for Qwen3): learned absolute position embeddings, the @@ -53,8 +53,8 @@ struct embed_cuda_ctx { * LayerNorm weight/bias. */ float *position_embeddings; float *token_type_embedding; - float *embed_ln_w; - float *embed_ln_b; + float *ffwd_ln_w; + float *ffwd_ln_b; int seq_cap; int batch_cap; @@ -143,7 +143,7 @@ static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) { static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F; static int g_gemm_compute_set = 0; -int embed_cuda_set_fast_gemm(const char *mode) { +int ffwd_cuda_set_fast_gemm(const char *mode) { cublasComputeType_t ct; if (parse_gemm_compute(mode, &ct) != 0) return -1; @@ -184,13 +184,13 @@ static uint16_t f32_to_bf16(float f) { static int g_weights_bf16 = 0; static int g_weights_bf16_set = 0; -int embed_cuda_set_weights_bf16(int on) { +int ffwd_cuda_set_weights_bf16(int on) { g_weights_bf16 = on ? 1 : 0; g_weights_bf16_set = 1; return 0; } -static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, size_t count) { +static int copy_weight_host_f32(float *dst, const ffwd_weight_ref_t *w, size_t count) { if (!dst || !w || !w->data) return -1; if (w->dtype == DTYPE_F32) { @@ -244,7 +244,7 @@ static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, int r return 0; } -static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, int cols) { +static int load_matrix(cuda_matrix_t *m, const ffwd_weight_ref_t *w, int rows, int cols) { if (!m || !w || rows <= 0 || cols <= 0) return -1; size_t count = (size_t)rows * (size_t)cols; @@ -260,7 +260,7 @@ static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, return r; } -static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) { +static int load_qkv_matrix(cuda_matrix_t *m, const ffwd_layer_t *src, const ffwd_config_t *c) { if (!m || !src || !c) return -1; int rows = c->q_dim + 2 * c->kv_dim; @@ -283,7 +283,7 @@ static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const emb } static int -load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) { +load_gate_up_matrix(cuda_matrix_t *m, const ffwd_layer_t *src, const ffwd_config_t *c) { if (!m || !src || !c) return -1; int rows = 2 * c->intermediate_size; @@ -321,7 +321,7 @@ static int load_vector(float **out, const float *src, int n) { return 0; } -static int load_qkv_bias(float **out, const embed_layer_t *src, const embed_config_t *c) { +static int load_qkv_bias(float **out, const ffwd_layer_t *src, const ffwd_config_t *c) { int n = c->q_dim + 2 * c->kv_dim; float *host = (float *)malloc((size_t)n * sizeof(float)); if (!host) @@ -341,7 +341,7 @@ __global__ static void add_row_bias_kernel(float *x, const float *bias, int rows x[idx] += bias[idx % dim]; } -__global__ static void embed_lookup_kernel( +__global__ static void ffwd_lookup_kernel( float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int count = total * hidden; @@ -492,8 +492,8 @@ __global__ static void rms_norm_rope_qk_kernel(float *x, // over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16 // path writes a packed [rows x intermediate] BF16 matrix for the down GEMM. template <typename OUT_T> -__global__ static void silu_mul_packed_kernel( - OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) { +__global__ static void +silu_mul_packed_kernel(OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) { int i = blockIdx.x * blockDim.x + threadIdx.x; int n = rows * intermediate; if (i < n) { @@ -541,8 +541,7 @@ __global__ static void attn_expand_kv_kernel(float *kexp, // One block per (head, query) row; scores are pre-scaled by the QK^T GEMM. // P_T is float with probs == scores (in place), or __nv_bfloat16 when the // probabilities feed a BF16 @V GEMM. -template <typename P_T> -__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) { +template <typename P_T> __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) { float *s = scores + (size_t)blockIdx.x * L; P_T *p = probs + (size_t)blockIdx.x * L; int tid = threadIdx.x; @@ -669,14 +668,9 @@ __global__ static void span_pool_kernel( // BERT LayerNorm: mean-subtract, biased variance (/dim), scale by gamma, shift // by beta. One block per row, two block reductions before any write, so it is -// safe in place (out == x). Matches the CPU embed_layer_norm. -__global__ static void layer_norm_kernel(float *out, - const float *x, - const float *gamma, - const float *beta, - int rows, - int dim, - float eps) { +// safe in place (out == x). Matches the CPU layer_norm. +__global__ static void layer_norm_kernel( + float *out, const float *x, const float *gamma, const float *beta, int rows, int dim, float eps) { int row = blockIdx.x; int tid = threadIdx.x; const float *xr = x + (size_t)row * dim; @@ -744,13 +738,13 @@ static int launch_check(void) { } // Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm). -// EMBED_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core +// FFWD_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core // matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands // internally. static cublasComputeType_t gemm_compute(void) { static int env_init = 0; if (!g_gemm_compute_set && !env_init) { - const char *e = getenv("EMBED_CUDA_GEMM_MODE"); + const char *e = getenv("FFWD_CUDA_GEMM_MODE"); if (e && parse_gemm_compute(e, &g_gemm_compute) == 0) g_gemm_compute_set = 1; env_init = 1; @@ -760,7 +754,7 @@ static cublasComputeType_t gemm_compute(void) { // Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only // BF16 path materializes one F32 weight at a time into this buffer. -static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) { +static int ensure_weight_f32(ffwd_cuda_ctx_t *ctx, size_t elems) { if (ctx->weight_f32_elems >= elems) return 0; cudaFree(ctx->weight_f32); @@ -777,7 +771,7 @@ static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) { // back to F32 here, so the GEMM is exact (honoring gemm_compute): BF16 is then a // storage choice that costs no quality, which the post-norm LayerNorm needs. // With F32 weights this is the exact path. -static int linear_ex(embed_cuda_ctx_t *ctx, +static int linear_ex(ffwd_cuda_ctx_t *ctx, const cuda_matrix_t *w, const void *x, int x_is_bf16, @@ -789,7 +783,7 @@ static int linear_ex(embed_cuda_ctx_t *ctx, float beta) { const float alpha = 1.0f; if (w->bf16) { - if (!x_is_bf16 && ctx->config.family == EMBED_FAMILY_BERT) { + if (!x_is_bf16 && ctx->config.family == FFWD_FAMILY_BERT) { size_t welems = (size_t)out_dim * (size_t)in_dim; if (ensure_weight_f32(ctx, welems) != 0) return -1; @@ -814,19 +808,18 @@ static int linear_ex(embed_cuda_ctx_t *ctx, return -1; xb = ctx->act_bf16; } - CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, - &alpha, w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride, - &beta, y, CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F, - CUBLAS_GEMM_DEFAULT)); + CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha, + w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride, &beta, y, + CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); return 0; } CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha, - w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y, - CUDA_R_32F, out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT)); + w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y, CUDA_R_32F, + out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT)); return 0; } -static int linear(embed_cuda_ctx_t *ctx, +static int linear(ffwd_cuda_ctx_t *ctx, const cuda_matrix_t *w, const float *x, float *y, @@ -837,7 +830,7 @@ static int linear(embed_cuda_ctx_t *ctx, } // x points at activations already stored as BF16 (e.g. ctx->act_bf16). -static int linear_bf16x(embed_cuda_ctx_t *ctx, +static int linear_bf16x(ffwd_cuda_ctx_t *ctx, const cuda_matrix_t *w, const void *x, float *y, @@ -849,7 +842,7 @@ static int linear_bf16x(embed_cuda_ctx_t *ctx, return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta); } -static int linear_accum(embed_cuda_ctx_t *ctx, +static int linear_accum(ffwd_cuda_ctx_t *ctx, const cuda_matrix_t *w, const float *x, float *y, @@ -879,9 +872,8 @@ static int gemm_strided_batched(cublasHandle_t blas, int batch_count) { cublasComputeType_t ct = gemm_compute(); if (ct == CUBLAS_COMPUTE_32F) { - CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda, - strideA, B, ldb, strideB, beta, C, ldc, strideC, - batch_count)); + CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda, strideA, + B, ldb, strideB, beta, C, ldc, strideC, batch_count)); } else { CUBLAS_CHECK(cublasGemmStridedBatchedEx( blas, transa, transb, m, n, k, alpha, A, CUDA_R_32F, lda, strideA, B, CUDA_R_32F, ldb, @@ -901,8 +893,8 @@ static int gemm_strided_batched(cublasHandle_t blas, // layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged // batches, single sequences, or when one sequence's scores exceed the // budget. -static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) { - const embed_config_t *c = &ctx->config; +static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) { + const ffwd_config_t *c = &ctx->config; ctx->attn_G = 0; if (batch < 2) return 0; @@ -991,7 +983,7 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, in // out_bf16 != NULL selects the BF16-weight layout: V and the softmax // probabilities are stored BF16 and the @V GEMM emits BF16 attention output // at out_bf16 (stride q_dim), ready for the wo projection without a cast. -static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, +static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx, const int *h_offsets, int batch, int q_offset, @@ -1000,12 +992,12 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, int qkv_dim, float scale, __nv_bfloat16 *out_bf16) { - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; int q_dim = c->q_dim; int hd = c->head_dim; int H = c->n_heads; int total = h_offsets[batch]; - int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; + int causal = c->attention_mode == FFWD_ATTENTION_CAUSAL; const float alpha = scale, beta = 0.0f, one = 1.0f; int threads = 256; @@ -1016,8 +1008,7 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, k_offset, v_offset); } else { attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>( - ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset, - v_offset); + ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset, v_offset); } if (launch_check() != 0) return -1; @@ -1038,9 +1029,9 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, size_t off = (size_t)s * H; /* scores = scale * Q @ K^T, batched over (seq, head) */ CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, - K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F, - qkv_dim, &beta, S + off, CUDA_R_32F, L, n, - gemm_compute(), CUBLAS_GEMM_DEFAULT)); + K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F, qkv_dim, + &beta, S + off, CUDA_R_32F, L, n, gemm_compute(), + CUBLAS_GEMM_DEFAULT)); if (out_bf16) { if (causal) attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>( @@ -1072,8 +1063,8 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset; const float *K = ctx->kexp + (size_t)start * q_dim; /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */ - if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim, - hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L, + if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim, hd, + Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L, H) != 0) return -1; if (out_bf16) { @@ -1087,10 +1078,10 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, if (launch_check() != 0) return -1; /* O[h] = P[h] @ V[h] (row-major C = A @ B) */ - CUBLAS_CHECK(cublasGemmStridedBatchedEx( - ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, CUDA_R_16BF, q_dim, hd, P, - CUDA_R_16BF, L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, hd, H, - CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + CUBLAS_CHECK(cublasGemmStridedBatchedEx(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, + &one, V, CUDA_R_16BF, q_dim, hd, P, CUDA_R_16BF, + L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, + hd, H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); } else { const float *V = ctx->vexp + (size_t)start * q_dim; float *O = ctx->attn_out + (size_t)start * q_dim; @@ -1112,8 +1103,8 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx, return 0; } -static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) { - const embed_config_t *c = &ctx->config; +static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_seq) { + const ffwd_config_t *c = &ctx->config; if (total > ctx->seq_cap) { cudaFree(ctx->x); cudaFree(ctx->x_norm); @@ -1131,8 +1122,7 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s ctx->kexp = ctx->vexp = NULL; ctx->act_bf16 = NULL; CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float))); - CUDA_CHECK( - cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->qkv, (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float))); @@ -1143,8 +1133,8 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float))); CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float))); if (g_weights_bf16) - CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * 2 * c->intermediate_size * - sizeof(__nv_bfloat16))); + CUDA_CHECK(cudaMalloc(&ctx->act_bf16, + (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16))); ctx->seq_cap = total; } if (batch + 1 > ctx->batch_cap) { @@ -1198,11 +1188,9 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s cudaMalloc((void **)&ctx->rope_cos, (size_t)max_seq * c->head_dim * sizeof(float))); CUDA_CHECK( cudaMalloc((void **)&ctx->rope_sin, (size_t)max_seq * c->head_dim * sizeof(float))); - CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, - (size_t)max_seq * c->head_dim * sizeof(float), + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, (size_t)max_seq * c->head_dim * sizeof(float), cudaMemcpyHostToDevice, ctx->stream)); - CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin, - (size_t)max_seq * c->head_dim * sizeof(float), + CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin, (size_t)max_seq * c->head_dim * sizeof(float), cudaMemcpyHostToDevice, ctx->stream)); free(hcos); free(hsin); @@ -1211,19 +1199,18 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s return 0; } -static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) { +static int ensure_pooled_rows(ffwd_cuda_ctx_t *ctx, int rows) { if (rows <= ctx->pooled_rows_cap) return 0; - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; cudaFree(ctx->pooled_out); ctx->pooled_out = NULL; - CUDA_CHECK( - cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float))); + CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float))); ctx->pooled_rows_cap = rows; return 0; } -static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) { +static int ensure_span_buffers(ffwd_cuda_ctx_t *ctx, int n_spans) { if (n_spans <= ctx->span_cap) return 0; cudaFree(ctx->span_starts); @@ -1236,7 +1223,7 @@ static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) { return 0; } -static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) { +static int load_layer(cuda_layer_t *dst, const ffwd_layer_t *src, const ffwd_config_t *c) { return load_qkv_matrix(&dst->qkv, src, c) || load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || (c->qkv_bias && load_qkv_bias(&dst->qkv_bias, src, c)) || @@ -1252,7 +1239,7 @@ static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_c * kv_dim == hidden); the two block LayerNorm weights load into input_norm * (post-attention) and post_attn_norm (post-FFN); gate_up_proj holds the single * up_proj. The added fields carry the biases the Qwen slots lack. */ -static int load_bert_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) { +static int load_bert_layer(cuda_layer_t *dst, const ffwd_layer_t *src, const ffwd_config_t *c) { return load_qkv_matrix(&dst->qkv, src, c) || load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) || load_qkv_bias(&dst->qkv_bias, src, c) || @@ -1285,75 +1272,75 @@ static void free_layer(cuda_layer_t *l) { } /* Build a device context from an already-loaded CPU model. own_cpu records - * whether embed_cuda_free should release that CPU model: embed_cuda_load owns + * whether ffwd_cuda_free should release that CPU model: ffwd_cuda_load owns * the model it loaded; the late path passes a model owned by its CPU late * model and sets own_cpu = 0. */ -static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) { +static ffwd_cuda_ctx_t *cuda_ctx_from_model(ffwd_model_t *cpu, int own_cpu) { if (!cpu) return NULL; - embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx)); + ffwd_cuda_ctx_t *ctx = (ffwd_cuda_ctx_t *)calloc(1, sizeof(*ctx)); if (!ctx) { if (own_cpu) - embed_model_free(cpu); + ffwd_model_free(cpu); return NULL; } ctx->cpu = cpu; ctx->own_cpu = own_cpu; ctx->config = cpu->config; - /* Without an explicit --cuda-weight-dtype, store weights in the model + /* Without an explicit --gpu-weight-dtype, store weights in the model * file's dtype: BF16 snapshots load as BF16 (bit-exact pass-through), * F32 snapshots keep the exact F32 default. */ if (!g_weights_bf16_set) { - g_weights_bf16 = cpu->weights.embed_tokens.dtype == DTYPE_BF16; + g_weights_bf16 = cpu->weights.ffwd_tokens.dtype == DTYPE_BF16; if (g_weights_bf16) fprintf(stderr, "cuda: BF16 model file; storing weights as BF16 " - "(use --cuda-weight-dtype f32 to override)\n"); + "(use --gpu-weight-dtype f32 to override)\n"); } if (cudaStreamCreate(&ctx->stream) != cudaSuccess || cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) { - embed_cuda_free(ctx); + ffwd_cuda_free(ctx); return NULL; } - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers)); if (!ctx->layers) { - embed_cuda_free(ctx); + ffwd_cuda_free(ctx); return NULL; } - if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, c->vocab_size, + if (load_matrix(&ctx->ffwd_tokens, &cpu->weights.ffwd_tokens, c->vocab_size, c->hidden_size) != 0) { - embed_cuda_free(ctx); + ffwd_cuda_free(ctx); return NULL; } - if (c->family == EMBED_FAMILY_BERT) { + if (c->family == FFWD_FAMILY_BERT) { /* BERT has no final norm; it adds learned position + token-type[0] * embeddings and an embedding LayerNorm (only row 0 of token-type is * ever used, so load just that row). */ - const embed_weights_t *w = &cpu->weights; + const ffwd_weights_t *w = &cpu->weights; if (load_vector(&ctx->position_embeddings, w->position_embeddings, c->max_position_embeddings * c->hidden_size) != 0 || load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) != 0 || - load_vector(&ctx->embed_ln_w, w->embed_ln_w, c->hidden_size) != 0 || - load_vector(&ctx->embed_ln_b, w->embed_ln_b, c->hidden_size) != 0) { - embed_cuda_free(ctx); + load_vector(&ctx->ffwd_ln_w, w->ffwd_ln_w, c->hidden_size) != 0 || + load_vector(&ctx->ffwd_ln_b, w->ffwd_ln_b, c->hidden_size) != 0) { + ffwd_cuda_free(ctx); return NULL; } } else if (load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) { - embed_cuda_free(ctx); + ffwd_cuda_free(ctx); return NULL; } for (int i = 0; i < c->n_layers; i++) { - int rc = c->family == EMBED_FAMILY_BERT + int rc = c->family == FFWD_FAMILY_BERT ? load_bert_layer(&ctx->layers[i], &cpu->weights.layers[i], c) : load_layer(&ctx->layers[i], &cpu->weights.layers[i], c); if (rc != 0) { fprintf(stderr, "cuda: failed to load layer %d\n", i); - embed_cuda_free(ctx); + ffwd_cuda_free(ctx); return NULL; } } @@ -1361,14 +1348,14 @@ static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) { return ctx; } -embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) { - return cuda_ctx_from_model(embed_model_load(model_dir), 1); +ffwd_cuda_ctx_t *ffwd_cuda_load(const char *model_dir) { + return cuda_ctx_from_model(ffwd_model_load(model_dir), 1); } -void embed_cuda_free(embed_cuda_ctx_t *ctx) { +void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx) { if (!ctx) return; - cuda_matrix_free(&ctx->embed_tokens); + cuda_matrix_free(&ctx->ffwd_tokens); if (ctx->layers) { for (int i = 0; i < ctx->config.n_layers; i++) free_layer(&ctx->layers[i]); @@ -1377,8 +1364,8 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) { cudaFree(ctx->norm); cudaFree(ctx->position_embeddings); cudaFree(ctx->token_type_embedding); - cudaFree(ctx->embed_ln_w); - cudaFree(ctx->embed_ln_b); + cudaFree(ctx->ffwd_ln_w); + cudaFree(ctx->ffwd_ln_b); cudaFree(ctx->x); cudaFree(ctx->x_norm); cudaFree(ctx->qkv); @@ -1406,19 +1393,19 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) { if (ctx->stream) cudaStreamDestroy(ctx->stream); if (ctx->own_cpu) - embed_model_free(ctx->cpu); + ffwd_model_free(ctx->cpu); free(ctx); } -const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) { +const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx) { return ctx ? &ctx->config : NULL; } // Pack inputs into the device scratch every forward shares: token ids, the // per-sequence positions 0..L-1, and the batch offsets. Grows scratch via // ensure_buffers. Returns the packed token count in *out_total, or -1 on error. -static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx, - const embed_input_t *inputs, +static int cuda_upload_packed_inputs(ffwd_cuda_ctx_t *ctx, + const ffwd_input_t *inputs, int batch, int *out_total) { int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int)); @@ -1435,7 +1422,7 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx, if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; } - if (ctx->config.family == EMBED_FAMILY_BERT && + if (ctx->config.family == FFWD_FAMILY_BERT && max_seq > ctx->config.max_position_embeddings - ctx->config.position_id_offset) { free(h_offsets); return -1; @@ -1489,8 +1476,9 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx, // F32 activations: BERT checkpoints are F32, so the linear/linear_accum F32 // entry points are used throughout (they still handle bf16 weights via an // internal cast, which keeps a future bf16 BERT correct if not yet optimized). -static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) { - const embed_config_t *c = &ctx->config; +static int +cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, int batch) { + const ffwd_config_t *c = &ctx->config; int total = 0; if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0) return -1; @@ -1504,8 +1492,8 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i int threads = 256; int blocks_hidden = (total * hidden + threads - 1) / threads; - embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( - ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, hidden, + ffwd_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->token_ids, ctx->ffwd_tokens.d, ctx->ffwd_tokens.bf16, total, hidden, c->vocab_size); if (launch_check() != 0) return -1; @@ -1513,8 +1501,8 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i ctx->x, ctx->positions, ctx->position_embeddings, ctx->token_type_embedding, total, hidden); if (launch_check() != 0) return -1; - layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->embed_ln_w, - ctx->embed_ln_b, total, hidden, eps); + layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->ffwd_ln_w, + ctx->ffwd_ln_b, total, hidden, eps); if (launch_check() != 0) return -1; @@ -1532,15 +1520,15 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i total, qkv_dim); if (launch_check() != 0) return -1; - if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, - qkv_dim, scale, NULL) != 0) + if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, qkv_dim, + scale, NULL) != 0) return -1; // Output dense(+bias), residual into x, then post-attention LayerNorm. if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, hidden) != 0) return -1; - add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>( - ctx->x, l->o_bias, total, hidden); + add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>(ctx->x, l->o_bias, + total, hidden); if (launch_check() != 0) return -1; layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->input_norm, @@ -1556,7 +1544,7 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i ctx->ffn_gate_up, l->ffn_inter_bias, total, inter); if (launch_check() != 0) return -1; - if (c->ffn_act == EMBED_ACT_GELU_TANH) + if (c->ffn_act == FFWD_ACT_GELU_TANH) gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>( ctx->ffn_gate_up, (size_t)inter_count); else @@ -1575,15 +1563,15 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i if (launch_check() != 0) return -1; } - // No final norm; embed_cuda_encode_batch pools ctx->x directly. + // No final norm; ffwd_cuda_encode_batch pools ctx->x directly. return 0; } -static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) { +static int cuda_forward_batch(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, int batch) { if (!ctx || !inputs || batch <= 0) return -1; - const embed_config_t *c = &ctx->config; - if (c->family == EMBED_FAMILY_BERT) + const ffwd_config_t *c = &ctx->config; + if (c->family == FFWD_FAMILY_BERT) return cuda_forward_batch_bert(ctx, inputs, batch); int total = 0; if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0) @@ -1591,9 +1579,9 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs int threads = 256; int blocks_hidden = (total * c->hidden_size + threads - 1) / threads; - embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( - ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, c->hidden_size, - c->vocab_size); + ffwd_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>( + ctx->x, ctx->token_ids, ctx->ffwd_tokens.d, ctx->ffwd_tokens.bf16, total, + c->hidden_size, c->vocab_size); if (launch_check() != 0) return -1; @@ -1621,8 +1609,8 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs c->hidden_size, 0.0f) != 0) return -1; } else { - rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm, - total, c->hidden_size, c->rms_norm_eps); + rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm, total, + c->hidden_size, c->rms_norm_eps); if (launch_check() != 0) return -1; if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, total, c->hidden_size, qkv_dim) != 0) @@ -1644,8 +1632,8 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs if (launch_check() != 0) return -1; - if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, - qkv_dim, scale, wbf16 ? act : NULL) != 0) + if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, qkv_dim, + scale, wbf16 ? act : NULL) != 0) return -1; if (wbf16) { @@ -1689,40 +1677,39 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs if (launch_check() != 0) return -1; if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, total, - c->intermediate_size, c->hidden_size, 2 * c->intermediate_size, - 1.0f) != 0) + c->intermediate_size, c->hidden_size, 2 * c->intermediate_size, 1.0f) != 0) return -1; } } - rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total, - c->hidden_size, c->rms_norm_eps); + rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total, c->hidden_size, + c->rms_norm_eps); if (launch_check() != 0) return -1; return 0; } -int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, - const embed_input_t *inputs, - int batch, - float *out_embeddings) { +int ffwd_cuda_encode_batch(ffwd_cuda_ctx_t *ctx, + const ffwd_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; if (cuda_forward_batch(ctx, inputs, batch) != 0) return -1; if (ensure_pooled_rows(ctx, batch) != 0) return -1; - if (c->pooling_mode == EMBED_POOL_LAST_TOKEN) - last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, - batch, c->hidden_size); - else if (c->pooling_mode == EMBED_POOL_CLS) + if (c->pooling_mode == FFWD_POOL_LAST_TOKEN) + last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, batch, + c->hidden_size); + else if (c->pooling_mode == FFWD_POOL_CLS) first_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, batch, c->hidden_size); else - mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, - batch, c->hidden_size); + mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, batch, + c->hidden_size); if (launch_check() != 0) return -1; cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, @@ -1736,7 +1723,7 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, } if (c->normalize_embeddings) { for (int b = 0; b < batch; b++) { - if (embed_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) != + if (ffwd_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) != 0) return -1; } @@ -1744,17 +1731,17 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, return 0; } -int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, - const embed_context_input_t *inputs, - int batch, - float *out_embeddings) { +int ffwd_cuda_encode_spans_batch(ffwd_cuda_ctx_t *ctx, + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; int total_spans = 0; int total_tokens = 0; for (int b = 0; b < batch; b++) { - const embed_context_input_t *input = &inputs[b]; + const ffwd_context_input_t *input = &inputs[b]; int n_tokens = input->input.n_tokens; if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans || total_tokens > INT_MAX - n_tokens) @@ -1769,7 +1756,7 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, total_tokens += n_tokens; } - embed_input_t *packed = (embed_input_t *)malloc((size_t)batch * sizeof(*packed)); + ffwd_input_t *packed = (ffwd_input_t *)malloc((size_t)batch * sizeof(*packed)); int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int)); int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int)); if (!packed || !h_starts || !h_lens) { @@ -1782,7 +1769,7 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, int token_offset = 0; int span_offset = 0; for (int b = 0; b < batch; b++) { - const embed_context_input_t *input = &inputs[b]; + const ffwd_context_input_t *input = &inputs[b]; packed[b] = input->input; for (int s = 0; s < input->n_spans; s++) { h_starts[span_offset] = token_offset + input->spans[s].start; @@ -1812,8 +1799,8 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, if (launch_check() != 0) goto cleanup; ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out, - (size_t)total_spans * c->hidden_size * sizeof(float), - cudaMemcpyDeviceToHost, ctx->stream); + (size_t)total_spans * c->hidden_size * sizeof(float), cudaMemcpyDeviceToHost, + ctx->stream); if (ce == cudaSuccess) ce = cudaStreamSynchronize(ctx->stream); if (ce != cudaSuccess) { @@ -1829,22 +1816,22 @@ cleanup: return rc; } -int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, - const int *token_ids, - int n_tokens, - float *out_embedding) { - embed_input_t input = {token_ids, n_tokens}; - return embed_cuda_encode_batch(ctx, &input, 1, out_embedding); +int ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding) { + ffwd_input_t input = {token_ids, n_tokens}; + return ffwd_cuda_encode_batch(ctx, &input, 1, out_embedding); } -float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) { +float *ffwd_cuda_encode(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) { if (!ctx) return NULL; int hidden = ctx->config.hidden_size; float *out = (float *)malloc((size_t)hidden * sizeof(float)); if (!out) return NULL; - if (embed_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) { + if (ffwd_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) { free(out); return NULL; } @@ -1857,65 +1844,67 @@ float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_toke * The transformer forward and the 1_Dense projection run on the GPU; MaxSim * scoring stays on the host, where the grouped-GEMM scorer is far faster than a * device graph for this small op. So the encoders copy the projected token - * vectors back to host memory and the caller runs embed_late_maxsim_batch there. + * vectors back to host memory and the caller runs ffwd_late_maxsim_batch there. * ======================================================================== */ -struct embed_cuda_late_ctx { - embed_cuda_ctx_t *base; // device base model; does not own its CPU model - embed_late_model_t *cpu_late; // owns the CPU base model + projection host data - cuda_matrix_t projection; // device [token_dim, hidden] - float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand - size_t proj_cap; // capacity of proj_dev in floats +struct ffwd_cuda_late_ctx { + ffwd_cuda_ctx_t *base; // device base model; does not own its CPU model + ffwd_late_model_t *cpu_late; // owns the CPU base model + projection host data + cuda_matrix_t projection; // device [token_dim, hidden] + float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand + size_t proj_cap; // capacity of proj_dev in floats int token_dim; }; -embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir) { - embed_late_model_t *cpu_late = embed_late_model_load(model_dir); +ffwd_cuda_late_ctx_t *ffwd_cuda_late_load(const char *model_dir) { + ffwd_late_model_t *cpu_late = ffwd_late_model_load(model_dir); if (!cpu_late) return NULL; - embed_cuda_late_ctx_t *ctx = (embed_cuda_late_ctx_t *)calloc(1, sizeof(*ctx)); + ffwd_cuda_late_ctx_t *ctx = (ffwd_cuda_late_ctx_t *)calloc(1, sizeof(*ctx)); if (!ctx) { - embed_late_model_free(cpu_late); + ffwd_late_model_free(cpu_late); return NULL; } ctx->cpu_late = cpu_late; - ctx->token_dim = embed_late_model_token_dim(cpu_late); + ctx->token_dim = ffwd_late_model_token_dim(cpu_late); // Upload the base transformer weights; the CPU late model keeps ownership. - ctx->base = cuda_ctx_from_model(embed_late_model_base(cpu_late), 0); + ctx->base = cuda_ctx_from_model(ffwd_late_model_base(cpu_late), 0); if (!ctx->base) { - embed_cuda_late_free(ctx); + ffwd_cuda_late_free(ctx); return NULL; } // Upload the 1_Dense projection [token_dim, hidden]. - const embed_weight_ref_t *proj = embed_late_model_projection(cpu_late); + const ffwd_weight_ref_t *proj = ffwd_late_model_projection(cpu_late); if (!proj || ctx->token_dim <= 0 || load_matrix(&ctx->projection, proj, ctx->token_dim, ctx->base->config.hidden_size) != 0) { - embed_cuda_late_free(ctx); + ffwd_cuda_late_free(ctx); return NULL; } return ctx; } -void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx) { +void ffwd_cuda_late_free(ffwd_cuda_late_ctx_t *ctx) { if (!ctx) return; cuda_matrix_free(&ctx->projection); cudaFree(ctx->proj_dev); - embed_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model - embed_late_model_free(ctx->cpu_late); // frees the CPU base model + projection + ffwd_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model + ffwd_late_model_free(ctx->cpu_late); // frees the CPU base model + projection free(ctx); } -const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx) { +const ffwd_config_t *ffwd_cuda_late_config(const ffwd_cuda_late_ctx_t *ctx) { return ctx ? &ctx->base->config : NULL; } -int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } +int ffwd_cuda_late_token_dim(const ffwd_cuda_late_ctx_t *ctx) { + return ctx ? ctx->token_dim : 0; +} -static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) { +static int ensure_late_proj(ffwd_cuda_late_ctx_t *ctx, int total) { size_t need = (size_t)total * (size_t)ctx->token_dim; if (need <= ctx->proj_cap) return 0; @@ -1932,13 +1921,13 @@ static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) { * [total, token_dim] F32 result to a freshly malloc'd host buffer (caller frees). * On success *proj_host and *total_out are set; base->offsets_host holds the * packed per-document token offsets. */ -static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx, - const embed_input_t *inputs, +static int cuda_late_forward_project(ffwd_cuda_late_ctx_t *ctx, + const ffwd_input_t *inputs, int n_docs, float **proj_host, int *total_out) { - embed_cuda_ctx_t *base = ctx->base; - const embed_config_t *c = &base->config; + ffwd_cuda_ctx_t *base = ctx->base; + const ffwd_config_t *c = &base->config; if (cuda_forward_batch(base, inputs, n_docs) != 0) return -1; int total = base->offsets_host[n_docs]; @@ -1965,14 +1954,14 @@ static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx, return 0; } -int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors) { +int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { if (!ctx || !token_ids || n_tokens <= 0 || !out_vectors || ctx->token_dim <= 0) return -1; - embed_input_t input = {token_ids, n_tokens}; + ffwd_input_t input = {token_ids, n_tokens}; float *proj_host = NULL; int total = 0; if (cuda_late_forward_project(ctx, &input, 1, &proj_host, &total) != 0) @@ -1981,27 +1970,27 @@ int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx, free(proj_host); if (normalize) { for (int i = 0; i < total; i++) - if (embed_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0) + if (ffwd_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0) return -1; } return 0; } -int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - float *out_vectors, - int *out_offsets) { +int ffwd_cuda_late_encode_docs(ffwd_cuda_late_ctx_t *ctx, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets) { if (!ctx || !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors || !out_offsets || ctx->token_dim <= 0) return -1; int dim = ctx->token_dim; - embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); + ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); if (!inputs) return -1; @@ -2049,7 +2038,7 @@ int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx, if (normalize) { for (int i = 0; i < total_keep; i++) - if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + if (ffwd_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) return -1; } return 0; diff --git a/src/cuda.h b/libffwd/cuda.h index aad20ab..8be1dd4 100644 --- a/src/cuda.h +++ b/libffwd/cuda.h @@ -1,31 +1,31 @@ /* - * embed_cuda.h - CUDA/cuBLAS backend + * ffwd_cuda.h - CUDA/cuBLAS backend */ -#ifndef EMBED_CUDA_H -#define EMBED_CUDA_H +#ifndef FFWD_CUDA_H +#define FFWD_CUDA_H /* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying EMBED_API are exported + * -fvisibility=hidden, so only declarations carrying FFWD_API are exported * from the shared library; everything else stays internal. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif +#ifndef FFWD_API +# if defined(__GNUC__) +# define FFWD_API __attribute__((visibility("default"))) +# else +# define FFWD_API +# endif #endif -#include "embed.h" +#include "ffwd.h" -typedef struct embed_cuda_ctx embed_cuda_ctx_t; -typedef struct embed_cuda_late_ctx embed_cuda_late_ctx_t; +typedef struct ffwd_cuda_ctx ffwd_cuda_ctx_t; +typedef struct ffwd_cuda_late_ctx ffwd_cuda_late_ctx_t; -EMBED_API int embed_cuda_set_fast_gemm(const char *mode); -EMBED_API int embed_cuda_set_weights_bf16(int on); +FFWD_API int ffwd_cuda_set_fast_gemm(const char *mode); +FFWD_API int ffwd_cuda_set_weights_bf16(int on); -EMBED_API embed_cuda_ctx_t *embed_cuda_load(const char *model_dir); -EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx); +FFWD_API ffwd_cuda_ctx_t *ffwd_cuda_load(const char *model_dir); +FFWD_API void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx); /* * Late-interaction CUDA path. Encoding (transformer forward + projection) runs @@ -33,13 +33,13 @@ EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx); * much faster than a device graph for this small op. The encoders therefore copy * the projected token vectors back to host memory. */ -EMBED_API embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir); -EMBED_API void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx); -EMBED_API const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx); -EMBED_API int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx); +FFWD_API ffwd_cuda_late_ctx_t *ffwd_cuda_late_load(const char *model_dir); +FFWD_API void ffwd_cuda_late_free(ffwd_cuda_late_ctx_t *ctx); +FFWD_API const ffwd_config_t *ffwd_cuda_late_config(const ffwd_cuda_late_ctx_t *ctx); +FFWD_API int ffwd_cuda_late_token_dim(const ffwd_cuda_late_ctx_t *ctx); /* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */ -EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx, +FFWD_API int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, @@ -49,9 +49,9 @@ EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx, * Encode n_docs documents in one packed forward and pack each document's kept * token vectors back-to-back into out_vectors[total_keep, token_dim] (host) in * document order. out_offsets[n_docs + 1] receives the prefix sum of kept counts - * (the layout embed_late_maxsim_batch consumes). Returns 0 on success. + * (the layout ffwd_late_maxsim_batch consumes). Returns 0 on success. */ -EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx, +FFWD_API int ffwd_cuda_late_encode_docs(ffwd_cuda_late_ctx_t *ctx, const int *const *doc_ids, const int *n_tokens, const int *const *keep, @@ -61,20 +61,20 @@ EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx, float *out_vectors, int *out_offsets); -EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx); +FFWD_API const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx); -EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx, +FFWD_API int ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding); -EMBED_API float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens); -EMBED_API int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx, - const embed_input_t *inputs, +FFWD_API float *ffwd_cuda_encode(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens); +FFWD_API int ffwd_cuda_encode_batch(ffwd_cuda_ctx_t *ctx, + const ffwd_input_t *inputs, int batch, float *out_embeddings); -EMBED_API int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx, - const embed_context_input_t *inputs, +FFWD_API int ffwd_cuda_encode_spans_batch(ffwd_cuda_ctx_t *ctx, + const ffwd_context_input_t *inputs, int batch, float *out_embeddings); -#endif /* EMBED_CUDA_H */ +#endif /* FFWD_CUDA_H */ diff --git a/src/dtype.h b/libffwd/dtype.h index 2a25cae..7a7a2ad 100644 --- a/src/dtype.h +++ b/libffwd/dtype.h @@ -1,5 +1,5 @@ -#ifndef EMBED_DTYPE_H -#define EMBED_DTYPE_H +#ifndef FFWD_DTYPE_H +#define FFWD_DTYPE_H #include <stdint.h> #include <string.h> @@ -8,8 +8,8 @@ * left shift reinterpreted as a float. Shared by every host-side caller (the * CPU loader, the safetensors reader); the CUDA backend keeps its own * __nv_bfloat16 device kernels, and kernels.c keeps the SIMD buffer variant - * embed_bf16_to_f32_buf for hot, contiguous conversions. */ -static inline float embed_bf16_to_f32(uint16_t bf16) { + * bf16_to_f32_buf for hot, contiguous conversions. */ +static inline float ffwd_bf16_to_f32(uint16_t bf16) { uint32_t bits = (uint32_t)bf16 << 16; float f; memcpy(&f, &bits, sizeof(f)); @@ -22,7 +22,7 @@ static inline float embed_bf16_to_f32(uint16_t bf16) { * subnormals, and inf/NaN. Used by the loaders to accept F16 safetensors * snapshots (e.g. the public multilingual-e5 weights); there is no fused F16 * kernel, so weights are widened to F32 once at load. */ -static inline float embed_f16_to_f32(uint16_t f16) { +static inline float ffwd_f16_to_f32(uint16_t f16) { uint32_t sign = (uint32_t)(f16 & 0x8000u) << 16; uint32_t exp = (f16 >> 10) & 0x1fu; uint32_t mant = f16 & 0x3ffu; @@ -52,4 +52,4 @@ static inline float embed_f16_to_f32(uint16_t f16) { return f; } -#endif /* EMBED_DTYPE_H */ +#endif /* FFWD_DTYPE_H */ diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h new file mode 100644 index 0000000..6f947b6 --- /dev/null +++ b/libffwd/ffwd.h @@ -0,0 +1,280 @@ +/* + * ffwd.h - ffwd API + */ + +#ifndef FFWD_H +#define FFWD_H + +/* Public-API export annotation. The libraries are built with + * -fvisibility=hidden, so only declarations carrying FFWD_API are exported + * from the shared library; everything else stays internal. */ +#ifndef FFWD_API +# if defined(__GNUC__) +# define FFWD_API __attribute__((visibility("default"))) +# else +# define FFWD_API +# endif +#endif + +#include <stddef.h> +#include <stdint.h> + +/* ======================================================================== + * Shared constants and family defaults + * ======================================================================== */ + +#define FFWD_VOCAB_SIZE 151936 +#define FFWD_HEAD_DIM 128 +#define FFWD_MAX_LAYERS 64 /* upper bound for stack arrays */ +#define FFWD_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ + +/* Late-interaction (ColBERT) special-token id fallbacks and the query pad + * length, used when a model's tokenizer does not define the tokens itself. */ +#define FFWD_LATE_QUERY_TOKENS 32 +#define FFWD_LATE_MASK_TOKEN_ID 151642 +#define FFWD_LATE_QUERY_PREFIX_ID 151669 +#define FFWD_LATE_DOCUMENT_PREFIX_ID 151670 + +/* ======================================================================== + * Model Configuration (populated from config.json at load time) + * ======================================================================== */ + +typedef enum { + FFWD_ATTENTION_BIDIRECTIONAL = 0, + FFWD_ATTENTION_CAUSAL = 1, +} ffwd_attention_mode_t; + +typedef enum { + FFWD_POOL_MEAN = 0, + FFWD_POOL_LAST_TOKEN = 1, + FFWD_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */ +} ffwd_pooling_mode_t; + +typedef enum { + FFWD_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (Qwen2/3, pplx-embed) */ + FFWD_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */ +} ffwd_family_t; + +/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf + * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or + * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */ +typedef enum { + FFWD_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */ + FFWD_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */ +} ffwd_act_t; + +typedef struct { + ffwd_family_t family; + ffwd_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */ + int hidden_size; + int n_layers; + int n_heads; + int n_kv_heads; + int head_dim; + int q_dim; /* n_heads * head_dim */ + int kv_dim; /* n_kv_heads * head_dim */ + int intermediate_size; + int vocab_size; + int max_position_embeddings; /* BERT learned position table size */ + int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */ + int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */ + float rms_norm_eps; + float layer_norm_eps; /* BERT LayerNorm epsilon */ + float rope_theta; + int qk_norm; + int qkv_bias; + ffwd_attention_mode_t attention_mode; + ffwd_pooling_mode_t pooling_mode; + int normalize_embeddings; + /* Qwen3-Embedding pools the last token, which is the tokenizer's + * <|endoftext|> suffix added by its post-processor - not the model's chat + * eos_token_id (<|im_end|>). Text frontends append it and resolve the id + * from the tokenizer exactly like the contextual separator + * (FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The + * model forward consumes token ids exactly and never appends. */ + int append_terminal_token; +} ffwd_config_t; + +typedef struct { + const int *ids; + int n_tokens; +} ffwd_input_t; + +typedef struct { + int start; + int n_tokens; +} ffwd_span_t; + +typedef struct { + ffwd_input_t input; + const ffwd_span_t *spans; + int n_spans; +} ffwd_context_input_t; + +/* ======================================================================== + * Engine API — the backend-selected inference handle + * + * This is the library's primary public interface and the entire supported + * surface of the shared library. Exactly one backend (CPU, Apple Silicon GPU, + * or NVIDIA GPU) is linked at build time and implements these functions; + * callers never branch on the backend. The CLI, the server, and any language + * binding use only these functions plus the math helpers below. + * + * A model loads in two phases so a GPU context can be created on the thread + * that will own it. ffwd_open runs on the caller's thread and loads a CPU + * model immediately; a GPU build only records what to load. ffwd_activate then + * runs on the inference thread and creates the GPU context (a no-op for CPU). + * Teardown mirrors this: ffwd_worker_free releases the GPU context from the + * inference thread, ffwd_free releases the rest. + * ======================================================================== */ + +/* Opaque inference handle; its layout is private to the linked backend. */ +typedef struct embed ffwd_t; + +/* Backend tuning, parsed from the command line. A backend reads the fields it + * supports and ignores the rest, so the front-ends parse them with no backend + * branch. */ +typedef struct { + int n_threads; /* CPU worker threads; <= 0 auto-detects */ + int gpu_quant_bits; /* GPU load-time weight quantization (0 or 8) */ + int gpu_quant_group_size; /* GPU quantization group size (> 0) */ + const char *gpu_gemm_mode; /* GPU GEMM compute mode, or NULL */ + const char *gpu_weight_dtype; /* GPU weight storage dtype, or NULL */ + double memory_utilization; /* GPU memory budget fraction; 0 = default */ +} ffwd_options_t; + +/* One late-interaction rerank: a query and N candidate documents, each given as + * token ids plus the kept-token indices. The vector dimension is the loaded + * late model's token_dim. */ +typedef struct { + const int *query_ids; + int query_n_tokens; + int query_n_keep; + const int *const *doc_ids; + const int *doc_n_tokens; + const int *const *doc_keep; + const int *doc_n_keep; + int n_docs; +} ffwd_rerank_input_t; + +/* Apply process-wide settings before any load (CPU thread count, GPU GEMM and + * weight settings, GPU option checks) and print the startup banner. Returns -1 and + * writes a reason into err on a rejected option value. */ +FFWD_API int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen); + +/* First load phase, on the caller's thread: load a CPU model now, or record what + * a GPU build will load in ffwd_activate. is_late selects a late-interaction + * model. NULL on failure with a reason in err. */ +FFWD_API ffwd_t *ffwd_open( + const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen); +/* Second load phase, on the inference thread: create the GPU context. CPU + * no-op. Returns -1 with a reason in err on failure. */ +FFWD_API int ffwd_activate(ffwd_t *e, char *err, size_t errlen); +/* Release the inference-thread-owned GPU context; CPU no-op. Idempotent. */ +FFWD_API void ffwd_worker_free(ffwd_t *e); +/* Release CPU state and the handle; call after ffwd_worker_free. */ +FFWD_API void ffwd_free(ffwd_t *e); + +/* Available once the model is loaded (after open for CPU, after activate for + * GPU); config is NULL until then. */ +FFWD_API const ffwd_config_t *ffwd_config(const ffwd_t *e); +/* Late projection dimension, or 0 for a standard model. */ +FFWD_API int ffwd_token_dim(const ffwd_t *e); +/* Whether the backend pads batches to a common length rather than packing them. */ +FFWD_API int ffwd_uses_dense_batches(const ffwd_t *e); + +/* Encode a packed batch into out[batch, hidden_size]. Returns 0 on success. */ +FFWD_API int +ffwd_encode_batch(ffwd_t *e, const ffwd_input_t *inputs, int batch, float *out); +/* Encode a contextual document batch, pooling each selected span in order. */ +FFWD_API int ffwd_encode_spans_batch(ffwd_t *e, + const ffwd_context_input_t *inputs, + int batch, + float *out); +/* Score one query against its candidate documents into scores[n_docs]. */ +FFWD_API int ffwd_rerank(ffwd_t *e, const ffwd_rerank_input_t *in, float *scores); +/* Scratch memory the model currently holds, for diagnostics; 0 on GPU. */ +FFWD_API size_t ffwd_scratch_nbytes(const ffwd_t *e); + +/* Capability label for --build-info and the startup banner: "CPU", "Apple + * Silicon GPU", or "NVIDIA GPU". A function, not a macro, so a shared object + * does not bake in one backend's label and stays correct across a rebuild. */ +FFWD_API const char *ffwd_capability(void); + +/* First-arrival micro-batch wait the backend prefers, in microseconds. */ +FFWD_API int ffwd_default_batch_wait_us(void); +/* Reject a model set that would not fit host memory (GPU backends); 0 otherwise. On + * rejection returns -1 with a reason in err. paths are the model directories. */ +FFWD_API int ffwd_preflight( + const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen); + +/* + * L2-normalize vec[dim] in place. + * + * Returns 0 on success. Returns -1 for invalid arguments or a zero-length + * vector, which cannot be normalized. + */ +FFWD_API int ffwd_l2_normalize(float *vec, int dim); + +/* + * Cosine similarity between two vectors. + * Returns 0 for invalid arguments or if either vector has zero length. + */ +FFWD_API float ffwd_cosine_similarity(const float *a, const float *b, int dim); + +/* ======================================================================== + * Tokenization — text into the token ids a model expects + * + * The tokenizer is backend-independent (text -> ids is identical on CPU or + * GPU), so it is its own handle, separate from ffwd_t. ffwd_tok_open picks the + * tokenizer by the files in model_dir (WordPiece vocab.txt, SentencePiece + * .model, or byte-level vocab.json) and resolves the model's special-token ids. + * ffwd_tokenize then applies that model's layout: BERT and XLM-R wrap the ids + * with [CLS]..[SEP] (<s>..</s>), Qwen appends its terminal token. + * ======================================================================== */ + +/* Opaque tokenizer handle; owns the tokenizer and its resolved special tokens. */ +typedef struct ffwd_tok ffwd_tok_t; + +/* Late-interaction tokens: ids, plus the document positions to score. For a + * query, keep is NULL (every position is kept) and n_keep == n_tokens. */ +typedef struct { + int *ids; + int n_tokens; + int *keep; + int n_keep; +} ffwd_late_tokens_t; + +/* Load the tokenizer for the model in model_dir and resolve its special-token + * ids. is_late additionally resolves the late-interaction [Q]/[D]/[MASK] ids. + * NULL on failure with a reason in err. */ +FFWD_API ffwd_tok_t * +ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen); +FFWD_API void ffwd_tok_free(ffwd_tok_t *t); + +/* Tokenize text for this model: applies the model's special-token layout and + * optionally prepends a query instruction. Returns a malloc'd id array (caller + * frees) and sets *n_out. NULL on error. */ +FFWD_API int * +ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out); + +/* Late-interaction tokenization: [Q]/[D] prefix, [MASK]-pad queries to a fixed + * length, and mark the document positions to score. Returns 0 on success. */ +FFWD_API int +ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out); +FFWD_API void ffwd_late_tokens_free(ffwd_late_tokens_t *out); + +/* The contextual chunk-separator id (the tokenizer's <|endoftext|>), for + * frontends that split a document into chunks. */ +FFWD_API int ffwd_tok_context_separator_id(const ffwd_tok_t *t); + +/* Whether ffwd_tokenize appends the model's terminal token (<|endoftext|>) for + * last-token pooling. Off until set; the caller sets it from the loaded model's + * config (ffwd_config(...)->append_terminal_token), since it is model-specific + * and only known after the backend is open. */ +FFWD_API void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on); + +/* Verbose level: 0=quiet, 1=info, 2=debug */ +FFWD_API extern int ffwd_verbose; + +#endif /* FFWD_H */ diff --git a/src/forward.c b/libffwd/forward.c index 28243af..e8a5cdf 100644 --- a/src/forward.c +++ b/libffwd/forward.c @@ -1,4 +1,4 @@ -#include "embed.h" +#include "ffwd.h" #include "model_internal.h" #include "dtype.h" #include "kernels.h" @@ -11,8 +11,8 @@ /* Below these packed-sequence lengths the fused BF16 row kernels beat widening * the weight to F32 and dispatching through BLAS; above them, widen once. */ -#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16 -#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4 +#define FFWD_BF16_QKV_FUSE_MAX_SEQ 16 +#define FFWD_BF16_PAIR_FUSE_MAX_SEQ 4 /* ======================================================================== * Linear projections (weight-dtype dispatch) and workspace scratch growth @@ -20,10 +20,10 @@ static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { for (int i = 0; i < n; i++) - dst[i] = embed_bf16_to_f32(src[i]); + dst[i] = ffwd_bf16_to_f32(src[i]); } -static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) { +static void copy_weight_row(float *dst, const ffwd_weight_ref_t *w, size_t row, int row_width) { if (w->dtype == DTYPE_F32) { const float *src = (const float *)w->data + row * (size_t)row_width; memcpy(dst, src, (size_t)row_width * sizeof(float)); @@ -33,7 +33,7 @@ static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, } } -static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) { +static int ensure_bf16_widen(ffwd_workspace_t *ws, size_t count) { if (ws->bf16_widen_count >= count) return 0; float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float)); @@ -44,15 +44,15 @@ static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) { return 0; } -void linear_nobias_weight(embed_workspace_t *ws, +void linear_nobias_weight(ffwd_workspace_t *ws, float *y, const float *x, - const embed_weight_ref_t *w, + const ffwd_weight_ref_t *w, int seq_len, int in_dim, int out_dim) { if (w->dtype == DTYPE_F32) { - embed_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); + linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); return; } if (seq_len > 16) { @@ -60,31 +60,30 @@ void linear_nobias_weight(embed_workspace_t *ws, * BLAS-backed F32 path; below that the fused BF16 row kernels win. */ size_t count = (size_t)out_dim * (size_t)in_dim; if (ensure_bf16_widen(ws, count) == 0) { - embed_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count); - embed_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); + bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count); + linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); return; } } - embed_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim); + linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim); } -static void linear_qkv_weight(embed_workspace_t *ws, +static void linear_qkv_weight(ffwd_workspace_t *ws, float *q, float *k, float *v, const float *x, - const embed_weight_ref_t *wq, - const embed_weight_ref_t *wk, - const embed_weight_ref_t *wv, + const ffwd_weight_ref_t *wq, + const ffwd_weight_ref_t *wk, + const ffwd_weight_ref_t *wv, int seq_len, int in_dim, int q_dim, int kv_dim) { if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 && - seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) { - embed_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, - (const uint16_t *)wk->data, (const uint16_t *)wv->data, - seq_len, in_dim, q_dim, kv_dim); + seq_len <= FFWD_BF16_QKV_FUSE_MAX_SEQ) { + linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, (const uint16_t *)wk->data, + (const uint16_t *)wv->data, seq_len, in_dim, q_dim, kv_dim); return; } @@ -103,20 +102,20 @@ static void add_bias_rows(float *x, const float *bias, int rows, int dim) { } } -static void linear_pair_weight(embed_workspace_t *ws, +static void linear_pair_weight(ffwd_workspace_t *ws, float *a, float *b, const float *x, - const embed_weight_ref_t *wa, - const embed_weight_ref_t *wb, + const ffwd_weight_ref_t *wa, + const ffwd_weight_ref_t *wb, int seq_len, int in_dim, int a_dim, int b_dim) { if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && - seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) { - embed_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, - (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim); + seq_len <= FFWD_BF16_PAIR_FUSE_MAX_SEQ) { + linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, (const uint16_t *)wb->data, + seq_len, in_dim, a_dim, b_dim); return; } @@ -128,11 +127,18 @@ static void linear_pair_weight(embed_workspace_t *ws, * Packed forward pass * * The workspace-growth helpers these call (ensure_buffers, ensure_rope_cache, - * ensure_attention_scores, ensure_offsets) live in model.c beside the workspace - * lifecycle and embed_workspace_nbytes, whose size accounting mirrors them. + * ensure_attention_scores, ensure_offsets) live in model_blas.c beside the + * workspace lifecycle and ffwd_workspace_nbytes, whose size accounting mirrors + * them. * ======================================================================== */ -int build_offsets(const embed_input_t *inputs, +static ffwd_attention_fn attention_for_config(const ffwd_config_t *cfg) { + return cfg->attention_mode == FFWD_ATTENTION_CAUSAL + ? causal_gqa_attention_packed_with_scratch + : bidirectional_gqa_attention_packed_with_scratch; +} + +int build_offsets(const ffwd_input_t *inputs, int batch, int require_ids, int *offsets, @@ -159,9 +165,9 @@ int build_offsets(const embed_input_t *inputs, return 0; } -static int forward_packed_slice_inplace(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, +static int forward_packed_slice_inplace(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, int batch, const int *offsets, int total_seq, @@ -170,12 +176,12 @@ static int forward_packed_slice_inplace(const embed_model_t *model, int layer_start, int layer_end, int apply_final_norm) { - if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || - total_seq <= 0 || max_seq <= 0) + if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || total_seq <= 0 || + max_seq <= 0) return -1; - const embed_config_t *cfg = &model->config; - const embed_weights_t *w = &model->weights; + const ffwd_config_t *cfg = &model->config; + const ffwd_weights_t *w = &model->weights; if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers || layer_start < model->layer_start || layer_end > model->layer_end || (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) || @@ -219,7 +225,7 @@ static int forward_packed_slice_inplace(const embed_model_t *model, for (int i = 0; i < inputs[b].n_tokens; i++) { int id = inputs[b].ids[i]; if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, + fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, i); return -1; } @@ -236,53 +242,54 @@ static int forward_packed_slice_inplace(const embed_model_t *model, /* 2. Transformer layers. Linear/MLP ops run over all packed tokens; * attention is explicitly block-diagonal by sequence offsets. */ for (int layer = layer_start; layer < layer_end; layer++) { - const embed_layer_t *l = &w->layers[layer]; + const ffwd_layer_t *l = &w->layers[layer]; - embed_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); + rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); - linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, - hidden, q_dim, kv_dim); + linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, hidden, + q_dim, kv_dim); add_bias_rows(q_buf, l->q_bias, total_seq, q_dim); add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim); add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim); if (cfg->qk_norm) { - embed_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); - embed_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); + rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); + rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); } for (int b = 0; b < batch; b++) { int start = offsets[b]; int len = offsets[b + 1] - start; - embed_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, - head_dim); - embed_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, - n_kv_heads, head_dim); + apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, + head_dim); + apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, n_kv_heads, + head_dim); } - model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, - head_dim, scale, ws->attn_scores, ws->attn_scores_bytes); + ffwd_attention_fn attention = attention_for_config(cfg); + attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, head_dim, scale, + ws->attn_scores, ws->attn_scores_bytes); linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); + add_inplace(x, proj_out, total_seq * hidden); - embed_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); + rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq, hidden, inter, inter); - embed_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); + silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); + add_inplace(x, proj_out, total_seq * hidden); } if (apply_final_norm) - embed_rms_norm(x, x, w->norm, total_seq, hidden, eps); + rms_norm(x, x, w->norm, total_seq, hidden, eps); return 0; } -int forward_packed_inplace(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, +int forward_packed_inplace(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, int batch, const int *offsets, int total_seq, @@ -299,19 +306,19 @@ int forward_packed_inplace(const embed_model_t *model, * cfg->position_id_offset (0 for BERT, pad_id + 1 for RoBERTa/XLM-R). Leaves the * final hidden states in ws->x; pooling reads them as-is because the last * layer's LayerNorm is already the model's final normalization. */ -static int forward_packed_bert(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, +static int forward_packed_bert(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, int batch, const int *offsets, int total_seq, int max_seq) { - if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || - total_seq <= 0 || max_seq <= 0) + if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || total_seq <= 0 || + max_seq <= 0) return -1; - const embed_config_t *cfg = &model->config; - const embed_weights_t *w = &model->weights; + const ffwd_config_t *cfg = &model->config; + const ffwd_weights_t *w = &model->weights; int hidden = cfg->hidden_size; int n_heads = cfg->n_heads; int head_dim = cfg->head_dim; @@ -319,11 +326,12 @@ static int forward_packed_bert(const embed_model_t *model, float eps = cfg->layer_norm_eps; if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || - !w->embed_ln_w || !w->embed_ln_b) + !w->ffwd_ln_w || !w->ffwd_ln_b) return -1; if (max_seq > cfg->max_position_embeddings - cfg->position_id_offset) { fprintf(stderr, - "embed: sequence length %d plus position offset %d exceeds max_position_embeddings %d\n", + "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings " + "%d\n", max_seq, cfg->position_id_offset, cfg->max_position_embeddings); return -1; } @@ -342,22 +350,22 @@ static int forward_packed_bert(const embed_model_t *model, for (int i = 0; i < inputs[b].n_tokens; i++) { int id = inputs[b].ids[i]; if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, i); + fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, i); return -1; } float *row = x + (size_t)(start + i) * hidden; copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden); int pos = cfg->position_id_offset + i; - embed_add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); - embed_add_inplace(row, w->token_type_embeddings, hidden); + add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); + add_inplace(row, w->token_type_embeddings, hidden); } } - embed_layer_norm(x, x, w->embed_ln_w, w->embed_ln_b, total_seq, hidden, eps); + layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, total_seq, hidden, eps); float scale = 1.0f / sqrtf((float)head_dim); for (int layer = 0; layer < cfg->n_layers; layer++) { - const embed_layer_t *l = &w->layers[layer]; + const ffwd_layer_t *l = &w->layers[layer]; /* Self-attention (post-norm): projections read x directly. */ linear_qkv_weight(ws, q_buf, k_buf, v_buf, x, &l->wq, &l->wk, &l->wv, total_seq, hidden, @@ -366,35 +374,36 @@ static int forward_packed_bert(const embed_model_t *model, add_bias_rows(k_buf, l->k_bias, total_seq, hidden); add_bias_rows(v_buf, l->v_bias, total_seq, hidden); - model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim, - scale, ws->attn_scores, ws->attn_scores_bytes); + ffwd_attention_fn attention = attention_for_config(cfg); + attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim, scale, + ws->attn_scores, ws->attn_scores_bytes); linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, hidden, hidden); add_bias_rows(proj_out, l->o_bias, total_seq, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); - embed_layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps); + add_inplace(x, proj_out, total_seq * hidden); + layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps); /* Feed-forward (post-norm): dense -> GeLU -> dense. */ linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter); add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter); - if (cfg->ffn_act == EMBED_ACT_GELU_TANH) - embed_gelu_tanh_inplace(ffn_up, total_seq * inter); + if (cfg->ffn_act == FFWD_ACT_GELU_TANH) + gelu_tanh_inplace(ffn_up, total_seq * inter); else - embed_gelu_inplace(ffn_up, total_seq * inter); + gelu_inplace(ffn_up, total_seq * inter); linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden); add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden); - embed_add_inplace(x, proj_out, total_seq * hidden); - embed_layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps); + add_inplace(x, proj_out, total_seq * hidden); + layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps); } return 0; } -static int pool_embeddings(const embed_model_t *model, - const embed_workspace_t *ws, +static int pool_embeddings(const ffwd_model_t *model, + const ffwd_workspace_t *ws, const int *offsets, int batch, float *out_embeddings) { - const embed_config_t *cfg = &model->config; + const ffwd_config_t *cfg = &model->config; int hidden = cfg->hidden_size; float eps = cfg->rms_norm_eps; /* Qwen pools pre-final-norm states and applies the final RMSNorm here; BERT @@ -410,10 +419,10 @@ static int pool_embeddings(const embed_model_t *model, int len = end - start; float *emb = out_embeddings + (size_t)b * hidden; - if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) { + if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) { /* Single-token pooling: CLS takes the first token, last-token the * final one. */ - int idx = cfg->pooling_mode == EMBED_POOL_CLS ? start : end - 1; + int idx = cfg->pooling_mode == FFWD_POOL_CLS ? start : end - 1; const float *row = x + (size_t)idx * hidden; if (norm_weight) { float sum_sq = 0.0f; @@ -446,7 +455,7 @@ static int pool_embeddings(const embed_model_t *model, emb[d] *= inv_len; } - if (cfg->normalize_embeddings && embed_l2_normalize(emb, hidden) != 0) + if (cfg->normalize_embeddings && ffwd_l2_normalize(emb, hidden) != 0) return -1; } return 0; @@ -456,11 +465,11 @@ static int pool_embeddings(const embed_model_t *model, * Public forward/embed APIs * ======================================================================== */ -int embed_model_forward_into(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_states) { +int ffwd_model_forward_into(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_states) { if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) return -1; @@ -469,7 +478,7 @@ int embed_model_forward_into(const embed_model_t *model, mul_size(count, sizeof(float), &bytes) != 0) return -1; - embed_input_t input = {token_ids, n_tokens}; + ffwd_input_t input = {token_ids, n_tokens}; int offsets[2] = {0, n_tokens}; if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0) return -1; @@ -478,12 +487,12 @@ int embed_model_forward_into(const embed_model_t *model, return 0; } -int embed_pool_spans(const embed_config_t *cfg, - const float *states, - int n_tokens, - const embed_span_t *spans, - int n_spans, - float *out_embeddings) { +int ffwd_pool_spans(const ffwd_config_t *cfg, + const float *states, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings) { if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings) return -1; @@ -514,19 +523,19 @@ int embed_pool_spans(const embed_config_t *cfg, return 0; } -int embed_model_encode_spans_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_context_input_t *inputs, - int batch, - float *out_embeddings) { +int ffwd_model_encode_spans_batch(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; if (ensure_offsets(ws, batch) != 0) return -1; - if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t)) + if ((size_t)batch > SIZE_MAX / sizeof(ffwd_input_t)) return -1; - embed_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); + ffwd_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs)); if (!packed_inputs) return -1; int *offsets = ws->offsets; @@ -534,7 +543,7 @@ int embed_model_encode_spans_batch(const embed_model_t *model, int max_seq = 0; int total_spans = 0; for (int b = 0; b < batch; b++) { - const embed_context_input_t *input = &inputs[b]; + const ffwd_context_input_t *input = &inputs[b]; int n_tokens = input->input.n_tokens; if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) { @@ -568,35 +577,35 @@ int embed_model_encode_spans_batch(const embed_model_t *model, int span_offset = 0; int hidden = model->config.hidden_size; for (int b = 0; b < batch; b++) { - const embed_context_input_t *input = &inputs[b]; - if (embed_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden, - input->input.n_tokens, input->spans, input->n_spans, - out_embeddings + (size_t)span_offset * hidden) != 0) + const ffwd_context_input_t *input = &inputs[b]; + if (ffwd_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden, + input->input.n_tokens, input->spans, input->n_spans, + out_embeddings + (size_t)span_offset * hidden) != 0) return -1; span_offset += input->n_spans; } return 0; } -int embed_model_encode_spans(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - const embed_span_t *spans, - int n_spans, - float *out_embeddings) { - embed_context_input_t input = { +int ffwd_model_encode_spans(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings) { + ffwd_context_input_t input = { .input = {token_ids, n_tokens}, .spans = spans, .n_spans = n_spans, }; - return embed_model_encode_spans_batch(model, ws, &input, 1, out_embeddings); + return ffwd_model_encode_spans_batch(model, ws, &input, 1, out_embeddings); } -float *embed_model_forward(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens) { +float *ffwd_model_forward(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens) { if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; @@ -607,18 +616,18 @@ float *embed_model_forward(const embed_model_t *model, if (!out) return NULL; - if (embed_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) { + if (ffwd_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) { free(out); return NULL; } return out; } -int embed_model_encode_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, - int batch, - float *out_embeddings) { +int ffwd_model_encode_batch(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, + int batch, + float *out_embeddings) { if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1; @@ -631,7 +640,7 @@ int embed_model_encode_batch(const embed_model_t *model, if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0) return -1; - int rc = model->config.family == EMBED_FAMILY_BERT + int rc = model->config.family == FFWD_FAMILY_BERT ? forward_packed_bert(model, ws, inputs, batch, offsets, total_seq, max_seq) : forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0); if (rc == 0) @@ -640,11 +649,11 @@ int embed_model_encode_batch(const embed_model_t *model, return rc; } -int embed_pool_batch(const embed_config_t *cfg, - const float *states, - const int *seq_lengths, - int batch, - float *out_embeddings) { +int ffwd_pool_batch(const ffwd_config_t *cfg, + const float *states, + const int *seq_lengths, + int batch, + float *out_embeddings) { if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0) return -1; @@ -655,10 +664,10 @@ int embed_pool_batch(const embed_config_t *cfg, return -1; float *emb = out_embeddings + (size_t)b * cfg->hidden_size; - if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) { + if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) { /* CLS takes the first token, last-token the final one; these states * already carry the model's final RMSNorm. */ - int idx = cfg->pooling_mode == EMBED_POOL_CLS ? offset : offset + len - 1; + int idx = cfg->pooling_mode == FFWD_POOL_CLS ? offset : offset + len - 1; const float *row = states + (size_t)idx * cfg->hidden_size; memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float)); } else { @@ -672,26 +681,26 @@ int embed_pool_batch(const embed_config_t *cfg, for (int d = 0; d < cfg->hidden_size; d++) emb[d] *= inv_len; } - if (cfg->normalize_embeddings && embed_l2_normalize(emb, cfg->hidden_size) != 0) + if (cfg->normalize_embeddings && ffwd_l2_normalize(emb, cfg->hidden_size) != 0) return -1; offset += len; } return 0; } -int embed_model_encode_into(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding) { - embed_input_t input = {token_ids, n_tokens}; - return embed_model_encode_batch(model, ws, &input, 1, out_embedding); +int ffwd_model_encode_into(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding) { + ffwd_input_t input = {token_ids, n_tokens}; + return ffwd_model_encode_batch(model, ws, &input, 1, out_embedding); } -float *embed_model_encode(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens) { +float *ffwd_model_encode(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens) { if (!model || !ws || !token_ids || n_tokens <= 0) return NULL; @@ -702,7 +711,7 @@ float *embed_model_encode(const embed_model_t *model, if (!emb) return NULL; - if (embed_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) { + if (ffwd_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) { free(emb); return NULL; } diff --git a/src/kernels_gemm.c b/libffwd/gemm.c index aa8dadf..bb54fb9 100644 --- a/src/kernels_gemm.c +++ b/libffwd/gemm.c @@ -1,5 +1,10 @@ +/* + * gemm.c - general Matrix Multiplication + */ + #include "kernels.h" -#include "kernels_impl.h" +#include "impl.h" +#include "threadpool.h" #include <math.h> #include <stdlib.h> @@ -7,22 +12,22 @@ #if (defined(__AVX512F__) || defined(__AVX2__)) && \ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) -#include <immintrin.h> +# include <immintrin.h> #endif #if defined(__ARM_NEON) || defined(__ARM_NEON__) -#include <arm_neon.h> +# include <arm_neon.h> #endif #ifdef USE_BLAS -#ifdef __APPLE__ -#include <Accelerate/Accelerate.h> -#else -#include <cblas.h> -#endif +# ifdef __APPLE__ +# include <Accelerate/Accelerate.h> +# else +# include <cblas.h> +# endif #endif #ifndef M_PI -#define M_PI 3.14159265358979323846 +# define M_PI 3.14159265358979323846 #endif /* Matrix and vector ops: GEMM, the dtype-dispatch linear kernels, and the @@ -32,7 +37,7 @@ * Basic Element-wise Operations * ======================================================================== */ -void embed_add_inplace(float *a, const float *b, int n) { +void add_inplace(float *a, const float *b, int n) { for (int i = 0; i < n; i++) a[i] += b[i]; } @@ -41,7 +46,7 @@ void embed_add_inplace(float *a, const float *b, int n) { * Matrix Operations * ======================================================================== */ -void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { +void matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { #ifdef USE_BLAS cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); #else @@ -57,17 +62,11 @@ void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int #endif } -void embed_linear(float *y, - const float *x, - const float *W, - const float *b, - int seq_len, - int in_dim, - int out_dim) { +void linear( + float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) { #ifdef USE_BLAS if (seq_len == 1) { - cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, - 1); + cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1); } else { cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x, in_dim, W, in_dim, 0.0f, y, out_dim); @@ -95,13 +94,12 @@ void embed_linear(float *y, #endif } -void embed_linear_nobias( - float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { - embed_linear(y, x, W, NULL, seq_len, in_dim, out_dim); +void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { + linear(y, x, W, NULL, seq_len, in_dim, out_dim); } /* Convert bf16 buffer to f32 buffer */ -void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { +void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { #if defined(__ARM_NEON) || defined(__ARM_NEON__) size_t i = 0; uint32_t *d = (uint32_t *)(void *)dst; @@ -127,7 +125,7 @@ void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { */ static void bf16_matvec_fused( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { - embed_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); + bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); } /* Threaded matvec: split output rows across threads */ @@ -156,7 +154,7 @@ static void matvec_worker(int tid, int n_threads, void *arg) { static void bf16_matvec_threaded( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { - if (embed_num_threads() <= 1) { + if (tp_num_threads() <= 1) { bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim); return; } @@ -207,7 +205,7 @@ static void bf16_linear_rows(float *y, .out_dim = out_dim, }; - if (embed_num_threads() > 1 && seq_len >= 2) + if (tp_num_threads() > 1 && seq_len >= 2) parallel_for(bf16_linear_rows_worker, &task); else bf16_linear_rows_worker(0, 1, &task); @@ -272,19 +270,19 @@ static void pair_matvec_worker(int tid, int n_threads, void *arg) { } } -void embed_linear_nobias_bf16_pair(float *a, - float *b, - const float *x, - const uint16_t *Wa_bf16, - const uint16_t *Wb_bf16, - int seq_len, - int in_dim, - int a_dim, - int b_dim) { +void linear_nobias_bf16_pair(float *a, + float *b, + const float *x, + const uint16_t *Wa_bf16, + const uint16_t *Wb_bf16, + int seq_len, + int in_dim, + int a_dim, + int b_dim) { if (seq_len <= 0) return; - if (embed_num_threads() <= 1) { + if (tp_num_threads() <= 1) { for (int s = 0; s < seq_len; s++) { const float *x_row = x + (size_t)s * in_dim; bf16_matvec_fused(a + (size_t)s * a_dim, x_row, Wa_bf16, NULL, in_dim, a_dim); @@ -380,21 +378,21 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) { } } -void embed_linear_nobias_bf16_qkv(float *q, - float *k, - float *v, - const float *x, - const uint16_t *Wq_bf16, - const uint16_t *Wk_bf16, - const uint16_t *Wv_bf16, - int seq_len, - int in_dim, - int q_dim, - int kv_dim) { +void linear_nobias_bf16_qkv(float *q, + float *k, + float *v, + const float *x, + const uint16_t *Wq_bf16, + const uint16_t *Wk_bf16, + const uint16_t *Wv_bf16, + int seq_len, + int in_dim, + int q_dim, + int kv_dim) { if (seq_len <= 0) return; - if (embed_num_threads() <= 1) { + if (tp_num_threads() <= 1) { for (int s = 0; s < seq_len; s++) { const float *x_row = x + (size_t)s * in_dim; bf16_matvec_fused(q + (size_t)s * q_dim, x_row, Wq_bf16, NULL, in_dim, q_dim); @@ -421,14 +419,13 @@ void embed_linear_nobias_bf16_qkv(float *q, parallel_for(qkv_matvec_worker, &task); } -void embed_linear_nobias_bf16( +void linear_nobias_bf16( float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) { if (seq_len == 1) { bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); return; } /* Callers route longer sequences through an F32-widened weight matrix in - * caller-owned scratch (see embed.c); this per-row path stays correct for - * any length. */ + * caller-owned scratch; this per-row path stays correct for any length. */ bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim); } diff --git a/src/kernels_generic.c b/libffwd/generic.c index 7e85298..8798107 100644 --- a/src/kernels_generic.c +++ b/libffwd/generic.c @@ -1,12 +1,12 @@ /* - * embed_kernels_generic.c - architecture-generic hot kernels + * generic.c - architecture-generic hot kernels / non-SIMD fallback */ -#include "kernels_impl.h" +#include "impl.h" #include <string.h> -void embed_bf16_matvec_fused_generic( +void bf16_matvec_fused_generic( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { for (int o = 0; o < out_dim; o++) { const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; @@ -21,24 +21,24 @@ void embed_bf16_matvec_fused_generic( } } -float embed_dot_f32_generic(const float *a, const float *b, int n) { +float dot_f32_generic(const float *a, const float *b, int n) { float sum = 0.0f; for (int i = 0; i < n; i++) sum += a[i] * b[i]; return sum; } -void embed_vec_scale_inplace_generic(float *dst, float scale, int n) { +void vec_scale_inplace_generic(float *dst, float scale, int n) { for (int i = 0; i < n; i++) dst[i] *= scale; } -void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { +void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { for (int i = 0; i < n; i++) dst[i] += alpha * src[i]; } -void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n) { +void vec_scale_add_generic(float *dst, const float *src, float correction, int n) { for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i]; } diff --git a/libffwd/impl.h b/libffwd/impl.h new file mode 100644 index 0000000..39baa40 --- /dev/null +++ b/libffwd/impl.h @@ -0,0 +1,279 @@ +/* + * impl.h - internal architecture dispatch for hot kernels + * + * The small vector primitives (dot product, scale, axpy, scale-add, scaled + * multiply) are defined here as static inline so they fold directly into the + * caller's loop: a row of RMSNorm or attention issues no call and the compiler + * keeps the row in registers across the reduction and the write-back. The + * generic scalar versions stay out of line in generic.c as a reference for the + * kernel tests and as the fallback on platforms with no SIMD path. + * + * bf16_matvec_fused is large (a full matrix-vector product) and is called once + * per output tile rather than inside a tight loop, so it stays out of line. + */ + +#ifndef KERNELS_IMPL_H +#define KERNELS_IMPL_H + +#include <stdint.h> + +/* Scalar references (generic.c): the kernel tests compare the SIMD results + * against these, and the no-SIMD build dispatches to them directly. */ +void bf16_matvec_fused_generic( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); +float dot_f32_generic(const float *a, const float *b, int n); +void vec_scale_inplace_generic(float *dst, float scale, int n); +void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); +void vec_scale_add_generic(float *dst, const float *src, float correction, int n); + +#ifdef __ARM_NEON + +# include <arm_neon.h> + +void bf16_matvec_fused_neon( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); +# define bf16_matvec_fused_impl bf16_matvec_fused_neon + +static inline float dot_f32_impl(const float *a, const float *b, int n) { + int i = 0; + float32x4_t acc0 = vdupq_n_f32(0.0f); + float32x4_t acc1 = vdupq_n_f32(0.0f); + for (; i + 8 <= n; i += 8) { + float32x4_t a0 = vld1q_f32(a + i); + float32x4_t b0 = vld1q_f32(b + i); + float32x4_t a1 = vld1q_f32(a + i + 4); + float32x4_t b1 = vld1q_f32(b + i + 4); + acc0 = vfmaq_f32(acc0, a0, b0); + acc1 = vfmaq_f32(acc1, a1, b1); + } + float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +} + +static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { + int i = 0; + float32x4_t s = vdupq_n_f32(scale); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); + vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); + } + for (; i < n; i++) + dst[i] *= scale; +} + +static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { + int i = 0; + float32x4_t a = vdupq_n_f32(alpha); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); + vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +} + +static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { + int i = 0; + float32x4_t c = vdupq_n_f32(correction); + for (; i + 8 <= n; i += 8) { + float32x4_t d0 = vld1q_f32(dst + i); + float32x4_t s0 = vld1q_f32(src + i); + float32x4_t d1 = vld1q_f32(dst + i + 4); + float32x4_t s1 = vld1q_f32(src + i + 4); + vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); + vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +} + +#elif defined(__AVX2__) && defined(__FMA__) + +# include <immintrin.h> + +void bf16_matvec_fused_avx( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); +# define bf16_matvec_fused_impl bf16_matvec_fused_avx + +static inline float dot_f32_impl(const float *a, const float *b, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 acc0 = _mm512_setzero_ps(); + __m512 acc1 = _mm512_setzero_ps(); + __m512 acc2 = _mm512_setzero_ps(); + __m512 acc3 = _mm512_setzero_ps(); + for (; i + 64 <= n; i += 64) { + acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0); + acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1); + acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2); + acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3); + } + __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)); + for (; i + 16 <= n; i += 16) { + acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc); + } + float sum = _mm512_reduce_add_ps(acc); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +# else + int i = 0; + __m256 acc0 = _mm256_setzero_ps(); + __m256 acc1 = _mm256_setzero_ps(); + __m256 acc2 = _mm256_setzero_ps(); + __m256 acc3 = _mm256_setzero_ps(); + for (; i + 32 <= n; i += 32) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); + acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1); + acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2); + acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3); + } + acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3)); + for (; i + 8 <= n; i += 8) { + acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0); + } + __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1)); + r = _mm_hadd_ps(r, r); + r = _mm_hadd_ps(r, r); + float sum = _mm_cvtss_f32(r); + for (; i < n; i++) + sum += a[i] * b[i]; + return sum; +# endif +} + +static inline void vec_scale_inplace_impl(float *dst, float scale, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 s = _mm512_set1_ps(scale); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s)); + _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s)); + _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s)); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s)); + } + for (; i < n; i++) + dst[i] *= scale; +# else + int i = 0; + __m256 s = _mm256_set1_ps(scale); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); + _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s)); + _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s)); + _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s)); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s)); + } + for (; i < n; i++) + dst[i] *= scale; +# endif +} + +static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 a = _mm512_set1_ps(alpha); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a, + _mm512_loadu_ps(dst + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a, + _mm512_loadu_ps(dst + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a, + _mm512_loadu_ps(dst + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i))); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +# else + int i = 0; + __m256 a = _mm256_set1_ps(alpha); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a, + _mm256_loadu_ps(dst + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a, + _mm256_loadu_ps(dst + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a, + _mm256_loadu_ps(dst + i + 24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i))); + } + for (; i < n; i++) + dst[i] += alpha * src[i]; +# endif +} + +static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) { +# if defined(__AVX512F__) + int i = 0; + __m512 c = _mm512_set1_ps(correction); + for (; i + 64 <= n; i += 64) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c, + _mm512_loadu_ps(src + i + 16))); + _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c, + _mm512_loadu_ps(src + i + 32))); + _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c, + _mm512_loadu_ps(src + i + 48))); + } + for (; i + 16 <= n; i += 16) { + _mm512_storeu_ps(dst + i, + _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i))); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +# else + int i = 0; + __m256 c = _mm256_set1_ps(correction); + for (; i + 32 <= n; i += 32) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); + _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c, + _mm256_loadu_ps(src + i + 8))); + _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c, + _mm256_loadu_ps(src + i + 16))); + _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c, + _mm256_loadu_ps(src + i + 24))); + } + for (; i + 8 <= n; i += 8) { + _mm256_storeu_ps(dst + i, + _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i))); + } + for (; i < n; i++) + dst[i] = dst[i] * correction + src[i]; +# endif +} + +#else /* no SIMD path: dispatch to the scalar references */ + +# define bf16_matvec_fused_impl bf16_matvec_fused_generic +# define dot_f32_impl dot_f32_generic +# define vec_scale_inplace_impl vec_scale_inplace_generic +# define vec_axpy_inplace_impl vec_axpy_inplace_generic +# define vec_scale_add_impl vec_scale_add_generic + +#endif + +#endif /* KERNELS_IMPL_H */ diff --git a/libffwd/internal.h b/libffwd/internal.h new file mode 100644 index 0000000..0338bc2 --- /dev/null +++ b/libffwd/internal.h @@ -0,0 +1,134 @@ +/* + * internal.h - non-exported, in-tree API shared by the frontends, + * backends, tests, and benchmarks. The public ABI is ffwd.h; none of these + * symbols carry FFWD_API visibility or appear in the export maps. + */ + +#ifndef FFWD_INTERNAL_H +#define FFWD_INTERNAL_H + +#include "ffwd.h" + +#include <stddef.h> +#include <stdio.h> + +typedef struct ffwd_model ffwd_model_t; +typedef struct ffwd_workspace ffwd_workspace_t; +typedef struct ffwd_late_model ffwd_late_model_t; +typedef struct ffwd_late_workspace ffwd_late_workspace_t; + +/* Backend-specific help text used by the in-tree frontends. */ +void ffwd_cli_help(FILE *f); +void ffwd_server_help(FILE *f); + +/* + * Lower-level CPU model API. These functions are used by backend glue, tests, + * benchmarks, and parity harnesses; they are not exported from libffwd. + */ +ffwd_model_t *ffwd_model_load(const char *model_dir); +void ffwd_model_free(ffwd_model_t *model); + +ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model); +void ffwd_workspace_free(ffwd_workspace_t *ws); + +const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model); +size_t ffwd_workspace_nbytes(const ffwd_workspace_t *ws); + +float *ffwd_model_encode(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens); + +int ffwd_model_encode_into(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding); + +int ffwd_model_encode_batch(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, + int batch, + float *out_embeddings); + +int ffwd_pool_batch(const ffwd_config_t *cfg, + const float *states, + const int *seq_lengths, + int batch, + float *out_embeddings); + +float *ffwd_model_forward(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens); + +int ffwd_model_forward_into(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_states); + +int ffwd_pool_spans(const ffwd_config_t *cfg, + const float *states, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings); + +int ffwd_model_encode_spans(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const int *token_ids, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings); + +int ffwd_model_encode_spans_batch(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings); + +/* + * Late-interaction CPU model API. The public engine path exposes reranking via + * ffwd_rerank(); these entry points are for in-tree tests/backends only. + */ +ffwd_late_model_t *ffwd_late_model_load(const char *model_dir); +void ffwd_late_model_free(ffwd_late_model_t *model); + +ffwd_late_workspace_t *ffwd_late_workspace_new(const ffwd_late_model_t *model); +void ffwd_late_workspace_free(ffwd_late_workspace_t *ws); + +const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model); +int ffwd_late_model_token_dim(const ffwd_late_model_t *model); + +int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model, + ffwd_late_workspace_t *ws, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors); + +int ffwd_late_model_encode_docs(const ffwd_late_model_t *model, + ffwd_late_workspace_t *ws, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets); + +float ffwd_late_maxsim( + const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim); + +int ffwd_late_maxsim_batch(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores); + +#endif /* FFWD_INTERNAL_H */ diff --git a/src/kernels.h b/libffwd/kernels.h index 17c45c0..f36d0d3 100644 --- a/src/kernels.h +++ b/libffwd/kernels.h @@ -1,22 +1,11 @@ /* - * embed_kernels.h - Math kernels for Qwen3-architecture inference + * kernels.h - tkern public API: transformer inference kernels (CPU, float32) * * Low-level math operations. All operate on float32 tensors in row-major order. */ -#ifndef EMBED_KERNELS_H -#define EMBED_KERNELS_H - -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying EMBED_API are exported - * from the shared library; everything else stays internal. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif +#ifndef KERNELS_H +#define KERNELS_H #include <stddef.h> #include <stdint.h> @@ -25,30 +14,30 @@ * Basic Operations * ======================================================================== */ -void embed_add_inplace(float *a, const float *b, int n); +void add_inplace(float *a, const float *b, int n); /* ======================================================================== * Matrix Operations * ======================================================================== */ /* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ -void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N); +void matmul_t(float *C, const float *A, const float *B, int M, int K, int N); /* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */ -void embed_linear( +void linear( float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); -void embed_linear_nobias( +void linear_nobias( float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); /* bf16 weight variants */ -void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n); -void embed_linear_nobias_bf16( +void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n); +void linear_nobias_bf16( float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim); /* Small-sequence BF16 path: compute two projections with one threaded dispatch. */ -void embed_linear_nobias_bf16_pair(float *a, +void linear_nobias_bf16_pair(float *a, float *b, const float *x, const uint16_t *Wa_bf16, @@ -59,7 +48,7 @@ void embed_linear_nobias_bf16_pair(float *a, int b_dim); /* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ -void embed_linear_nobias_bf16_qkv(float *q, +void linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, @@ -72,26 +61,26 @@ void embed_linear_nobias_bf16_qkv(float *q, int kv_dim); /* Dot product using the best available local SIMD implementation. */ -float embed_dot_f32(const float *a, const float *b, int n); +float dot_f32(const float *a, const float *b, int n); /* ======================================================================== * Normalization * ======================================================================== */ /* RMS Normalization: out = x / rms(x) * weight */ -void embed_rms_norm( +void rms_norm( float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); /* Per-head RMS Normalization for Q/K norms in decoder * x: [seq, n_heads, head_dim], weight: [head_dim] * Normalizes each head independently */ -void embed_rms_norm_per_head( +void rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); /* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps) * + beta, with mean and biased (population) variance over the hidden axis per * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ -void embed_layer_norm(float *out, +void layer_norm(float *out, const float *x, const float *gamma, const float *beta, @@ -103,17 +92,17 @@ void embed_layer_norm(float *out, * Activation Functions * ======================================================================== */ -void embed_softmax(float *x, int rows, int cols); +void softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ -void embed_silu_mul_inplace(float *gate, const float *up, int n); +void silu_mul_inplace(float *gate, const float *up, int n); /* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ -void embed_gelu_inplace(float *x, int n); +void gelu_inplace(float *x, int n); /* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh): * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */ -void embed_gelu_tanh_inplace(float *x, int n); +void gelu_tanh_inplace(float *x, int n); /* ======================================================================== * Attention Operations @@ -129,7 +118,7 @@ void embed_gelu_tanh_inplace(float *x, int n); * Attention is block-diagonal: tokens only attend within their own * offsets[b]..offsets[b+1] span. */ -void embed_bidirectional_gqa_attention_packed(float *out, +void bidirectional_gqa_attention_packed(float *out, const float *Q, const float *K, const float *V, @@ -141,8 +130,8 @@ void embed_bidirectional_gqa_attention_packed(float *out, float scale); /* Optional reusable scratch for the BLAS-tiled packed attention path. */ -size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); -void embed_bidirectional_gqa_attention_packed_with_scratch(float *out, +size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); +void bidirectional_gqa_attention_packed_with_scratch(float *out, const float *Q, const float *K, const float *V, @@ -159,7 +148,7 @@ void embed_bidirectional_gqa_attention_packed_with_scratch(float *out, * Causal GQA attention over a packed/ragged batch. Query position i attends * only to positions at or before i within its sequence. */ -void embed_causal_gqa_attention_packed(float *out, +void causal_gqa_attention_packed(float *out, const float *Q, const float *K, const float *V, @@ -169,7 +158,7 @@ void embed_causal_gqa_attention_packed(float *out, int n_kv_heads, int head_dim, float scale); -void embed_causal_gqa_attention_packed_with_scratch(float *out, +void causal_gqa_attention_packed_with_scratch(float *out, const float *Q, const float *K, const float *V, @@ -191,7 +180,7 @@ void embed_causal_gqa_attention_packed_with_scratch(float *out, * cos_out, sin_out: [seq, head_dim] * cos[d] and cos[half+d] are the same (duplicated for full head_dim). */ -void embed_compute_rope_neox( +void compute_rope_neox( float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta); /* @@ -199,18 +188,7 @@ void embed_compute_rope_neox( * x: [seq, n_heads * head_dim] (in-place) * cos_vals, sin_vals: [seq, head_dim] */ -void embed_apply_rope_neox( +void apply_rope_neox( float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim); -/* ======================================================================== - * Threading - * ======================================================================== */ - -/* The persistent worker pool lives in threadpool.{c,h}; included here so kernel - * callers keep getting embed_set_threads / embed_get_num_cpus from one header. */ -#include "threadpool.h" - -/* Global verbose flag */ -EMBED_API extern int embed_verbose; - -#endif /* EMBED_KERNELS_H */ +#endif /* KERNELS_H */ diff --git a/libffwd/late.c b/libffwd/late.c new file mode 100644 index 0000000..05d0341 --- /dev/null +++ b/libffwd/late.c @@ -0,0 +1,218 @@ +#include "ffwd.h" +#include "model_internal.h" +#include "kernels.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <limits.h> +#include <float.h> + +/* ======================================================================== + * Late-interaction model + * + * A late-interaction model is a pooled base model plus a 1_Dense linear + * projection from hidden states to token vectors. Instead of pooling to one + * vector, it keeps one projected vector per kept token and scores query/doc + * pairs with MaxSim. It reuses the base model's packed forward pass and + * workspace; only the projection, the token-vector buffer, and the scoring + * live here. + * ======================================================================== */ + +struct ffwd_late_model { + ffwd_model_t *base; + safetensors_file_t *dense_sf; + ffwd_weight_ref_t projection; /* [token_dim, hidden] */ + int token_dim; +}; + +static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) { + if (!sf || !name) + return NULL; + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) + return &sf->tensors[i]; + } + return NULL; +} + +ffwd_late_model_t *ffwd_late_model_load(const char *model_dir) { + if (!model_dir_has_late_projection(model_dir)) { + fprintf(stderr, "ffwd_late_model_load: missing 1_Dense projection\n"); + return NULL; + } + + ffwd_late_model_t *late = (ffwd_late_model_t *)calloc(1, sizeof(*late)); + if (!late) + return NULL; + + late->base = model_load_range_ex(model_dir, 0, -1, 1); + if (!late->base) + goto fail; + + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir); + if (n < 0 || (size_t)n >= sizeof(path)) { + fprintf(stderr, "ffwd_late_model_load: model path too long\n"); + goto fail; + } + + late->dense_sf = safetensors_open(path); + if (!late->dense_sf) { + fprintf(stderr, "ffwd_late_model_load: cannot open %s\n", path); + goto fail; + } + + const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight"); + if (!t) { + fprintf(stderr, "ffwd_late_model_load: missing linear.weight\n"); + goto fail; + } + if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) { + fprintf(stderr, "ffwd_late_model_load: bad projection shape\n"); + goto fail; + } + + int64_t shape[2] = {t->shape[0], late->base->config.hidden_size}; + if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1)) + goto fail; + + /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */ + late->projection = weight_ref_from_tensor(late->dense_sf, t); + if (!late->projection.data) + goto fail; + late->token_dim = (int)t->shape[0]; + return late; + +fail: + ffwd_late_model_free(late); + return NULL; +} + +void ffwd_late_model_free(ffwd_late_model_t *model) { + if (!model) + return; + ffwd_model_free(model->base); + /* Free the projection before the mmap: only an F16-widened ref owns its + * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */ + if (model->projection.owned) + free((void *)model->projection.data); + safetensors_close(model->dense_sf); + free(model); +} + +const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model) { + return model && model->base ? &model->base->config : NULL; +} + +int ffwd_late_model_token_dim(const ffwd_late_model_t *model) { + return model ? model->token_dim : 0; +} + +/* Internal (internal.h): expose the base model and projection so GPU + * backends can upload them to device without seeing struct ffwd_late_model. */ +ffwd_model_t *ffwd_late_model_base(const ffwd_late_model_t *model) { + return model ? model->base : NULL; +} + +const ffwd_weight_ref_t *ffwd_late_model_projection(const ffwd_late_model_t *model) { + return model ? &model->projection : NULL; +} + +float ffwd_late_maxsim( + const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim) { + if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0) + return 0.0f; + + float score = 0.0f; + for (int qi = 0; qi < query_tokens; qi++) { + const float *q = query_vectors + (size_t)qi * dim; + float best = -FLT_MAX; + for (int di = 0; di < doc_tokens; di++) { + const float *d = doc_vectors + (size_t)di * dim; + float dot = dot_f32(q, d, dim); + if (dot > best) + best = dot; + } + score += best; + } + return score; +} + +int ffwd_late_maxsim_batch(const float *query_vectors, + int query_tokens, + const float *doc_vectors, + const int *doc_offsets, + int docs, + int dim, + float *scores) { + if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || docs <= 0 || + dim <= 0 || doc_offsets[0] != 0) + return -1; + + for (int i = 0; i < docs; i++) { + if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i]) + return -1; + } + +#ifdef USE_BLAS + /* One similarity GEMM per group of whole candidates: + * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document + * row-max plus sum. The group token budget bounds the scratch matrix; + * one oversized document still runs alone in a group of its own. */ + enum { LATE_GROUP_TOKEN_BUDGET = 4096 }; + int total = doc_offsets[docs]; + int budget = LATE_GROUP_TOKEN_BUDGET; + size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget); + int largest = 0; + for (int i = 0; i < docs; i++) { + int len = doc_offsets[i + 1] - doc_offsets[i]; + if (len > largest) + largest = len; + } + if ((size_t)query_tokens * (size_t)largest > cap) + cap = (size_t)query_tokens * (size_t)largest; + float *sim = (float *)malloc(cap * sizeof(float)); + if (sim) { + int doc = 0; + while (doc < docs) { + int first = doc; + int start = doc_offsets[first]; + doc++; + while (doc < docs && doc_offsets[doc + 1] - start <= budget && + doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens)) + doc++; + int group_tokens = doc_offsets[doc] - start; + + matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, + group_tokens); + + for (int i = first; i < doc; i++) { + int s0 = doc_offsets[i] - start; + int s1 = doc_offsets[i + 1] - start; + float score = 0.0f; + for (int qi = 0; qi < query_tokens; qi++) { + const float *row = sim + (size_t)qi * group_tokens; + float best = row[s0]; + for (int t = s0 + 1; t < s1; t++) + if (row[t] > best) + best = row[t]; + score += best; + } + scores[i] = score; + } + } + free(sim); + return 0; + } + /* Allocation failure: fall through to the scalar path. */ +#endif + + for (int i = 0; i < docs; i++) { + int start = doc_offsets[i]; + int end = doc_offsets[i + 1]; + scores[i] = ffwd_late_maxsim(query_vectors, query_tokens, + doc_vectors + (size_t)start * dim, end - start, dim); + } + return 0; +} diff --git a/libffwd/late_blas.c b/libffwd/late_blas.c new file mode 100644 index 0000000..cc57596 --- /dev/null +++ b/libffwd/late_blas.c @@ -0,0 +1,173 @@ +#include "ffwd.h" +#include "model_internal.h" + +#include <limits.h> +#include <stdlib.h> +#include <string.h> + +/* CPU late-interaction encoding. The late model loader and MaxSim scoring live + * in late.c because CUDA reuses those without linking the CPU transformer. */ + +struct ffwd_late_workspace { + const ffwd_late_model_t *model; + ffwd_workspace_t *base_ws; + float *states; + int states_seq_cap; +}; + +static int ensure_late_state_buffer(ffwd_late_workspace_t *ws, int n_tokens) { + if (!ws || !ws->model || n_tokens <= 0) + return -1; + if (n_tokens <= ws->states_seq_cap) + return 0; + ffwd_model_t *base = ffwd_late_model_base(ws->model); + if (!base) + return -1; + int hidden = base->config.hidden_size; + if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0) + return -1; + ws->states_seq_cap = n_tokens; + return 0; +} + +ffwd_late_workspace_t *ffwd_late_workspace_new(const ffwd_late_model_t *model) { + ffwd_model_t *base = ffwd_late_model_base(model); + if (!model || !base) + return NULL; + ffwd_late_workspace_t *ws = (ffwd_late_workspace_t *)calloc(1, sizeof(*ws)); + if (!ws) + return NULL; + ws->model = model; + ws->base_ws = ffwd_workspace_new(base); + if (!ws->base_ws) { + free(ws); + return NULL; + } + return ws; +} + +void ffwd_late_workspace_free(ffwd_late_workspace_t *ws) { + if (!ws) + return; + ffwd_workspace_free(ws->base_ws); + free(ws->states); + free(ws); +} + +int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model, + ffwd_late_workspace_t *ws, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { + int dim = ffwd_late_model_token_dim(model); + ffwd_model_t *base = ffwd_late_model_base(model); + const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model); + if (!model || !ws || ws->model != model || !base || !projection || !projection->data || + !token_ids || n_tokens <= 0 || !out_vectors || dim <= 0) + return -1; + + int hidden = base->config.hidden_size; + if (ensure_late_state_buffer(ws, n_tokens) != 0) + return -1; + if (ffwd_model_forward_into(base, ws->base_ws, token_ids, n_tokens, ws->states) != 0) + return -1; + + linear_nobias_weight(ws->base_ws, out_vectors, ws->states, projection, n_tokens, hidden, dim); + if (normalize) { + for (int i = 0; i < n_tokens; i++) { + if (ffwd_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + } + return 0; +} + +/* Late-interaction batch encoder. Reuses the base model's packed/ragged batch + * forward pass, then gathers kept rows and projects them in one GEMM. */ +int ffwd_late_model_encode_docs(const ffwd_late_model_t *model, + ffwd_late_workspace_t *ws, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + float *out_vectors, + int *out_offsets) { + int dim = ffwd_late_model_token_dim(model); + ffwd_model_t *base = ffwd_late_model_base(model); + const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model); + if (!model || !ws || ws->model != model || !base || !projection || !projection->data || + !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors || !out_offsets || + dim <= 0) + return -1; + + int hidden = base->config.hidden_size; + + ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); + int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets)); + if (!inputs || !offsets) { + free(inputs); + free(offsets); + return -1; + } + + int total_keep = 0; + out_offsets[0] = 0; + for (int d = 0; d < n_docs; d++) { + if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 || + n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) { + free(inputs); + free(offsets); + return -1; + } + for (int k = 0; k < n_keep[d]; k++) { + if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) { + free(inputs); + free(offsets); + return -1; + } + } + inputs[d].ids = doc_ids[d]; + inputs[d].n_tokens = n_tokens[d]; + total_keep += n_keep[d]; + out_offsets[d + 1] = total_keep; + } + + int total_seq = 0, max_seq = 0; + int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq); + if (rc == 0) + rc = + forward_packed_inplace(base, ws->base_ws, inputs, n_docs, offsets, total_seq, max_seq, 1); + free(inputs); + if (rc != 0) { + free(offsets); + return -1; + } + + if (ensure_late_state_buffer(ws, total_keep) != 0) { + free(offsets); + return -1; + } + const float *states = ws->base_ws->x; + int pos = 0; + for (int d = 0; d < n_docs; d++) { + int start = offsets[d]; + for (int k = 0; k < n_keep[d]; k++) { + memcpy(ws->states + (size_t)pos * hidden, states + (size_t)(start + keep[d][k]) * hidden, + (size_t)hidden * sizeof(float)); + pos++; + } + } + free(offsets); + + linear_nobias_weight(ws->base_ws, out_vectors, ws->states, projection, total_keep, hidden, dim); + if (normalize) { + for (int i = 0; i < total_keep; i++) { + if (ffwd_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) + return -1; + } + } + return 0; +} diff --git a/src/mlx.c b/libffwd/mlx.c index 7d05e91..10299c1 100644 --- a/src/mlx.c +++ b/libffwd/mlx.c @@ -1,7 +1,7 @@ #include "mlx.h" #include "config.h" #include "dtype.h" -#include "embed.h" +#include "ffwd.h" #include "safetensors.h" #include <mlx/c/mlx.h> @@ -33,19 +33,19 @@ typedef struct { mlx_array q_norm, k_norm; mlx_array input_norm, post_attn_norm; mlx_linear_t gate_proj, up_proj, down_proj; - /* BERT family (unset for Qwen3; see embed_internal.h). wo/up_proj/down_proj + /* BERT family (unset for Qwen3; see internal.h). wo/up_proj/down_proj * and input_norm/post_attn_norm are reused as the BERT attention output, * FFN matrices, and the two block LayerNorm weights. */ mlx_array o_bias, attn_ln_bias, ffn_inter_bias, ffn_out_bias, ffn_ln_bias; } mlx_layer_t; -struct embed_mlx_ctx { - embed_config_t config; +struct ffwd_mlx_ctx { + ffwd_config_t config; mlx_array embed_tokens; mlx_layer_t *layers; /* heap-allocated [n_layers] */ mlx_array norm; /* BERT family (unset for Qwen3): learned positions, token-type, embed LN. */ - mlx_array position_embeddings, token_type_embeddings, embed_ln_w, embed_ln_b; + mlx_array position_embeddings, token_type_embeddings, ffwd_ln_w, ffwd_ln_b; mlx_stream stream; multi_safetensors_t *ms; int layer_start; /* loaded transformer range, inclusive */ @@ -55,15 +55,15 @@ struct embed_mlx_ctx { mlx_dtype weight_dtype; }; -struct embed_mlx_late_ctx { - embed_mlx_ctx_t *base; +struct ffwd_mlx_late_ctx { + ffwd_mlx_ctx_t *base; multi_safetensors_t *dense_ms; mlx_linear_t projection; int token_dim; }; -struct embed_mlx_late_vectors { - embed_mlx_late_ctx_t *owner; +struct ffwd_mlx_late_vectors { + ffwd_mlx_late_ctx_t *owner; mlx_array vectors; /* [tokens, dim] */ int tokens; int dim; @@ -82,7 +82,7 @@ static int mlx_mul_size(size_t a, size_t b, size_t *out) { return 0; } -static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; } +static mlx_dtype mlx_weight_dtype(const ffwd_mlx_ctx_t *ctx) { return ctx->weight_dtype; } static size_t safetensor_dtype_size(safetensor_dtype_t dtype) { switch (dtype) { @@ -148,8 +148,7 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } -static mlx_array -load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { +static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) { safetensors_file_t *sf = NULL; const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { @@ -169,7 +168,7 @@ load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndi return (mlx_array){0}; const uint16_t *src = (const uint16_t *)safetensors_data(sf, t); for (int64_t i = 0; i < n; i++) - tmp[i] = embed_f16_to_f32(src[i]); + tmp[i] = ffwd_f16_to_f32(src[i]); mlx_array a = mlx_array_new_data(tmp, shape, ndim, MLX_FLOAT32); free(tmp); return a; @@ -295,8 +294,8 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) { mlx_optional_int gs = {.value = W->group_size, .has_value = true}; mlx_optional_int bits = {.value = W->bits, .has_value = true}; mlx_array null_arr = (mlx_array){0}; - mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr, - true, gs, bits, "affine", s); + mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr, true, + gs, bits, "affine", s); return res; } mlx_array Wt; @@ -328,7 +327,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) { return res; } -const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) { +const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx) { return ctx ? &ctx->config : NULL; } @@ -365,12 +364,12 @@ static int mlx_model_dir_has_late_projection(const char *model_dir) { mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors"); } -static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) { - embed_mlx_options_t out = {0}; +static ffwd_mlx_options_t normalize_mlx_options(const ffwd_mlx_options_t *opts) { + ffwd_mlx_options_t out = {0}; if (opts) out = *opts; if (out.quantize_bits != 0 && out.quantize_bits != 8) { - fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n"); + fprintf(stderr, "mlx: --gpu-quant-bits supports only 8\n"); out.quantize_bits = -1; } if (out.quantize_bits && out.quantize_group_size <= 0) @@ -400,8 +399,8 @@ static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) { /* BERT-family MLX loader, mirroring load_bert_weights on the CPU side. The * shared post-load below (stream, transposes, optional BF16 upcast) runs after * either family fills ctx->layers. */ -static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int layer_end) { - const embed_config_t *c = &ctx->config; +static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer_end) { + const ffwd_config_t *c = &ctx->config; int h = c->hidden_size, inter = c->intermediate_size; char name[256]; @@ -429,12 +428,12 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye ctx->token_type_embeddings = load_tensor(ctx->ms, name, type_shape, 2); int ln_shape[] = {h}; snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p); - ctx->embed_ln_w = load_tensor(ctx->ms, name, ln_shape, 1); + ctx->ffwd_ln_w = load_tensor(ctx->ms, name, ln_shape, 1); snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); - ctx->embed_ln_b = load_tensor(ctx->ms, name, ln_shape, 1); + ctx->ffwd_ln_b = load_tensor(ctx->ms, name, ln_shape, 1); if (!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->position_embeddings) || - !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->embed_ln_w) || - !arr_ok(ctx->embed_ln_b)) + !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->ffwd_ln_w) || + !arr_ok(ctx->ffwd_ln_b)) return -1; } @@ -483,12 +482,12 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye return 0; } -static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, - int layer_start, - int layer_end, - const embed_mlx_options_t *opts, - int allow_late) { - embed_mlx_options_t options = normalize_mlx_options(opts); +static ffwd_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, + int layer_start, + int layer_end, + const ffwd_mlx_options_t *opts, + int allow_late) { + ffwd_mlx_options_t options = normalize_mlx_options(opts); if (options.quantize_bits < 0) return NULL; if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) { @@ -498,18 +497,18 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, return NULL; } - embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t)); + ffwd_mlx_ctx_t *ctx = calloc(1, sizeof(ffwd_mlx_ctx_t)); if (!ctx) return NULL; ctx->quantize_bits = options.quantize_bits; ctx->quantize_group_size = options.quantize_group_size; - if (embed_config_parse(&ctx->config, model_dir) != 0) { + if (ffwd_config_parse(&ctx->config, model_dir) != 0) { free(ctx); return NULL; } - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; if (layer_end == -1) layer_end = c->n_layers; if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) { @@ -529,7 +528,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, return NULL; } - if (c->family == EMBED_FAMILY_BERT) { + if (c->family == FFWD_FAMILY_BERT) { if (load_mlx_bert_weights(ctx, layer_start, layer_end) != 0) goto fail; goto loaded; @@ -614,7 +613,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir, LD(down_proj, "%slayers.%d.mlp.down_proj.weight", h, inter); #undef LD - if (embed_verbose >= 2) + if (ffwd_verbose >= 2) fprintf(stderr, " mlx layer %d loaded\n", i); } @@ -655,8 +654,8 @@ loaded: * embeddings and the embedding LayerNorm all feed the F32 residual. */ upcast_f32_inplace(&ctx->position_embeddings, ctx->stream); upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream); - upcast_f32_inplace(&ctx->embed_ln_w, ctx->stream); - upcast_f32_inplace(&ctx->embed_ln_b, ctx->stream); + upcast_f32_inplace(&ctx->ffwd_ln_w, ctx->stream); + upcast_f32_inplace(&ctx->ffwd_ln_b, ctx->stream); } if (ctx->quantize_bits) { for (int i = layer_start; i < layer_end; i++) { @@ -665,7 +664,7 @@ loaded: goto fail; } mlx_clear_cache(); - if (embed_verbose >= 1) + if (ffwd_verbose >= 1) fprintf(stderr, "mlx: quantized linear weights to %d-bit " "(group_size=%d)\n", @@ -677,7 +676,7 @@ loaded: } } - if (embed_verbose >= 2) { + if (ffwd_verbose >= 2) { size_t active = 0, cache = 0, peak = 0; mlx_get_active_memory(&active); mlx_get_cache_memory(&cache); @@ -690,22 +689,22 @@ loaded: (double)peak / (1024.0 * 1024.0 * 1024.0)); } - if (embed_verbose >= 1) + if (ffwd_verbose >= 1) fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h); return ctx; fail: - embed_mlx_free(ctx); + ffwd_mlx_free(ctx); return NULL; } -embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) { +ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir) { return mlx_load_range_ex(model_dir, 0, -1, NULL, 0); } -embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, - const embed_mlx_options_t *opts) { +ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, + const ffwd_mlx_options_t *opts) { return mlx_load_range_ex(model_dir, 0, -1, opts, 0); } @@ -735,7 +734,7 @@ static void free_mlx_layer(mlx_layer_t *l) { mlx_array_free(l->ffn_ln_bias); } -void embed_mlx_free(embed_mlx_ctx_t *ctx) { +void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx) { if (!ctx) return; mlx_array_free(ctx->embed_tokens); @@ -747,8 +746,8 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx) { mlx_array_free(ctx->norm); mlx_array_free(ctx->position_embeddings); mlx_array_free(ctx->token_type_embeddings); - mlx_array_free(ctx->embed_ln_w); - mlx_array_free(ctx->embed_ln_b); + mlx_array_free(ctx->ffwd_ln_w); + mlx_array_free(ctx->ffwd_ln_b); if (ctx->stream.ctx) mlx_stream_free(ctx->stream); if (ctx->ms) @@ -756,14 +755,14 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx) { free(ctx); } -embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir, - const embed_mlx_options_t *opts) { +ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir, + const ffwd_mlx_options_t *opts) { if (!mlx_model_dir_has_late_projection(model_dir)) { fprintf(stderr, "mlx late: missing 1_Dense projection\n"); return NULL; } - embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); + ffwd_mlx_late_ctx_t *late = calloc(1, sizeof(*late)); if (!late) return NULL; @@ -803,34 +802,34 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir, goto fail; late->token_dim = shape[0]; - if (embed_verbose >= 1) + if (ffwd_verbose >= 1) fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim); return late; fail: - embed_mlx_late_free(late); + ffwd_mlx_late_free(late); return NULL; } -embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) { - return embed_mlx_late_load_with_options(model_dir, NULL); +ffwd_mlx_late_ctx_t *ffwd_mlx_late_load(const char *model_dir) { + return ffwd_mlx_late_load_with_options(model_dir, NULL); } -void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) { +void ffwd_mlx_late_free(ffwd_mlx_late_ctx_t *ctx) { if (!ctx) return; free_linear(&ctx->projection); if (ctx->dense_ms) multi_safetensors_close(ctx->dense_ms); - embed_mlx_free(ctx->base); + ffwd_mlx_free(ctx->base); free(ctx); } -const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) { +const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx) { return ctx && ctx->base ? &ctx->base->config : NULL; } -int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } +int ffwd_mlx_late_token_dim(const ffwd_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; } /* ======================================================================== * Forward pass @@ -904,7 +903,7 @@ static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) { /* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU * feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the * x it is given and returns the new hidden states. */ -static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx, +static mlx_array mlx_forward_bert(ffwd_mlx_ctx_t *ctx, mlx_array x, int batch, int max_seq, @@ -912,7 +911,7 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx, mlx_array attn_mask, int layer_start, int layer_end) { - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int n_heads = c->n_heads, head_dim = c->head_dim, q_dim = c->q_dim; float scale = 1.0f / sqrtf((float)head_dim); @@ -990,8 +989,8 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx, mlx_array mid_b = mlx_array_new(); mlx_add(&mid_b, mid, l->ffn_inter_bias, S); mlx_array_free(mid); - mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S) - : mlx_gelu_erf(mid_b, S); + mlx_array g = ctx->config.ffn_act == FFWD_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S) + : mlx_gelu_erf(mid_b, S); mlx_array_free(mid_b); mlx_array out = linear(g, &l->down_proj, S); mlx_array_free(g); @@ -1009,7 +1008,7 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx, return x; } -static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, +static mlx_array mlx_forward_layers(ffwd_mlx_ctx_t *ctx, mlx_array x, int batch, int max_seq, @@ -1017,7 +1016,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, mlx_array attn_mask, int layer_start, int layer_end) { - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int n_heads = c->n_heads; int n_kv_heads = c->n_kv_heads; @@ -1025,7 +1024,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, int q_dim = c->q_dim; float scale = 1.0f / sqrtf((float)head_dim); mlx_array null_arr = (mlx_array){0}; - const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : ""; + const char *mask_mode = c->attention_mode == FFWD_ATTENTION_CAUSAL ? "causal" : ""; /* For non-quantized BF16 weights, run the residual stream, norms, and * attention in F32; matmuls downcast their operands to BF16 internally @@ -1034,8 +1033,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, * models are unaffected. */ mlx_array local_mask = attn_mask; int free_mask = 0; - int mixed_bf16 = - mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized; + int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized; if (mixed_bf16) { if (mlx_array_dtype(x) != MLX_FLOAT32) { mlx_array xf = mlx_array_new(); @@ -1172,20 +1170,20 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx, return x; } -embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize) { +ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize) { if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection)) return NULL; - embed_mlx_ctx_t *base = ctx->base; + ffwd_mlx_ctx_t *base = ctx->base; if (base->layer_start != 0 || base->layer_end != base->config.n_layers || !arr_ok(base->embed_tokens) || !arr_ok(base->norm)) return NULL; - const embed_config_t *c = &base->config; + const ffwd_config_t *c = &base->config; mlx_stream S = base->stream; for (int i = 0; i < n_tokens; i++) { if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) { @@ -1209,7 +1207,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx if (!arr_ok(ids)) return NULL; - embed_mlx_late_vectors_t *result = NULL; + ffwd_mlx_late_vectors_t *result = NULL; mlx_array x = mlx_array_new(); mlx_array x_normed = (mlx_array){0}; mlx_array projected = (mlx_array){0}; @@ -1276,7 +1274,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx if (!arr_ok(reshaped)) goto cleanup; - result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result)); + result = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*result)); if (!result) goto cleanup; result->owner = ctx; @@ -1293,22 +1291,22 @@ cleanup: return result; } -void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) { +void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs) { if (!vecs) return; mlx_array_free(vecs->vectors); free(vecs); } -int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) { +int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs) { return vecs ? vecs->tokens : 0; } -int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) { +int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs) { return vecs ? vecs->dim : 0; } -int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) { +int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors) { if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0) return -1; @@ -1343,7 +1341,7 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out * document's kept token vectors into a single packed [total_keep, token_dim] * array in document order. out_offsets receives the prefix sum of kept counts * (out_offsets[i + 1] - out_offsets[i] == n_keep[i]), the exact layout - * embed_mlx_late_maxsim_batch_device consumes. This collapses N per-candidate + * ffwd_mlx_late_maxsim_batch_device consumes. This collapses N per-candidate * transformer calls into one forward. * * Right-padding keeps real tokens at positions 0..n_tokens[d]-1 so RoPE @@ -1352,26 +1350,26 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out * batch path uses). Kept-token vectors therefore match per-document encoding to * within MLX's batched-GEMM rounding. */ -embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - int *out_offsets) { +ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx, + const int *const *doc_ids, + const int *n_tokens, + const int *const *keep, + const int *n_keep, + int n_docs, + int normalize, + int *out_offsets) { if (!ctx || !ctx->base || !doc_ids || !n_tokens || !keep || !n_keep || !out_offsets || n_docs <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection)) return NULL; - embed_mlx_ctx_t *base = ctx->base; + ffwd_mlx_ctx_t *base = ctx->base; if (base->layer_start != 0 || base->layer_end != base->config.n_layers || !arr_ok(base->embed_tokens) || !arr_ok(base->norm)) return NULL; - const embed_config_t *c = &base->config; + const ffwd_config_t *c = &base->config; mlx_stream S = base->stream; - int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; + int causal = c->attention_mode == FFWD_ATTENTION_CAUSAL; /* Validate inputs and size the batch: longest sequence and total kept * tokens. out_offsets is the running prefix sum of kept counts. */ @@ -1476,7 +1474,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t } } - embed_mlx_late_vectors_t *result = NULL; + ffwd_mlx_late_vectors_t *result = NULL; mlx_array x = mlx_array_new(); mlx_array x_normed = (mlx_array){0}; mlx_array projected = (mlx_array){0}; @@ -1555,7 +1553,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t goto cleanup; } - result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result)); + result = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*result)); if (!result) goto cleanup; result->owner = ctx; @@ -1576,17 +1574,17 @@ cleanup: } /* Device-resident select/concat primitives. The server rerank path batches - * candidates through embed_mlx_late_encode_docs_device, but the late-interaction - * verification harness (scripts/check_late_interaction.py) drives these directly + * candidates through ffwd_mlx_late_encode_docs_device, but the late-interaction + * verification harness (tests/integration/check_late_interaction.py) drives these directly * to validate per-document gather and concat against the CPU MaxSim reference. */ -embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) { +ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat( + ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count) { if (!ctx || !ctx->base || !items || count <= 0) return NULL; int dim = 0; int total = 0; for (int i = 0; i < count; i++) { - const embed_mlx_late_vectors_t *v = items[i]; + const ffwd_mlx_late_vectors_t *v = items[i]; if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0) return NULL; if (i == 0) @@ -1616,7 +1614,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( return NULL; } - embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + ffwd_mlx_late_vectors_t *out = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*out)); if (!out) { mlx_array_free(joined); return NULL; @@ -1628,10 +1626,10 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( return out; } -embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx, - const embed_mlx_late_vectors_t *vecs, - const int *token_indices, - int count) { +ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, + const ffwd_mlx_late_vectors_t *vecs, + const int *token_indices, + int count) { if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) || !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0) return NULL; @@ -1664,7 +1662,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ct return NULL; } - embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out)); + ffwd_mlx_late_vectors_t *out = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*out)); if (!out) { mlx_array_free(selected); return NULL; @@ -1676,12 +1674,12 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ct return out; } -int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx, - const embed_mlx_late_vectors_t *query, - const embed_mlx_late_vectors_t *docs, - const int *doc_offsets, - int docs_count, - float *scores) { +int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx, + const ffwd_mlx_late_vectors_t *query, + const ffwd_mlx_late_vectors_t *docs, + const int *doc_offsets, + int docs_count, + float *scores) { if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores || query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim || query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) || @@ -1884,39 +1882,39 @@ cleanup: return rc; } -int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors) { - embed_mlx_late_vectors_t *vecs = - embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize); +int ffwd_mlx_late_encode_tokens(ffwd_mlx_late_ctx_t *ctx, + const int *token_ids, + int n_tokens, + int normalize, + float *out_vectors) { + ffwd_mlx_late_vectors_t *vecs = + ffwd_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize); if (!vecs) return -1; - int rc = embed_mlx_late_vectors_copy(vecs, out_vectors); - embed_mlx_late_vectors_free(vecs); + int rc = ffwd_mlx_late_vectors_copy(vecs, out_vectors); + ffwd_mlx_late_vectors_free(vecs); return rc; } -static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, - const embed_input_t *inputs, - int batch, - float *out_embeddings) { +static int ffwd_mlx_encode_batch_dense(ffwd_mlx_ctx_t *ctx, + const ffwd_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers) return -1; - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int hidden = c->hidden_size; int max_seq = 0; int has_padding = 0; - int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL; - int cls_pool = c->pooling_mode == EMBED_POOL_CLS; + int causal = c->attention_mode == FFWD_ATTENTION_CAUSAL; + int cls_pool = c->pooling_mode == FFWD_POOL_CLS; /* CLS and last-token are both single-token pooling: gather one row per * sequence (the first for CLS, the last otherwise) rather than mean-pool. */ - int single_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN || cls_pool; + int single_token_pool = c->pooling_mode == FFWD_POOL_LAST_TOKEN || cls_pool; if (hidden <= 0 || c->vocab_size <= 0) return -1; @@ -2023,9 +2021,9 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, : (mlx_array){0}; mlx_array lengths = needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0}; - mlx_array last_indices = - single_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) - : (mlx_array){0}; + mlx_array last_indices = single_token_pool + ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32) + : (mlx_array){0}; free(padded_ids); free(pool_mask_data); free(attn_mask_data); @@ -2065,7 +2063,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, * already final, so pooling uses them directly. Qwen runs the decoder block * and applies the final RMSNorm here. */ mlx_array x_normed; - if (c->family == EMBED_FAMILY_BERT) { + if (c->family == FFWD_FAMILY_BERT) { int start2[] = {0, 0}, stride2[] = {1, 1}; int pos_start[] = {c->position_id_offset, 0}; int pos_stop[] = {c->position_id_offset + max_seq, hidden}, type_stop[] = {1, hidden}; @@ -2091,7 +2089,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array_free(xp); mlx_array_free(tt); mlx_array xe = mlx_array_new(); - mlx_fast_layer_norm(&xe, xt, ctx->embed_ln_w, ctx->embed_ln_b, c->layer_norm_eps, S); + mlx_fast_layer_norm(&xe, xt, ctx->ffwd_ln_w, ctx->ffwd_ln_b, c->layer_norm_eps, S); mlx_array_free(xt); x_normed = mlx_forward_bert(ctx, xe, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers); @@ -2149,7 +2147,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, if (c->normalize_embeddings) { rc = 0; for (int b = 0; b < batch; b++) { - if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) { + if (ffwd_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) { rc = -1; break; } @@ -2164,7 +2162,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, typedef struct { int index; - const embed_input_t *input; + const ffwd_input_t *input; } mlx_batch_item_t; static int mlx_batch_item_cmp(const void *a, const void *b) { @@ -2181,7 +2179,7 @@ static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) { return padded * 5 > tokens * 11; } -static int mlx_batch_should_split(const embed_input_t *inputs, +static int mlx_batch_should_split(const ffwd_input_t *inputs, int batch, int *out_total_tokens, int *out_max_seq) { @@ -2210,7 +2208,7 @@ static int mlx_batch_should_split(const embed_input_t *inputs, return mlx_padding_exceeds_limit(padded, (size_t)total_tokens); } -static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx, +static int mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx, int group_count, int group_max, int next_tokens) { @@ -2232,19 +2230,19 @@ static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx, return merged <= split_total; } -int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, - const embed_input_t *inputs, - int batch, - float *out_embeddings) { +int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx, + const ffwd_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; if (!mlx_batch_should_split(inputs, batch, NULL, NULL)) - return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings); + return ffwd_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings); const int hidden = ctx->config.hidden_size; mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items)); - embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); + ffwd_input_t *group_inputs = (ffwd_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); size_t group_values, group_bytes; if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 || mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { @@ -2290,7 +2288,7 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, int group_count = end - start; for (int i = 0; i < group_count; i++) group_inputs[i] = *items[start + i].input; - rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out); + rc = ffwd_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out); if (rc != 0) break; for (int i = 0; i < group_count; i++) { @@ -2307,18 +2305,18 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, return rc; } -int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, - const int *token_ids, - int n_tokens, - float *out_embedding) { - embed_input_t input = {token_ids, n_tokens}; - return embed_mlx_encode_batch(ctx, &input, 1, out_embedding); +int ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + float *out_embedding) { + ffwd_input_t input = {token_ids, n_tokens}; + return ffwd_mlx_encode_batch(ctx, &input, 1, out_embedding); } typedef struct { int index; int span_offset; - const embed_context_input_t *input; + const ffwd_context_input_t *input; } mlx_context_batch_item_t; static int mlx_context_batch_item_cmp(const void *a, const void *b) { @@ -2335,7 +2333,7 @@ static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) { return padded * 4 > tokens * 5; } -static int mlx_context_batch_should_split(const embed_context_input_t *inputs, +static int mlx_context_batch_should_split(const ffwd_context_input_t *inputs, int batch, int *out_total_spans) { int max_seq = 0; @@ -2363,22 +2361,22 @@ static int mlx_context_batch_should_split(const embed_context_input_t *inputs, return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens); } -static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, - int batch, - float *out_embeddings) { +static int ffwd_mlx_encode_spans_batch_dense(ffwd_mlx_ctx_t *ctx, + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 || ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm)) return -1; - const embed_config_t *c = &ctx->config; + const ffwd_config_t *c = &ctx->config; mlx_stream S = ctx->stream; int hidden = c->hidden_size; int max_seq = 0; int total_spans = 0; int has_padding = 0; for (int b = 0; b < batch; b++) { - const embed_context_input_t *input = &inputs[b]; + const ffwd_context_input_t *input = &inputs[b]; int n_tokens = input->input.n_tokens; if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 || total_spans > INT_MAX - input->n_spans) @@ -2424,7 +2422,7 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, return -1; } for (int b = 0; b < batch; b++) { - const embed_input_t *input = &inputs[b].input; + const ffwd_input_t *input = &inputs[b].input; for (int t = 0; t < max_seq; t++) { size_t row = (size_t)b * max_seq + t; if (t < input->n_tokens) { @@ -2513,7 +2511,7 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, int pooled_count = 0; for (int b = 0; b < batch; b++) { for (int s = 0; s < inputs[b].n_spans; s++) { - const embed_span_t *span = &inputs[b].spans[s]; + const ffwd_span_t *span = &inputs[b].spans[s]; int start[] = {b, span->start, 0}; int stop[] = {b + 1, span->start + span->n_tokens, hidden}; int strides[] = {1, 1, 1}; @@ -2551,7 +2549,7 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx, int span_count = 0; for (int b = 0; b < batch; b++) { for (int s = 0; s < inputs[b].n_spans; s++) { - const embed_span_t *span = &inputs[b].spans[s]; + const ffwd_span_t *span = &inputs[b].spans[s]; int base = b * max_seq; int start = span->start; int stop = span->start + span->n_tokens; @@ -2664,22 +2662,22 @@ cleanup: return rc; } -int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, - int batch, - float *out_embeddings) { +int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx, + const ffwd_context_input_t *inputs, + int batch, + float *out_embeddings) { if (!ctx || !inputs || batch <= 0 || !out_embeddings) return -1; int total_spans = 0; if (!mlx_context_batch_should_split(inputs, batch, &total_spans)) - return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings); + return ffwd_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings); const int hidden = ctx->config.hidden_size; mlx_context_batch_item_t *items = (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items)); - embed_context_input_t *group_inputs = - (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); + ffwd_context_input_t *group_inputs = + (ffwd_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs)); size_t group_values, group_bytes; if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 || mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) { @@ -2729,15 +2727,15 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, int group_count = end - start; for (int i = 0; i < group_count; i++) group_inputs[i] = *items[start + i].input; - rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out); + rc = ffwd_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out); if (rc != 0) break; int src_span = 0; for (int i = 0; i < group_count; i++) { int n_spans = group_inputs[i].n_spans; int dst_span = items[start + i].span_offset; - memcpy(out_embeddings + (size_t)dst_span * hidden, - group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float)); + memcpy(out_embeddings + (size_t)dst_span * hidden, group_out + (size_t)src_span * hidden, + (size_t)n_spans * hidden * sizeof(float)); src_span += n_spans; } start = end; @@ -2749,21 +2747,21 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, return rc; } -int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, - const int *token_ids, - int n_tokens, - const embed_span_t *spans, - int n_spans, - float *out_embeddings) { - embed_context_input_t input = { +int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx, + const int *token_ids, + int n_tokens, + const ffwd_span_t *spans, + int n_spans, + float *out_embeddings) { + ffwd_context_input_t input = { .input = {token_ids, n_tokens}, .spans = spans, .n_spans = n_spans, }; - return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings); + return ffwd_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings); } -float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { +float *ffwd_mlx_encode(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) { if (!ctx || !token_ids || n_tokens <= 0) return NULL; @@ -2775,7 +2773,7 @@ float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens if (!out) return NULL; - if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) { + if (ffwd_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) { free(out); return NULL; } diff --git a/src/mlx.h b/libffwd/mlx.h index 55f5666..d8a915f 100644 --- a/src/mlx.h +++ b/libffwd/mlx.h @@ -1,59 +1,59 @@ /* - * embed_mlx.h - MLX GPU backend for Apple Silicon + * ffwd_mlx.h - MLX GPU backend for Apple Silicon * * Uses Apple's mlx-c to run the transformer on Metal. */ -#ifndef EMBED_MLX_H -#define EMBED_MLX_H +#ifndef FFWD_MLX_H +#define FFWD_MLX_H /* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying EMBED_API are exported + * -fvisibility=hidden, so only declarations carrying FFWD_API are exported * from the shared library; everything else stays internal. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif +#ifndef FFWD_API +# if defined(__GNUC__) +# define FFWD_API __attribute__((visibility("default"))) +# else +# define FFWD_API +# endif #endif #include <stddef.h> #include <stdint.h> -#include "embed.h" +#include "ffwd.h" /* Opaque MLX model context */ -typedef struct embed_mlx_ctx embed_mlx_ctx_t; -typedef struct embed_mlx_late_ctx embed_mlx_late_ctx_t; -typedef struct embed_mlx_late_vectors embed_mlx_late_vectors_t; +typedef struct ffwd_mlx_ctx ffwd_mlx_ctx_t; +typedef struct ffwd_mlx_late_ctx ffwd_mlx_late_ctx_t; +typedef struct ffwd_mlx_late_vectors ffwd_mlx_late_vectors_t; typedef struct { int quantize_bits; /* 0 disables; currently 8 */ int quantize_group_size; /* default 64 when quantize_bits is set */ -} embed_mlx_options_t; +} ffwd_mlx_options_t; /* * Load model into MLX arrays from safetensors + config.json. * Returns NULL on error. */ -EMBED_API embed_mlx_ctx_t *embed_mlx_load(const char *model_dir); -EMBED_API embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir, - const embed_mlx_options_t *opts); +FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir); +FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir, + const ffwd_mlx_options_t *opts); -EMBED_API void embed_mlx_free(embed_mlx_ctx_t *ctx); +FFWD_API void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx); /* * Late-interaction MLX path. This is separate from pooled embeddings because * late snapshots produce one projected token vector per input token for MaxSim * scoring, not one pooled document vector. */ -EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir); -EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir, - const embed_mlx_options_t *opts); -EMBED_API void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx); +FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load(const char *model_dir); +FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir, + const ffwd_mlx_options_t *opts); +FFWD_API void ffwd_mlx_late_free(ffwd_mlx_late_ctx_t *ctx); -EMBED_API const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx); -EMBED_API int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx); +FFWD_API const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx); +FFWD_API int ffwd_mlx_late_token_dim(const ffwd_mlx_late_ctx_t *ctx); /* * Encode token_ids into out_vectors[n_tokens, token_dim]. @@ -61,11 +61,8 @@ EMBED_API int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx); * If normalize is non-zero, every token vector is L2-normalized on device * before copying back to CPU memory. */ -EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors); +FFWD_API int ffwd_mlx_late_encode_tokens( + ffwd_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, float *out_vectors); /* * Device-resident late-interaction token vectors. @@ -74,22 +71,22 @@ EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx, * outlive ctx. Copying is optional: MaxSim can consume these handles directly * and copy back only final candidate scores. */ -EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx, +FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize); -EMBED_API void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs); -EMBED_API int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs); -EMBED_API int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs); -EMBED_API int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors); +FFWD_API void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs); +FFWD_API int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs); +FFWD_API int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs); +FFWD_API int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors); /* * Encode n_docs documents in one padded transformer pass and pack each * document's kept token vectors back-to-back into [total_keep, token_dim] in * document order. out_offsets[n_docs + 1] receives the prefix sum of kept - * counts, so the result feeds embed_mlx_late_maxsim_batch_device directly. + * counts, so the result feeds ffwd_mlx_late_maxsim_batch_device directly. * Returns NULL on error; the handle must not outlive ctx. */ -EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx, +FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx, const int *const *doc_ids, const int *n_tokens, const int *const *keep, @@ -98,18 +95,18 @@ EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_ int normalize, int *out_offsets); /* Device-resident select/concat primitives, retained for the late-interaction - * verification harness (scripts/check_late_interaction.py); the server rerank - * path uses embed_mlx_late_encode_docs_device. */ -EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat( - embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count); -EMBED_API embed_mlx_late_vectors_t * -embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx, - const embed_mlx_late_vectors_t *vecs, + * verification harness (tests/integration/check_late_interaction.py); the server rerank + * path uses ffwd_mlx_late_encode_docs_device. */ +FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat( + ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count); +FFWD_API ffwd_mlx_late_vectors_t * +ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx, + const ffwd_mlx_late_vectors_t *vecs, const int *token_indices, int count); -EMBED_API int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx, - const embed_mlx_late_vectors_t *query, - const embed_mlx_late_vectors_t *docs, +FFWD_API int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx, + const ffwd_mlx_late_vectors_t *query, + const ffwd_mlx_late_vectors_t *docs, const int *doc_offsets, int docs_count, float *scores); @@ -118,16 +115,14 @@ EMBED_API int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx, * Compute embedding for token_ids[0..n_tokens-1]. * Returns malloc'd float[hidden_size] (caller frees). NULL on error. */ -EMBED_API float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens); +FFWD_API float *ffwd_mlx_encode(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens); /* * Compute one embedding into caller-provided out[hidden_size]. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, - const int *token_ids, - int n_tokens, - float *out_embedding); +FFWD_API int +ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding); /* * Compute a true padded dense batch on MLX. @@ -136,8 +131,8 @@ EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx, * Pooling and attention masking follow the loaded model configuration. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, - const embed_input_t *inputs, +FFWD_API int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx, + const ffwd_input_t *inputs, int batch, float *out_embeddings); @@ -145,10 +140,10 @@ EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx, * Run one contextual sequence and pool selected token spans. * Returns 0 on success, -1 on error. */ -EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, +FFWD_API int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, - const embed_span_t *spans, + const ffwd_span_t *spans, int n_spans, float *out_embeddings); @@ -156,12 +151,12 @@ EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx, * Run a padded dense contextual document batch and pool every selected span. * out_embeddings contains spans in document order. */ -EMBED_API int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx, - const embed_context_input_t *inputs, +FFWD_API int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx, + const ffwd_context_input_t *inputs, int batch, float *out_embeddings); -/* Get the config (so embed_cli.c can read hidden_size etc.) */ -EMBED_API const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx); +/* Get the config (so ffwd_cli.c can read hidden_size etc.) */ +FFWD_API const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx); -#endif /* EMBED_MLX_H */ +#endif /* FFWD_MLX_H */ diff --git a/src/model.c b/libffwd/model.c index c39c526..faecfd4 100644 --- a/src/model.c +++ b/libffwd/model.c @@ -1,19 +1,12 @@ -#include "embed.h" +#include "ffwd.h" #include "model_internal.h" #include "config.h" -#include "kernels.h" +#include "dtype.h" #include <stdio.h> #include <stdlib.h> #include <stdint.h> -#include <math.h> -#include <limits.h> - -/* ======================================================================== - * Globals - * ======================================================================== */ - -int embed_verbose = 0; +#include <string.h> /* ======================================================================== * Model-directory probes @@ -81,12 +74,12 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, * to F32 by the caller (BF16 lazily per matmul, F16 once at load). */ int sixteen_ok = bf16_ok && (t->dtype == DTYPE_BF16 || t->dtype == DTYPE_F16); if (t->dtype != DTYPE_F32 && !sixteen_ok) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name, + fprintf(stderr, "ffwd: unsupported dtype for %s: got %s, expected %s\n", name, dtype_name(t->dtype), bf16_ok ? "F32, BF16, or F16" : "F32"); return 0; } if (t->ndim != ndim) { - fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); + fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); return 0; } @@ -94,13 +87,13 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, for (int i = 0; i < ndim; i++) { if (t->shape[i] != shape[i]) { fprintf(stderr, - "embed: bad shape for %s at dim %d: " + "ffwd: bad shape for %s at dim %d: " "got %lld, expected %lld\n", name, i, (long long)t->shape[i], (long long)shape[i]); return 0; } if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "embed: shape too large for %s\n", name); + fprintf(stderr, "ffwd: shape too large for %s\n", name); return 0; } numel *= (size_t)shape[i]; @@ -108,17 +101,17 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, size_t elem_size = dtype_size(t->dtype); if (elem_size == 0) { - fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype)); + fprintf(stderr, "ffwd: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype)); return 0; } size_t bytes; if (mul_size(numel, elem_size, &bytes) != 0) { - fprintf(stderr, "embed: tensor data too large for %s\n", name); + fprintf(stderr, "ffwd: tensor data too large for %s\n", name); return 0; } if (t->data_size != bytes) { - fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, + fprintf(stderr, "ffwd: bad data size for %s: got %zu, expected %zu\n", name, t->data_size, bytes); return 0; } @@ -126,14 +119,47 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, size_t data_start = 8 + sf->header_size; if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "embed: tensor data out of bounds for %s\n", name); + fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name); return 0; } return 1; } -embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t) { - embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0}; +/* Widen a safetensors tensor to f32 (caller frees). libsafetensors is a pure + * reader now, so the dtype conversion lives here, next to its only callers. */ +static float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { + int64_t n = safetensor_numel(t); + if (n <= 0) + return NULL; + float *out = malloc((size_t)n * sizeof(float)); + if (!out) + return NULL; + const void *data = safetensors_data(sf, t); + switch (t->dtype) { + case DTYPE_F32: + memcpy(out, data, (size_t)n * sizeof(float)); + break; + case DTYPE_BF16: { + const uint16_t *s = (const uint16_t *)data; + for (int64_t i = 0; i < n; i++) + out[i] = ffwd_bf16_to_f32(s[i]); + break; + } + case DTYPE_F16: { + const uint16_t *s = (const uint16_t *)data; + for (int64_t i = 0; i < n; i++) + out[i] = ffwd_f16_to_f32(s[i]); + break; + } + default: + free(out); + return NULL; + } + return out; +} + +ffwd_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t) { + ffwd_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0}; /* F32/BF16 are consumed directly by the forward pass, so the ref borrows the * mmap (zero copy; BF16 is widened lazily per matmul). F16 has neither a * fused kernel nor a zero-copy F32 view, so widen it once here into an owned @@ -152,13 +178,13 @@ embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const sa return ref; } -static embed_weight_ref_t +static ffwd_weight_ref_t load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { - embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0}; + ffwd_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0}; safetensors_file_t *sf = NULL; const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { - fprintf(stderr, "embed: tensor not found: %s\n", name); + fprintf(stderr, "ffwd: tensor not found: %s\n", name); return ref; } if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) @@ -172,7 +198,7 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i safetensors_file_t *sf = NULL; const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { - fprintf(stderr, "embed: tensor not found: %s\n", name); + fprintf(stderr, "ffwd: tensor not found: %s\n", name); return NULL; } if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) @@ -186,13 +212,13 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i /* BERT-family weight loader. The tensor names differ entirely from the Qwen * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry - * point. Reused embed_layer_t slots carry their BERT meaning: wo is the + * point. Reused ffwd_layer_t slots carry their BERT meaning: wo is the * attention output dense, input_norm/post_attn_norm are the two block LayerNorm - * weights, up_proj/down_proj are the two FFN matrices (see embed_internal.h). */ + * weights, up_proj/down_proj are the two FFN matrices (see internal.h). */ static int -load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { - const embed_config_t *cfg = &model->config; - embed_weights_t *w = &model->weights; +load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) { + const ffwd_config_t *cfg = &model->config; + ffwd_weights_t *w = &model->weights; char name[256]; /* sentence-transformers checkpoints store encoder tensors unprefixed; raw @@ -205,7 +231,7 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start else if (multi_safetensors_find(ms, "roberta.embeddings.word_embeddings.weight", NULL)) p = "roberta."; else { - fprintf(stderr, "embed_model_load: BERT-family word embeddings not found\n"); + fprintf(stderr, "ffwd_model_load: BERT-family word embeddings not found\n"); return -1; } @@ -223,20 +249,20 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2); const int64_t ln_shape[1] = {hidden}; snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p); - w->embed_ln_w = load_norm_f32(ms, name, ln_shape, 1); + w->ffwd_ln_w = load_norm_f32(ms, name, ln_shape, 1); snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p); - w->embed_ln_b = load_norm_f32(ms, name, ln_shape, 1); + w->ffwd_ln_b = load_norm_f32(ms, name, ln_shape, 1); if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || - !w->embed_ln_w || !w->embed_ln_b) + !w->ffwd_ln_w || !w->ffwd_ln_b) return -1; } - w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); + w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t)); if (!w->layers) return -1; for (int i = layer_start; i < layer_end; i++) { - embed_layer_t *l = &w->layers[i]; + ffwd_layer_t *l = &w->layers[i]; #define BERT_W(field, suffix, d0, d1) \ do { \ @@ -279,31 +305,31 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start return 0; } -embed_model_t * +ffwd_model_t * model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) { if (!allow_late && model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "embed_model_load: late-interaction models require " + fprintf(stderr, "ffwd_model_load: late-interaction models require " "token-level MaxSim support and are not valid pooled " "embedding models yet\n"); return NULL; } - embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t)); + ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t)); if (!model) return NULL; /* Parse config.json */ - if (embed_config_parse(&model->config, model_dir) != 0) { + if (ffwd_config_parse(&model->config, model_dir) != 0) { free(model); return NULL; } - const embed_config_t *cfg = &model->config; + const ffwd_config_t *cfg = &model->config; if (layer_end == -1) layer_end = cfg->n_layers; if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) { fprintf(stderr, - "embed_model_load: invalid layer range [%d, %d) " + "ffwd_model_load: invalid layer range [%d, %d) " "for %d-layer model\n", layer_start, layer_end, cfg->n_layers); free(model); @@ -311,42 +337,39 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a } model->layer_start = layer_start; model->layer_end = layer_end; - model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL - ? embed_causal_gqa_attention_packed_with_scratch - : embed_bidirectional_gqa_attention_packed_with_scratch; /* Open safetensors (handles single file or multi-shard) */ multi_safetensors_t *ms = multi_safetensors_open(model_dir); if (!ms) { - fprintf(stderr, "embed_model_load: failed to open safetensors in %s\n", model_dir); + fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir); free(model); return NULL; } model->safetensors = ms; - if (cfg->family == EMBED_FAMILY_BERT) { + if (cfg->family == FFWD_FAMILY_BERT) { if (load_bert_weights(model, ms, layer_start, layer_end) != 0) goto fail; - if (embed_verbose >= 1) - fprintf(stderr, "embed_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start, + if (ffwd_verbose >= 1) + fprintf(stderr, "ffwd_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, cfg->hidden_size); return model; } const char *weight_prefix = multi_safetensors_weight_prefix(ms, "embed_tokens.weight"); if (!weight_prefix) { - fprintf(stderr, "embed_model_load: neither embed_tokens.weight nor " + fprintf(stderr, "ffwd_model_load: neither embed_tokens.weight nor " "model.embed_tokens.weight was found\n"); goto fail; } - if (embed_verbose >= 1) - fprintf(stderr, "embed_model_load: loading weights from %s\n", model_dir); + if (ffwd_verbose >= 1) + fprintf(stderr, "ffwd_model_load: loading weights from %s\n", model_dir); char name[256]; /* Only the first stage needs token embeddings. */ - embed_weights_t *w = &model->weights; + ffwd_weights_t *w = &model->weights; if (layer_start == 0) { const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size}; snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix); @@ -356,13 +379,13 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a } /* Allocate layer array */ - w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t)); + w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t)); if (!w->layers) goto fail; /* Per-layer weights */ for (int i = layer_start; i < layer_end; i++) { - embed_layer_t *l = &w->layers[i]; + ffwd_layer_t *l = &w->layers[i]; #define LOAD_NORM(field, fmt, d0) \ do { \ @@ -406,7 +429,7 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a #undef LOAD2 #undef LOAD_NORM - if (embed_verbose >= 2) + if (ffwd_verbose >= 2) fprintf(stderr, " layer %d loaded\n", i); } @@ -419,38 +442,38 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a goto fail; } - if (embed_verbose >= 1) + if (ffwd_verbose >= 1) fprintf(stderr, - "embed_model_load: layers [%d, %d) loaded " + "ffwd_model_load: layers [%d, %d) loaded " "(%d-dim embeddings)\n", layer_start, layer_end, cfg->hidden_size); return model; fail: - embed_model_free(model); + ffwd_model_free(model); return NULL; } -embed_model_t *embed_model_load(const char *model_dir) { +ffwd_model_t *ffwd_model_load(const char *model_dir) { return model_load_range_ex(model_dir, 0, -1, 0); } /* Release a weight ref's backing store. Only F16-widened refs own their buffer; * borrowed F32/BF16 views into the mmap are a no-op (see weight_ref_from_tensor). */ -static void free_weight_ref(embed_weight_ref_t *w) { +static void free_weight_ref(ffwd_weight_ref_t *w) { if (w->owned) free((void *)w->data); w->data = NULL; w->owned = 0; } -void embed_model_free(embed_model_t *model) { +void ffwd_model_free(ffwd_model_t *model) { if (!model) return; if (model->weights.layers) { for (int i = 0; i < model->config.n_layers; i++) { - embed_layer_t *l = &model->weights.layers[i]; + ffwd_layer_t *l = &model->weights.layers[i]; free_weight_ref(&l->wq); free_weight_ref(&l->wk); free_weight_ref(&l->wv); @@ -476,213 +499,14 @@ void embed_model_free(embed_model_t *model) { free((void *)model->weights.norm); free((void *)model->weights.position_embeddings); free((void *)model->weights.token_type_embeddings); - free((void *)model->weights.embed_ln_w); - free((void *)model->weights.embed_ln_b); + free((void *)model->weights.ffwd_ln_w); + free((void *)model->weights.ffwd_ln_b); if (model->safetensors) multi_safetensors_close((multi_safetensors_t *)model->safetensors); free(model->weights.layers); free(model); } -embed_workspace_t *embed_workspace_new(const embed_model_t *model) { - if (!model) - return NULL; - embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t)); - if (!ws) - return NULL; - ws->model = model; - return ws; -} - -void embed_workspace_free(embed_workspace_t *ws) { - if (!ws) - return; - free(ws->x); - free(ws->x_norm); - free(ws->q); - free(ws->k); - free(ws->v); - free(ws->attn_out); - free(ws->proj_out); - free(ws->ffn_gate); - free(ws->ffn_up); - free(ws->attn_scores); - free(ws->offsets); - free(ws->rope_cos); - free(ws->rope_sin); - free(ws->bf16_widen); - free(ws); -} - -const embed_config_t *embed_model_config(const embed_model_t *model) { +const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) { return model ? &model->config : NULL; } - -/* ======================================================================== - * Workspace buffer growth - * - * Lazy per-call growth of the forward-pass scratch. Lives here, beside the - * workspace lifecycle and embed_workspace_nbytes below: nbytes reproduces these - * allocation sizes, so the allocator and the accountant stay in one file. - * ======================================================================== */ - -int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) { - if (seq <= ws->buf_seq_cap) - return 0; - - int cap; - if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) - return -1; - -#define R(ptr, n) \ - do { \ - if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ - return -1; \ - } while (0) - - R(ws->x, c->hidden_size); - R(ws->x_norm, c->hidden_size); - R(ws->q, c->q_dim); - R(ws->k, c->kv_dim); - R(ws->v, c->kv_dim); - R(ws->attn_out, c->q_dim); - R(ws->proj_out, c->hidden_size); - R(ws->ffn_gate, c->intermediate_size); - R(ws->ffn_up, c->intermediate_size); - -#undef R - - ws->buf_seq_cap = cap; - return 0; -} - -void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) { - size_t bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (bytes <= ws->attn_scores_bytes) - return; - - void *p = realloc(ws->attn_scores, bytes); - if (!p) - return; - ws->attn_scores = (float *)p; - ws->attn_scores_bytes = bytes; -} - -int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) { - if (n_pos <= ws->rope_cache_cap) - return 0; - - int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; - while (cap < n_pos) { - if (cap > INT_MAX / 2) - return -1; - cap *= 2; - } - - int head_dim = cfg->head_dim; - int half = head_dim / 2; - float theta = cfg->rope_theta; - - size_t count; - if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) - return -1; - if (realloc_floats(&ws->rope_cos, count) != 0) - return -1; - if (realloc_floats(&ws->rope_sin, count) != 0) - return -1; - - for (int pos = ws->rope_cache_cap; pos < cap; pos++) { - float fp = (float)pos; - float *c = ws->rope_cos + (size_t)pos * head_dim; - float *s = ws->rope_sin + (size_t)pos * head_dim; - for (int d = 0; d < half; d++) { - float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); - float angle = fp * inv; - float cv = cosf(angle), sv = sinf(angle); - c[d] = cv; - c[half + d] = cv; - s[d] = sv; - s[half + d] = sv; - } - } - - ws->rope_cache_cap = cap; - return 0; -} - -int ensure_offsets(embed_workspace_t *ws, int batch) { - if (!ws || batch <= 0 || batch == INT_MAX) - return -1; - - int needed = batch + 1; - if (needed <= ws->offsets_cap) - return 0; - - if (realloc_ints(&ws->offsets, (size_t)needed) != 0) - return -1; - ws->offsets_cap = needed; - return 0; -} - -size_t embed_workspace_nbytes(const embed_workspace_t *ws) { - if (!ws) - return 0; - - size_t total = sizeof(*ws); - const embed_model_t *model = ws->model; - if (!model) - return total; - - const embed_config_t *c = &model->config; - size_t row_floats = 0; - const int dims[] = { - c->hidden_size, /* x */ - c->hidden_size, /* x_norm */ - c->q_dim, /* q */ - c->kv_dim, /* k */ - c->kv_dim, /* v */ - c->q_dim, /* attn_out */ - c->hidden_size, /* proj_out */ - c->intermediate_size, /* ffn_gate */ - c->intermediate_size, /* ffn_up */ - }; - - for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { - if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0) - return SIZE_MAX; - } - - size_t buf_floats = 0; - if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) - return SIZE_MAX; - - size_t rope_floats = 0; - if (ws->rope_cache_cap > 0) { - size_t rope_rows = 0; - if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 || - mul_size(rope_rows, 2, &rope_floats) != 0) - return SIZE_MAX; - } - - size_t all_floats = 0, bytes = 0; - if (add_size(buf_floats, rope_floats, &all_floats) != 0 || - mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0) - return SIZE_MAX; - - size_t offset_bytes = 0; - if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) - return SIZE_MAX; - if (add_size(total, offset_bytes, &total) != 0) - return SIZE_MAX; - if (add_size(total, ws->attn_scores_bytes, &total) != 0) - return SIZE_MAX; - - size_t widen_bytes = 0; - if (ws->bf16_widen_count > 0 && - mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0) - return SIZE_MAX; - if (add_size(total, widen_bytes, &total) != 0) - return SIZE_MAX; - - return total; -} diff --git a/libffwd/model_blas.c b/libffwd/model_blas.c new file mode 100644 index 0000000..aae2a44 --- /dev/null +++ b/libffwd/model_blas.c @@ -0,0 +1,201 @@ +#include "ffwd.h" +#include "model_internal.h" +#include "kernels.h" + +#include <limits.h> +#include <math.h> +#include <stdint.h> +#include <stdlib.h> + +/* CPU workspace ownership and scratch growth. GPU builds use their own + * backend contexts and do not link this translation unit. */ + +ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model) { + if (!model) + return NULL; + ffwd_workspace_t *ws = (ffwd_workspace_t *)calloc(1, sizeof(ffwd_workspace_t)); + if (!ws) + return NULL; + ws->model = model; + return ws; +} + +void ffwd_workspace_free(ffwd_workspace_t *ws) { + if (!ws) + return; + free(ws->x); + free(ws->x_norm); + free(ws->q); + free(ws->k); + free(ws->v); + free(ws->attn_out); + free(ws->proj_out); + free(ws->ffn_gate); + free(ws->ffn_up); + free(ws->attn_scores); + free(ws->offsets); + free(ws->rope_cos); + free(ws->rope_sin); + free(ws->bf16_widen); + free(ws); +} + +int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) { + if (seq <= ws->buf_seq_cap) + return 0; + + int cap; + if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) + return -1; + +#define R(ptr, n) \ + do { \ + if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \ + return -1; \ + } while (0) + + R(ws->x, c->hidden_size); + R(ws->x_norm, c->hidden_size); + R(ws->q, c->q_dim); + R(ws->k, c->kv_dim); + R(ws->v, c->kv_dim); + R(ws->attn_out, c->q_dim); + R(ws->proj_out, c->hidden_size); + R(ws->ffn_gate, c->intermediate_size); + R(ws->ffn_up, c->intermediate_size); + +#undef R + + ws->buf_seq_cap = cap; + return 0; +} + +void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch) { + size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + if (bytes <= ws->attn_scores_bytes) + return; + + void *p = realloc(ws->attn_scores, bytes); + if (!p) + return; + ws->attn_scores = (float *)p; + ws->attn_scores_bytes = bytes; +} + +int ensure_rope_cache(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int n_pos) { + if (n_pos <= ws->rope_cache_cap) + return 0; + + int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512; + while (cap < n_pos) { + if (cap > INT_MAX / 2) + return -1; + cap *= 2; + } + + int head_dim = cfg->head_dim; + int half = head_dim / 2; + float theta = cfg->rope_theta; + + size_t count; + if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0) + return -1; + if (realloc_floats(&ws->rope_cos, count) != 0) + return -1; + if (realloc_floats(&ws->rope_sin, count) != 0) + return -1; + + for (int pos = ws->rope_cache_cap; pos < cap; pos++) { + float fp = (float)pos; + float *c = ws->rope_cos + (size_t)pos * head_dim; + float *s = ws->rope_sin + (size_t)pos * head_dim; + for (int d = 0; d < half; d++) { + float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = fp * inv; + float cv = cosf(angle), sv = sinf(angle); + c[d] = cv; + c[half + d] = cv; + s[d] = sv; + s[half + d] = sv; + } + } + + ws->rope_cache_cap = cap; + return 0; +} + +int ensure_offsets(ffwd_workspace_t *ws, int batch) { + if (!ws || batch <= 0 || batch == INT_MAX) + return -1; + + int needed = batch + 1; + if (needed <= ws->offsets_cap) + return 0; + + if (realloc_ints(&ws->offsets, (size_t)needed) != 0) + return -1; + ws->offsets_cap = needed; + return 0; +} + +size_t ffwd_workspace_nbytes(const ffwd_workspace_t *ws) { + if (!ws) + return 0; + + size_t total = sizeof(*ws); + const ffwd_model_t *model = ws->model; + if (!model) + return total; + + const ffwd_config_t *c = &model->config; + size_t row_floats = 0; + const int dims[] = { + c->hidden_size, /* x */ + c->hidden_size, /* x_norm */ + c->q_dim, /* q */ + c->kv_dim, /* k */ + c->kv_dim, /* v */ + c->q_dim, /* attn_out */ + c->hidden_size, /* proj_out */ + c->intermediate_size, /* ffn_gate */ + c->intermediate_size, /* ffn_up */ + }; + + for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) { + if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0) + return SIZE_MAX; + } + + size_t buf_floats = 0; + if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0) + return SIZE_MAX; + + size_t rope_floats = 0; + if (ws->rope_cache_cap > 0) { + size_t rope_rows = 0; + if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 || + mul_size(rope_rows, 2, &rope_floats) != 0) + return SIZE_MAX; + } + + size_t all_floats = 0, bytes = 0; + if (add_size(buf_floats, rope_floats, &all_floats) != 0 || + mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0) + return SIZE_MAX; + + size_t offset_bytes = 0; + if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0) + return SIZE_MAX; + if (add_size(total, offset_bytes, &total) != 0) + return SIZE_MAX; + if (add_size(total, ws->attn_scores_bytes, &total) != 0) + return SIZE_MAX; + + size_t widen_bytes = 0; + if (ws->bf16_widen_count > 0 && mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0) + return SIZE_MAX; + if (add_size(total, widen_bytes, &total) != 0) + return SIZE_MAX; + + return total; +} diff --git a/src/model_internal.h b/libffwd/model_internal.h index acbba07..97de064 100644 --- a/src/model_internal.h +++ b/libffwd/model_internal.h @@ -1,13 +1,13 @@ -#ifndef EMBED_MODEL_INTERNAL_H -#define EMBED_MODEL_INTERNAL_H +#ifndef FFWD_MODEL_INTERNAL_H +#define FFWD_MODEL_INTERNAL_H -#include "internal.h" +#include "model_types.h" #include "alloc.h" /* Cross-file function declarations private to the CPU model code. The companion - * to internal.h: that header holds the shared struct definitions, this one the - * shared functions. None are part of the public API (embed.h) and none carry - * EMBED_API visibility; they are shared between the translation units (model + * to model_types.h: that header holds the shared struct definitions, this one the + * shared functions. None are part of the public API (ffwd.h) and none carry + * FFWD_API visibility; they are shared between the translation units (model * loading, the forward pass, and late interaction) that used to live in one * file. The frontends and the GPU backends do not include this header. */ @@ -25,37 +25,37 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, * widened to an owned F32 buffer (no fused F16 kernel). Shared by the model * loader and the late-interaction projection loader; the owner frees ref.data * when ref.owned is set. Returns ref.data == NULL on allocation failure. */ -embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t); +ffwd_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t); /* True if model_dir carries a 1_Dense projection (a late-interaction model). */ int model_dir_has_late_projection(const char *model_dir); /* Load (a layer range of) a model. allow_late lets the late loader accept a * 1_Dense snapshot that the pooled loader rejects. layer_end == -1 means all. */ -embed_model_t * +ffwd_model_t * model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late); /* Workspace scratch growth, lazy per forward call. Defined in model.c beside the - * workspace lifecycle (embed_workspace_nbytes mirrors their sizing); called from + * workspace lifecycle (ffwd_workspace_nbytes mirrors their sizing); called from * the forward pass in forward.c. */ -int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq); -void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch); -int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos); -int ensure_offsets(embed_workspace_t *ws, int batch); +int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq); +void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch); +int ensure_rope_cache(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int n_pos); +int ensure_offsets(ffwd_workspace_t *ws, int batch); /* y[seq,out] = x[seq,in] @ w^T, no bias; dispatches F32/BF16 paths. Shared by * the forward pass and the late projection. */ -void linear_nobias_weight(embed_workspace_t *ws, +void linear_nobias_weight(ffwd_workspace_t *ws, float *y, const float *x, - const embed_weight_ref_t *w, + const ffwd_weight_ref_t *w, int seq_len, int in_dim, int out_dim); /* Compute packed sequence offsets [batch+1] plus total and max sequence length * over a batch of inputs. require_ids rejects inputs with a NULL id pointer. */ -int build_offsets(const embed_input_t *inputs, +int build_offsets(const ffwd_input_t *inputs, int batch, int require_ids, int *offsets, @@ -64,13 +64,13 @@ int build_offsets(const embed_input_t *inputs, /* Run the full transformer forward over a packed batch in ws, leaving hidden * states in ws->x. Used by pooled encoding and by late-interaction encoding. */ -int forward_packed_inplace(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, +int forward_packed_inplace(const ffwd_model_t *model, + ffwd_workspace_t *ws, + const ffwd_input_t *inputs, int batch, const int *offsets, int total_seq, int max_seq, int apply_final_norm); -#endif /* EMBED_MODEL_INTERNAL_H */ +#endif /* FFWD_MODEL_INTERNAL_H */ diff --git a/src/internal.h b/libffwd/model_types.h index c1429de..2719a5b 100644 --- a/src/internal.h +++ b/libffwd/model_types.h @@ -1,7 +1,15 @@ -#ifndef EMBED_INTERNAL_H -#define EMBED_INTERNAL_H +/* + * model_types.h - in-memory layout of a loaded model: the model and workspace + * structs, the per-layer weight references, and the accessors that expose them + * to the GPU backends. Shared by the CPU model code and the CUDA backend; not + * part of the public ABI (ffwd.h). Companion to model_internal.h, which + * declares the CPU model's private functions. + */ -#include "embed.h" +#ifndef FFWD_MODEL_TYPES_H +#define FFWD_MODEL_TYPES_H + +#include "internal.h" #include "safetensors.h" typedef struct { @@ -11,13 +19,13 @@ typedef struct { * case). 1: data is a heap buffer owned by the model and freed on unload. * Set when an F16 tensor is widened to F32 at load (no fused F16 kernel). */ int owned; -} embed_weight_ref_t; +} ffwd_weight_ref_t; typedef struct { - embed_weight_ref_t wq; /* [q_dim, hidden] */ - embed_weight_ref_t wk; /* [kv_dim, hidden] */ - embed_weight_ref_t wv; /* [kv_dim, hidden] */ - embed_weight_ref_t wo; /* [hidden, q_dim] */ + ffwd_weight_ref_t wq; /* [q_dim, hidden] */ + ffwd_weight_ref_t wk; /* [kv_dim, hidden] */ + ffwd_weight_ref_t wv; /* [kv_dim, hidden] */ + ffwd_weight_ref_t wo; /* [hidden, q_dim] */ const float *q_bias; /* [q_dim], optional */ const float *k_bias; /* [kv_dim], optional */ const float *v_bias; /* [kv_dim], optional */ @@ -25,9 +33,9 @@ typedef struct { const float *k_norm; /* [head_dim] */ const float *input_norm; /* [hidden] */ const float *post_attn_norm; /* [hidden] */ - embed_weight_ref_t gate_proj; /* [intermediate, hidden] */ - embed_weight_ref_t up_proj; /* [intermediate, hidden] */ - embed_weight_ref_t down_proj; /* [hidden, intermediate] */ + ffwd_weight_ref_t gate_proj; /* [intermediate, hidden] */ + ffwd_weight_ref_t up_proj; /* [intermediate, hidden] */ + ffwd_weight_ref_t down_proj; /* [hidden, intermediate] */ /* BERT family (all NULL for Qwen3). The attention projections reuse * wq/wk/wv/wo and q/k/v_bias; the two FFN matrices reuse up_proj * (intermediate.dense) and down_proj (output.dense); the two block @@ -38,21 +46,21 @@ typedef struct { const float *ffn_inter_bias; /* intermediate.dense bias [intermediate] */ const float *ffn_out_bias; /* output.dense bias [hidden] */ const float *ffn_ln_bias; /* post-FFN LayerNorm bias [hidden] */ -} embed_layer_t; +} ffwd_layer_t; typedef struct { - embed_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ - embed_layer_t *layers; /* [n_layers] heap-allocated */ + ffwd_weight_ref_t embed_tokens; /* [vocab_size, hidden] */ + ffwd_layer_t *layers; /* [n_layers] heap-allocated */ const float *norm; /* [hidden]; NULL for BERT (no final norm) */ /* BERT family (NULL for Qwen3): learned absolute position embeddings, * token-type (segment) embeddings, and the embedding LayerNorm. */ const float *position_embeddings; /* [max_position_embeddings, hidden] */ const float *token_type_embeddings; /* [type_vocab_size, hidden] */ - const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */ - const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */ -} embed_weights_t; + const float *ffwd_ln_w; /* embedding LayerNorm weight [hidden] */ + const float *ffwd_ln_b; /* embedding LayerNorm bias [hidden] */ +} ffwd_weights_t; -typedef void (*embed_attention_fn)(float *out, +typedef void (*ffwd_attention_fn)(float *out, const float *Q, const float *K, const float *V, @@ -65,17 +73,16 @@ typedef void (*embed_attention_fn)(float *out, float *scratch, size_t scratch_bytes); -struct embed_model { - embed_config_t config; - embed_weights_t weights; +struct ffwd_model { + ffwd_config_t config; + ffwd_weights_t weights; void *safetensors; /* multi_safetensors_t*, keeps mmap alive */ int layer_start; /* loaded transformer range, inclusive */ int layer_end; /* loaded transformer range, exclusive */ - embed_attention_fn attention; }; -struct embed_workspace { - const embed_model_t *model; /* immutable model this workspace belongs to */ +struct ffwd_workspace { + const ffwd_model_t *model; /* immutable model this workspace belongs to */ int buf_seq_cap; float *x; /* [seq, hidden] */ float *x_norm; /* [seq, hidden] */ @@ -107,8 +114,8 @@ struct embed_workspace { /* Internal accessors for the late-interaction model. GPU backends build a device * context from an already-loaded CPU late model because the model dir is a late * snapshot that the pooled loader rejects; these expose the base model and the - * projection weight without making struct embed_late_model public. */ -embed_model_t *embed_late_model_base(const embed_late_model_t *model); -const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model); + * projection weight without making struct ffwd_late_model public. */ +ffwd_model_t *ffwd_late_model_base(const ffwd_late_model_t *model); +const ffwd_weight_ref_t *ffwd_late_model_projection(const ffwd_late_model_t *model); -#endif /* EMBED_INTERNAL_H */ +#endif /* FFWD_MODEL_TYPES_H */ diff --git a/src/kernels_neon.c b/libffwd/neon.c index 3e2a423..7f44d82 100644 --- a/src/kernels_neon.c +++ b/libffwd/neon.c @@ -1,15 +1,15 @@ /* - * embed_kernels_neon.c - ARM NEON hot kernels + * neon.c - ARM NEON hot kernels */ -#include "kernels_impl.h" +#include "impl.h" #ifdef __ARM_NEON -#include <arm_neon.h> -#include <string.h> +# include <arm_neon.h> +# include <string.h> -void embed_bf16_matvec_fused_neon( +void bf16_matvec_fused_neon( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; @@ -115,65 +115,4 @@ void embed_bf16_matvec_fused_neon( } } -float embed_dot_f32_neon(const float *a, const float *b, int n) { - int i = 0; - float32x4_t acc0 = vdupq_n_f32(0.0f); - float32x4_t acc1 = vdupq_n_f32(0.0f); - for (; i + 8 <= n; i += 8) { - float32x4_t a0 = vld1q_f32(a + i); - float32x4_t b0 = vld1q_f32(b + i); - float32x4_t a1 = vld1q_f32(a + i + 4); - float32x4_t b1 = vld1q_f32(b + i + 4); - acc0 = vfmaq_f32(acc0, a0, b0); - acc1 = vfmaq_f32(acc1, a1, b1); - } - float sum = vaddvq_f32(vaddq_f32(acc0, acc1)); - for (; i < n; i++) - sum += a[i] * b[i]; - return sum; -} - -void embed_vec_scale_inplace_neon(float *dst, float scale, int n) { - int i = 0; - float32x4_t s = vdupq_n_f32(scale); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s)); - vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s)); - } - for (; i < n; i++) - dst[i] *= scale; -} - -void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) { - int i = 0; - float32x4_t a = vdupq_n_f32(alpha); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(d0, s0, a)); - vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a)); - } - for (; i < n; i++) - dst[i] += alpha * src[i]; -} - -void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n) { - int i = 0; - float32x4_t c = vdupq_n_f32(correction); - for (; i + 8 <= n; i += 8) { - float32x4_t d0 = vld1q_f32(dst + i); - float32x4_t s0 = vld1q_f32(src + i); - float32x4_t d1 = vld1q_f32(dst + i + 4); - float32x4_t s1 = vld1q_f32(src + i + 4); - vst1q_f32(dst + i, vfmaq_f32(s0, d0, c)); - vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c)); - } - for (; i < n; i++) - dst[i] = dst[i] * correction + src[i]; -} - #endif /* __ARM_NEON */ diff --git a/libffwd/norm.c b/libffwd/norm.c new file mode 100644 index 0000000..27fd4c4 --- /dev/null +++ b/libffwd/norm.c @@ -0,0 +1,142 @@ +/* + * norm.c - Normalization + */ + +#include "kernels.h" +#include "threadpool.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#ifdef USE_BLAS +# ifdef __APPLE__ +# include <Accelerate/Accelerate.h> +# else +# include <cblas.h> +# endif +#endif + +#ifndef M_PI +# define M_PI 3.14159265358979323846 +#endif + +/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the + * per-head RMSNorm used for QK-norm. */ + +/* ======================================================================== + * Normalization + * ======================================================================== */ + +/* Rows split across the pool only when the tensor is big enough to pay + * for the dispatch. */ +#define RMS_NORM_PARALLEL_ELEMS 262144 + +/* The sum of squares and the scaled write-back are plain loops: the compiler + * auto-vectorizes both in place, which is faster than reducing through a + * two-input dot kernel that reloads the row and is reduction-latency bound. */ +static void rms_norm_range( + float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) { + for (int s = start; s < end; s++) { + const float *x_row = x + (size_t)s * hidden; + float *out_row = out + (size_t)s * hidden; + + float sum_sq = 0.0f; + for (int i = 0; i < hidden; i++) + sum_sq += x_row[i] * x_row[i]; + float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps); + for (int i = 0; i < hidden; i++) + out_row[i] = x_row[i] * rms_inv * weight[i]; + } +} + +typedef struct { + float *out; + const float *x; + const float *weight; + int seq_len; + int hidden; + float eps; +} rms_norm_task_t; + +static void rms_norm_worker(int tid, int n_threads, void *arg) { + rms_norm_task_t *t = (rms_norm_task_t *)arg; + int chunk = (t->seq_len + n_threads - 1) / n_threads; + int start = tid * chunk; + int end = start + chunk; + if (end > t->seq_len) + end = t->seq_len; + if (start >= end) + return; + rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps); +} + +void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) { + if (seq_len <= 0 || hidden <= 0) + return; + + long long elems = (long long)seq_len * hidden; + if (tp_num_threads() > 1 && elems >= RMS_NORM_PARALLEL_ELEMS) { + rms_norm_task_t task = { + .out = out, + .x = x, + .weight = weight, + .seq_len = seq_len, + .hidden = hidden, + .eps = eps, + }; + parallel_for(rms_norm_worker, &task); + } else { + rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); + } +} + +void rms_norm_per_head( + float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) { + /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ + int hidden = n_heads * head_dim; + for (int s = 0; s < seq_len; s++) { + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + + float sum_sq = 0.0f; + for (int d = 0; d < head_dim; d++) + sum_sq += vec[d] * vec[d]; + float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps); + for (int d = 0; d < head_dim; d++) + vec[d] = vec[d] * rms_inv * weight[d]; + } + } +} + +/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each + * row's mean and variance are reduced before the row is rewritten. Scalar for + * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */ +void layer_norm(float *out, + const float *x, + const float *gamma, + const float *beta, + int seq_len, + int hidden, + float eps) { + if (seq_len <= 0 || hidden <= 0) + return; + float inv_h = 1.0f / (float)hidden; + for (int s = 0; s < seq_len; s++) { + const float *row = x + (size_t)s * hidden; + float *dst = out + (size_t)s * hidden; + float mean = 0.0f; + for (int d = 0; d < hidden; d++) + mean += row[d]; + mean *= inv_h; + float var = 0.0f; + for (int d = 0; d < hidden; d++) { + float diff = row[d] - mean; + var += diff * diff; + } + var *= inv_h; + float inv_std = 1.0f / sqrtf(var + eps); + for (int d = 0; d < hidden; d++) + dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; + } +} diff --git a/libffwd/platform.h b/libffwd/platform.h new file mode 100644 index 0000000..3c92c98 --- /dev/null +++ b/libffwd/platform.h @@ -0,0 +1,27 @@ +/* platform.h - compile-time backend selection. + * + * The build passes -DUSE_GPU for a GPU build. This header maps that flag and the + * host platform to a concrete backend: Apple Silicon uses MLX, any other GPU + * uses CUDA, and a non-GPU build is CPU. Use USE_GPU for the CPU/GPU split; + * USE_MLX and USE_CUDA only where a specific accelerator API is called. */ +#ifndef FFWD_PLATFORM_H +#define FFWD_PLATFORM_H + +#if defined(USE_GPU) +# if defined(__APPLE__) +# define USE_MLX 1 +# else +# define USE_CUDA 1 +# endif +#endif + +/* Hardware-class label for --build-info and the startup banner. */ +#if defined(USE_MLX) +# define FFWD_BACKEND_LABEL "Apple Silicon GPU" +#elif defined(USE_CUDA) +# define FFWD_BACKEND_LABEL "NVIDIA GPU" +#else +# define FFWD_BACKEND_LABEL "CPU" +#endif + +#endif /* FFWD_PLATFORM_H */ diff --git a/src/kernels_rope.c b/libffwd/rope.c index ef1709a..12057b2 100644 --- a/src/kernels_rope.c +++ b/libffwd/rope.c @@ -1,5 +1,9 @@ +/* + * rope.c - rotary position embeddings + */ + #include "kernels.h" -#include "kernels_impl.h" +#include "impl.h" #include <math.h> #include <stdlib.h> @@ -7,22 +11,22 @@ #if (defined(__AVX512F__) || defined(__AVX2__)) && \ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) -#include <immintrin.h> +# include <immintrin.h> #endif #if defined(__ARM_NEON) || defined(__ARM_NEON__) -#include <arm_neon.h> +# include <arm_neon.h> #endif #ifdef USE_BLAS -#ifdef __APPLE__ -#include <Accelerate/Accelerate.h> -#else -#include <cblas.h> -#endif +# ifdef __APPLE__ +# include <Accelerate/Accelerate.h> +# else +# include <cblas.h> +# endif #endif #ifndef M_PI -#define M_PI 3.14159265358979323846 +# define M_PI 3.14159265358979323846 #endif /* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ @@ -31,7 +35,7 @@ * Position Embeddings * ======================================================================== */ -void embed_compute_rope_neox( +void compute_rope_neox( float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { int half = head_dim / 2; @@ -51,7 +55,7 @@ void embed_compute_rope_neox( } } -void embed_apply_rope_neox( +void apply_rope_neox( float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) { /* * NeoX split-half style: diff --git a/src/safetensors.c b/libffwd/safetensors.c index 65d0d7c..0efbd57 100644 --- a/src/safetensors.c +++ b/libffwd/safetensors.c @@ -1,13 +1,13 @@ #include "safetensors.h" -#include "dtype.h" + +#include <dirent.h> +#include <fcntl.h> #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <fcntl.h> -#include <unistd.h> #include <sys/mman.h> #include <sys/stat.h> -#include <dirent.h> +#include <unistd.h> /* ======================================================================== * Minimal JSON parser for safetensors header @@ -343,40 +343,6 @@ int64_t safetensor_numel(const safetensor_t *t) { return n; } -float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { - int64_t n = safetensor_numel(t); - if (n <= 0) - return NULL; - - float *out = malloc(n * sizeof(float)); - if (!out) - return NULL; - - const void *data = safetensors_data(sf, t); - - switch (t->dtype) { - case DTYPE_F32: - memcpy(out, data, n * sizeof(float)); - break; - case DTYPE_BF16: { - const uint16_t *src = (const uint16_t *)data; - for (int64_t i = 0; i < n; i++) - out[i] = embed_bf16_to_f32(src[i]); - break; - } - case DTYPE_F16: { - const uint16_t *src = (const uint16_t *)data; - for (int64_t i = 0; i < n; i++) - out[i] = embed_f16_to_f32(src[i]); - break; - } - default: - free(out); - return NULL; - } - return out; -} - /* ======================================================================== * Multi-shard operations * ======================================================================== */ @@ -530,9 +496,8 @@ int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nby return 0; } -const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, - const char *name, - safetensors_file_t **out_sf) { +const safetensor_t * +multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) { for (int s = 0; s < ms->num_shards; s++) { safetensors_file_t *sf = ms->shards[s]; for (int i = 0; i < sf->num_tensors; i++) { @@ -546,8 +511,7 @@ const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, return NULL; } -const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms, - const char *tensor_name) { +const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms, const char *tensor_name) { if (multi_safetensors_find(ms, tensor_name, NULL)) return ""; diff --git a/src/safetensors.h b/libffwd/safetensors.h index 8c9b33f..13d4cba 100644 --- a/src/safetensors.h +++ b/libffwd/safetensors.h @@ -1,10 +1,9 @@ /* - * embed_safetensors.h - Safetensors file format reader (multi-shard support) - * Adapted from voxtral-realtime project. + * safetensors.h - Safetensors file format reader (multi-shard support) */ -#ifndef EMBED_SAFETENSORS_H -#define EMBED_SAFETENSORS_H +#ifndef SAFETENSORS_H +#define SAFETENSORS_H #include <stddef.h> #include <stdint.h> @@ -41,7 +40,8 @@ typedef struct { safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; } safetensors_file_t; -/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ +/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup + */ typedef struct { safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; int num_shards; @@ -57,9 +57,8 @@ void multi_safetensors_close(multi_safetensors_t *ms); int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes); /* Find a tensor by name across all shards */ -const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms, - const char *name, - safetensors_file_t **out_sf); +const safetensor_t * +multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf); /* Resolve an optional Hugging Face "model." namespace for a backbone tensor. * Returns "", "model.", or NULL when neither form exists. */ @@ -68,9 +67,6 @@ const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms, const /* Get raw pointer to tensor data (within mmap'd region) */ const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); -/* Get tensor data as float32 (allocates, caller must free) */ -float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t); - int64_t safetensor_numel(const safetensor_t *t); -#endif /* EMBED_SAFETENSORS_H */ +#endif /* SAFETENSORS_H */ diff --git a/src/tokenizer_sentencepiece.c b/libffwd/sentencepiece.c index 657e8c9..2372972 100644 --- a/src/tokenizer_sentencepiece.c +++ b/libffwd/sentencepiece.c @@ -1,5 +1,5 @@ /* - * tokenizer_sentencepiece.c - SentencePiece Unigram tokenizer. + * sentencepiece.c - SentencePiece Unigram tokenizer. * * The implementation reads the small subset of ModelProto needed at runtime: * pieces, scores, piece types, trainer model_type, and normalizer options. The @@ -8,7 +8,7 @@ * over UTF-8 byte offsets. */ -#include "tokenizer_sentencepiece.h" +#include "sentencepiece.h" #include <errno.h> #include <float.h> @@ -39,7 +39,7 @@ typedef struct { double score; } sp_best_t; -struct sentencepiece_workspace { +struct tok_spm_workspace { char *norm; size_t norm_len; size_t norm_cap; @@ -127,7 +127,7 @@ static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, siz return 0; } -static int reserve_best(sentencepiece_workspace_t *ws, int need) { +static int reserve_best(tok_spm_workspace_t *ws, int need) { if (need <= ws->best_cap) return 0; int nc = ws->best_cap ? ws->best_cap : 128; @@ -144,7 +144,7 @@ static int reserve_best(sentencepiece_workspace_t *ws, int need) { return 0; } -static int reserve_ids(sentencepiece_workspace_t *ws, int need) { +static int reserve_ids(tok_spm_workspace_t *ws, int need) { if (need <= ws->ids_cap) return 0; int nc = ws->ids_cap ? ws->ids_cap : 64; @@ -268,8 +268,7 @@ static char *memdup0(const unsigned char *data, size_t len) { return s; } -static int -parse_piece(sentencepiece_tokenizer_t *tok, int idx, const unsigned char *data, size_t len) { +static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) { const unsigned char *p = data, *end = data + len; char *piece = NULL; float score = 0.0f; @@ -316,7 +315,7 @@ fail: return -1; } -static int parse_trainer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) { +static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) { const unsigned char *p = data, *end = data + len; int model_type = SP_MODEL_UNIGRAM; while (p < end) { @@ -341,7 +340,7 @@ static int parse_trainer(sentencepiece_tokenizer_t *tok, const unsigned char *da return 0; } -static int parse_normalizer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) { +static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) { const unsigned char *p = data, *end = data + len; while (p < end) { int field = 0, wire = 0; @@ -407,7 +406,7 @@ static int count_pieces(const unsigned char *data, size_t len) { return n; } -static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) { +static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { int n_pieces = count_pieces(data, len); if (n_pieces <= 0) return -1; @@ -441,8 +440,7 @@ static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data } else if (wire == 2 && field == 3) { const unsigned char *msg = NULL; size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || - parse_normalizer(tok, msg, msg_len) != 0) + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0) return -1; } else if (pb_skip(&p, end, wire) != 0) { return -1; @@ -453,17 +451,16 @@ static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data return 0; } -static int raw_piece_id(const sentencepiece_tokenizer_t *tok, const char *piece) { +static int raw_piece_id(const tok_spm_t *tok, const char *piece) { if (!tok || !piece) return -1; - return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, - (int)strlen(piece)); + return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece)); } -static int raw_to_model_id(const sentencepiece_tokenizer_t *tok, int raw_id) { +static int raw_to_model_id(const tok_spm_t *tok, int raw_id) { if (!tok || raw_id < 0 || raw_id >= tok->piece_count) return -1; - if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { if (raw_id == 1) return 0; /* <s> */ if (raw_id == 2) @@ -475,10 +472,10 @@ static int raw_to_model_id(const sentencepiece_tokenizer_t *tok, int raw_id) { return raw_id; } -static int model_to_raw_id(const sentencepiece_tokenizer_t *tok, int id) { +static int model_to_raw_id(const tok_spm_t *tok, int id) { if (!tok || id < 0) return -1; - if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { if (id == 0) return 1; if (id == 2) @@ -579,20 +576,18 @@ static int load_file(const char *path, unsigned char **out, size_t *out_len) { return 0; } -static uint32_t xcda_base(const sentencepiece_tokenizer_t *tok, size_t idx) { +static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) { uint32_t v = tok->xcda[idx]; return (v >> 10) << ((v & (1u << 9)) >> 6); } -static uint32_t xcda_lcheck(const sentencepiece_tokenizer_t *tok, size_t idx) { +static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) { return tok->xcda[idx] & ((1u << 31) | 0xffu); } -static int xcda_leaf(const sentencepiece_tokenizer_t *tok, size_t idx) { - return (tok->xcda[idx] >> 8) & 1u; -} +static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; } -static uint32_t xcda_value(const sentencepiece_tokenizer_t *tok, size_t idx) { +static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) { return tok->xcda[idx] & ((1u << 31) - 1u); } @@ -602,10 +597,8 @@ typedef struct { size_t consumed; } norm_prefix_t; -static norm_prefix_t normalize_prefix(const sentencepiece_tokenizer_t *tok, - const char *input, - size_t input_len, - size_t off) { +static norm_prefix_t +normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) { static const char repl[] = "\xef\xbf\xbd"; if (off >= input_len) return (norm_prefix_t){input + off, 0, 0}; @@ -642,9 +635,7 @@ static norm_prefix_t normalize_prefix(const sentencepiece_tokenizer_t *tok, return (norm_prefix_t){repl, 3, 1}; } -static int normalize_into(const sentencepiece_tokenizer_t *tok, - sentencepiece_workspace_t *ws, - const char *text) { +static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) { ws->norm_len = 0; if (!text) return -1; @@ -664,8 +655,8 @@ static int normalize_into(const sentencepiece_tokenizer_t *tok, processing_non_ws = 1; if ((tok->add_dummy_prefix && !space_prepended) || tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, - space_n) != 0) + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != + 0) return -1; space_prepended = 1; } @@ -675,8 +666,7 @@ static int normalize_into(const sentencepiece_tokenizer_t *tok, } else { processing_non_ws = 0; if (!tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != - 0) + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0) return -1; } } @@ -688,7 +678,7 @@ static int normalize_into(const sentencepiece_tokenizer_t *tok, return 0; } -static double compute_unknown_score(const sentencepiece_tokenizer_t *tok) { +static double compute_unknown_score(const tok_spm_t *tok) { float min_score = FLT_MAX; for (int i = 0; i < tok->piece_count; i++) { if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score) @@ -699,7 +689,7 @@ static double compute_unknown_score(const sentencepiece_tokenizer_t *tok) { return (double)min_score - 10.0; } -static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace_t *ws) { +static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { size_t n = ws->norm_len; if (n == 0) return 0; @@ -722,8 +712,8 @@ static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace int single_cp_found = 0; for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) { - int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, - ws->norm + off, (int)(end - off)); + int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off, + (int)(end - off)); if (raw >= 0 && is_tokenizable_type(tok->types[raw])) { if (end - off == cp_len) single_cp_found = 1; @@ -770,7 +760,7 @@ static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace return out_n; } -sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) { +tok_spm_t *tok_spm_load(const char *model_dir) { if (!model_dir) return NULL; char path[1024]; @@ -785,8 +775,7 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) { return NULL; } - sentencepiece_tokenizer_t *tok = - (sentencepiece_tokenizer_t *)calloc(1, sizeof(sentencepiece_tokenizer_t)); + tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t)); if (!tok) { free(data); return NULL; @@ -794,12 +783,11 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) { tok->add_dummy_prefix = 1; tok->remove_extra_whitespaces = 1; tok->escape_whitespaces = 1; - tok->id_map = - detect_xlm_roberta(model_dir) ? SENTENCEPIECE_XLM_ROBERTA : SENTENCEPIECE_IDENTITY; + tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY; if (parse_model(tok, data, len) != 0) { fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path); free(data); - sentencepiece_tokenizer_free(tok); + tok_spm_free(tok); return NULL; } free(data); @@ -807,14 +795,14 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) { int cap = next_pow2(tok->piece_count * 2 + 1); tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t)); if (!tok->piece_map) { - sentencepiece_tokenizer_free(tok); + tok_spm_free(tok); return NULL; } tok->piece_map_cap = cap; for (int i = 0; i < tok->piece_count; i++) { if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i], tok->piece_lens[i], i) != 0) { - sentencepiece_tokenizer_free(tok); + tok_spm_free(tok); return NULL; } } @@ -822,27 +810,25 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) { tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>")); tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>")); tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>")); - tok->pad_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA - ? 1 - : raw_to_model_id(tok, raw_piece_id(tok, "<pad>")); - tok->mask_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA + tok->pad_id = + tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>")); + tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 1 : raw_to_model_id(tok, raw_piece_id(tok, "<mask>")); - tok->vocab_size = - tok->id_map == SENTENCEPIECE_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; + tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; tok->unknown_score = compute_unknown_score(tok); if (tok->unk_id < 0 || tok->max_piece_len <= 0) { fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path); - sentencepiece_tokenizer_free(tok); + tok_spm_free(tok); return NULL; } return tok; } -const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id) { +const char *tok_spm_decode(const tok_spm_t *tok, int id) { if (!tok || id < 0) return NULL; - if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { if (id == 1) return "<pad>"; if (id == tok->piece_count + 1) @@ -852,10 +838,10 @@ const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, return raw >= 0 ? tok->id_to_piece[raw] : NULL; } -int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, const char *token) { +int tok_spm_token_id(const tok_spm_t *tok, const char *token) { if (!tok || !token) return -1; - if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (tok->id_map == TOK_SPM_XLM_ROBERTA) { if (!strcmp(token, "<s>")) return 0; if (!strcmp(token, "<pad>")) @@ -870,12 +856,12 @@ int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, const return raw_to_model_id(tok, raw_piece_id(tok, token)); } -int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok, - sentencepiece_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens) { +int tok_spm_encode_into(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !ws || !text) @@ -895,16 +881,16 @@ int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok, return 0; } -int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok, - sentencepiece_workspace_t *ws, - const char *text, - int *out_n_tokens) { +int *tok_spm_encode_with_workspace(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !ws || !text) return NULL; int n = 0; - int rc = sentencepiece_tokenizer_encode_into(tok, ws, text, NULL, 0, &n); + int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n); if (rc < 0 && rc != -2) return NULL; if (out_n_tokens) @@ -918,25 +904,23 @@ int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer return ids; } -int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok, - const char *text, - int *out_n_tokens) { - sentencepiece_workspace_t *ws = sentencepiece_workspace_new(); +int *tok_spm_encode(const tok_spm_t *tok, const char *text, int *out_n_tokens) { + tok_spm_workspace_t *ws = tok_spm_workspace_new(); if (!ws) { if (out_n_tokens) *out_n_tokens = 0; return NULL; } - int *ids = sentencepiece_tokenizer_encode_with_workspace(tok, ws, text, out_n_tokens); - sentencepiece_workspace_free(ws); + int *ids = tok_spm_encode_with_workspace(tok, ws, text, out_n_tokens); + tok_spm_workspace_free(ws); return ids; } -sentencepiece_workspace_t *sentencepiece_workspace_new(void) { - return (sentencepiece_workspace_t *)calloc(1, sizeof(sentencepiece_workspace_t)); +tok_spm_workspace_t *tok_spm_workspace_new(void) { + return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t)); } -void sentencepiece_workspace_free(sentencepiece_workspace_t *ws) { +void tok_spm_workspace_free(tok_spm_workspace_t *ws) { if (!ws) return; free(ws->norm); @@ -945,7 +929,7 @@ void sentencepiece_workspace_free(sentencepiece_workspace_t *ws) { free(ws); } -void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok) { +void tok_spm_free(tok_spm_t *tok) { if (!tok) return; if (tok->id_to_piece) { diff --git a/libffwd/sentencepiece.h b/libffwd/sentencepiece.h new file mode 100644 index 0000000..7f5314a --- /dev/null +++ b/libffwd/sentencepiece.h @@ -0,0 +1,88 @@ +/* + * sentencepiece.h - SentencePiece tokenizer for XLM-R style models. + * + * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`, + * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R + * and multilingual E5. It encodes text only; frontends add the model's special + * token layout, e.g. <s> ... </s> for XLM-R. + */ + +#ifndef TOK_SPM_H +#define TOK_SPM_H + +#include <stddef.h> +#include <stdint.h> + +typedef enum { + TOK_SPM_IDENTITY = 0, + TOK_SPM_XLM_ROBERTA = 1, +} tok_spm_id_map_t; + +typedef struct { + char **id_to_piece; /* [piece_count] raw SentencePiece ids */ + uint16_t *piece_lens; /* byte length for id_to_piece entries */ + float *scores; /* SentencePiece score per raw id */ + uint8_t *types; /* SentencePiece piece type per raw id */ + int piece_count; /* raw SentencePiece vocabulary size */ + int vocab_size; /* model embedding vocab size after id mapping */ + int unk_id; /* model-space unknown-token id */ + int bos_id; /* model-space beginning-token id, or -1 */ + int eos_id; /* model-space end-token id, or -1 */ + int pad_id; /* model-space pad-token id, or -1 */ + int mask_id; /* model-space mask-token id, or -1 */ + int max_piece_len; /* max normal/user/unused piece byte length */ + double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ + int add_dummy_prefix; + int remove_extra_whitespaces; + int escape_whitespaces; + tok_spm_id_map_t id_map; + + void *piece_map; /* piece bytes -> raw SentencePiece id */ + int piece_map_cap; + + uint32_t *xcda; + size_t xcda_count; + char *prefix_replacements; + size_t prefix_replacements_len; +} tok_spm_t; + +typedef struct tok_spm_workspace tok_spm_workspace_t; + +/* Load the first supported SentencePiece model found in model_dir. Returns NULL + * on missing model file, unsupported model type, malformed protobuf, or OOM. */ +tok_spm_t *tok_spm_load(const char *model_dir); + +/* Decode one model-space id to an internal token string, or NULL when invalid. */ +const char *tok_spm_decode(const tok_spm_t *tok, int id); + +/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a + * model-space id, or -1 when absent. */ +int tok_spm_token_id(const tok_spm_t *tok, const char *token); + +/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns + * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an + * error. */ +int *tok_spm_encode(const tok_spm_t *tok, const char *text, int *out_n_tokens); + +tok_spm_workspace_t *tok_spm_workspace_new(void); +void tok_spm_workspace_free(tok_spm_workspace_t *ws); + +/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if + * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */ +int tok_spm_encode_into(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); + +/* Same tokenization as tok_spm_encode(), reusing ws internally + * and performing only the final returned int[] allocation. */ +int *tok_spm_encode_with_workspace(const tok_spm_t *tok, + tok_spm_workspace_t *ws, + const char *text, + int *out_n_tokens); + +void tok_spm_free(tok_spm_t *tok); + +#endif /* TOK_SPM_H */ diff --git a/src/threadpool.c b/libffwd/threadpool.c index 40d22c7..2521a6d 100644 --- a/src/threadpool.c +++ b/libffwd/threadpool.c @@ -1,5 +1,9 @@ +/* + * tp.c - CPU thread pool + */ + #if defined(__linux__) && !defined(_GNU_SOURCE) -#define _GNU_SOURCE +# define _GNU_SOURCE #endif #include "threadpool.h" @@ -11,28 +15,23 @@ #include <string.h> #ifdef __APPLE__ -#include <sys/sysctl.h> +# include <sys/sysctl.h> #else -#ifdef __linux__ -#include <sched.h> -#endif -#include <unistd.h> +# ifdef __linux__ +# include <sched.h> +# endif +# include <unistd.h> #endif #if defined(USE_OPENBLAS) void openblas_set_num_threads(int num_threads); #endif -/* The global verbose flag (defined in model.c); used only for optional pool - * diagnostics below. Declared here to keep the thread pool independent of the - * kernel/model headers. */ -extern int embed_verbose; - -#define EMBED_MAX_THREADS 16 +#define TP_MAX_THREADS 16 static struct { - pthread_t threads[EMBED_MAX_THREADS - 1]; - int tids[EMBED_MAX_THREADS - 1]; + pthread_t threads[TP_MAX_THREADS - 1]; + int tids[TP_MAX_THREADS - 1]; int n_threads; int shutdown; @@ -80,11 +79,11 @@ static void *worker_loop(void *arg) { } } -void embed_set_threads(int n) { +void tp_set_threads(int n) { if (n < 1) n = 1; - if (n > EMBED_MAX_THREADS) - n = EMBED_MAX_THREADS; + if (n > TP_MAX_THREADS) + n = TP_MAX_THREADS; #if defined(USE_OPENBLAS) /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the @@ -107,29 +106,13 @@ void embed_set_threads(int n) { } tp.n_threads = n; - if (n <= 1) { - if (embed_verbose >= 2) { -#if defined(USE_OPENBLAS) - fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads); -#else - fprintf(stderr, "Thread pool: %d threads\n", n); -#endif - } + if (n <= 1) return; - } for (int i = 0; i < n - 1; i++) { tp.tids[i] = i + 1; pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); } - - if (embed_verbose >= 2) { -#if defined(USE_OPENBLAS) - fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads); -#else - fprintf(stderr, "Thread pool: %d threads\n", n); -#endif - } } #ifndef __APPLE__ @@ -177,7 +160,7 @@ static int cgroup_cpu_quota(void) { } #endif -int embed_get_num_cpus(void) { +int tp_get_num_cpus(void) { #ifdef __APPLE__ int n = 0; size_t len = sizeof(n); @@ -187,14 +170,14 @@ int embed_get_num_cpus(void) { int online = (int)sysconf(_SC_NPROCESSORS_ONLN); if (online < 1) online = 1; -#ifdef __linux__ +# ifdef __linux__ cpu_set_t set; if (sched_getaffinity(0, sizeof(set), &set) == 0) { int affinity = CPU_COUNT(&set); if (affinity >= 1 && affinity < online) online = affinity; } -#endif +# endif /* Honor a cgroup CPU quota when it is tighter than the visible core count, * so the default thread pool does not oversubscribe a CPU-limited * container and get throttled. */ @@ -205,7 +188,7 @@ int embed_get_num_cpus(void) { #endif } -int embed_num_threads(void) { return tp.n_threads; } +int tp_num_threads(void) { return tp.n_threads; } /* Dispatch work to all threads; main thread is tid=0 */ void parallel_for(parallel_fn_t fn, void *arg) { diff --git a/src/threadpool.h b/libffwd/threadpool.h index 1813bb0..1dcc80a 100644 --- a/src/threadpool.h +++ b/libffwd/threadpool.h @@ -1,36 +1,28 @@ -#ifndef EMBED_THREADPOOL_H -#define EMBED_THREADPOOL_H +#ifndef TP_THREADPOOL_H +#define TP_THREADPOOL_H /* Persistent worker pool for the CPU math kernels: a fixed set of threads that * sleep until parallel_for() hands them a callback, plus the CPU-count helper * that sizes the pool (cgroup-quota aware). Separated from the math so the - * kernels depend only on the parallel_for / embed_num_threads interface, not on + * kernels depend only on the parallel_for / tp_num_threads interface, not on * the pool's internals. kernels.h includes this, so kernel callers need no * extra include. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - /* Worker callback: (thread index, total threads, user arg). */ typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); /* Set the persistent pool size (default 1). Creates/rebuilds the pool; call * before inference. */ -EMBED_API void embed_set_threads(int n); +void tp_set_threads(int n); /* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */ -EMBED_API int embed_get_num_cpus(void); +int tp_get_num_cpus(void); /* Current pool size (1 == serial). Kernels gate parallelism on this. */ -int embed_num_threads(void); +int tp_num_threads(void); /* Run fn across the pool; the calling thread participates as tid 0. Blocks * until every worker has finished. */ void parallel_for(parallel_fn_t fn, void *arg); -#endif /* EMBED_THREADPOOL_H */ +#endif /* TP_THREADPOOL_H */ diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c new file mode 100644 index 0000000..f7a50f7 --- /dev/null +++ b/libffwd/tokenize.c @@ -0,0 +1,350 @@ +/* + * tokenize.c - model-aware tokenization: text -> the token ids a model expects. + * + * Backend-independent, so this is its own handle (ffwd_tok_t), not part of the + * per-backend ffwd_t. ffwd_tok_open picks the tokenizer by the files present + * and resolves the model's special-token ids; ffwd_tokenize applies that + * model's layout (BERT/XLM-R [CLS]..[SEP], Qwen appended terminal token). + */ + +#include "ffwd.h" + +#include "bpe.h" +#include "sentencepiece.h" +#include "wordpiece.h" + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +typedef enum { TOK_KIND_BPE, TOK_KIND_WP, TOK_KIND_SPM } tok_kind_t; + +struct ffwd_tok { + tok_kind_t kind; + /* exactly one trio is populated, by kind */ + tok_bpe_t *bpe; + tok_bpe_workspace_t *bpe_ws; + tok_wp_t *wp; + tok_wp_workspace_t *wp_ws; + tok_spm_t *spm; + tok_spm_workspace_t *spm_ws; + + int cls_id; /* [CLS] / <s> — wp/spm wrap with these */ + int sep_id; /* [SEP] / </s> */ + int context_separator_id; + int terminal_id; /* appended on the bpe path when append_terminal */ + int append_terminal; /* Qwen3-Embedding pools the last token; see note */ + + int is_late; + int late_mask_id; + int late_query_prefix_id; + int late_document_prefix_id; + int late_skip_ids[64]; + int n_late_skip_ids; +}; + +static void set_err(char *err, size_t errlen, const char *msg) { + if (err && errlen) + snprintf(err, errlen, "%s", msg); +} + +ffwd_tok_t *ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen) { + if (!model_dir) { + set_err(err, errlen, "model_dir is NULL"); + return NULL; + } + ffwd_tok_t *t = calloc(1, sizeof(*t)); + if (!t) { + set_err(err, errlen, "out of memory"); + return NULL; + } + t->cls_id = -1; + t->sep_id = -1; + t->terminal_id = -1; + t->is_late = is_late; + + char path[1024]; + + /* WordPiece (BERT family): vocab.txt. */ + snprintf(path, sizeof(path), "%s/vocab.txt", model_dir); + if (access(path, R_OK) == 0) { + t->kind = TOK_KIND_WP; + t->wp = tok_wp_load(model_dir); + if (!t->wp) { + set_err(err, errlen, "failed to load WordPiece tokenizer"); + goto fail; + } + t->wp_ws = tok_wp_workspace_new(); + if (!t->wp_ws) { + set_err(err, errlen, "out of memory"); + goto fail; + } + t->cls_id = tok_wp_token_id(t->wp, "[CLS]"); + t->sep_id = tok_wp_token_id(t->wp, "[SEP]"); + if (t->cls_id < 0 || t->sep_id < 0) { + set_err(err, errlen, "WordPiece vocab missing [CLS]/[SEP]"); + goto fail; + } + return t; + } + + /* SentencePiece (XLM-R family): sentencepiece.bpe.model / tokenizer.model / + * spiece.model. */ + snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", model_dir); + if (access(path, R_OK) != 0) { + snprintf(path, sizeof(path), "%s/tokenizer.model", model_dir); + if (access(path, R_OK) != 0) + snprintf(path, sizeof(path), "%s/spiece.model", model_dir); + } + if (access(path, R_OK) == 0) { + t->kind = TOK_KIND_SPM; + t->spm = tok_spm_load(model_dir); + if (!t->spm) { + set_err(err, errlen, "failed to load SentencePiece tokenizer"); + goto fail; + } + t->spm_ws = tok_spm_workspace_new(); + if (!t->spm_ws) { + set_err(err, errlen, "out of memory"); + goto fail; + } + t->cls_id = tok_spm_token_id(t->spm, "<s>"); + t->sep_id = tok_spm_token_id(t->spm, "</s>"); + if (t->cls_id < 0 || t->sep_id < 0) { + set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); + goto fail; + } + return t; + } + + /* Byte-level BPE (Qwen family): vocab.json. */ + snprintf(path, sizeof(path), "%s/vocab.json", model_dir); + t->kind = TOK_KIND_BPE; + t->bpe = tok_bpe_load(path); + if (!t->bpe) { + set_err(err, errlen, "failed to load tokenizer"); + goto fail; + } + t->bpe_ws = tok_bpe_workspace_new(); + if (!t->bpe_ws) { + set_err(err, errlen, "out of memory"); + goto fail; + } + /* The contextual chunk separator is the tokenizer's <|endoftext|>; released + * models keep it out of vocab.json, so the family constant is the fallback. + * Qwen3-Embedding pools the last token (the same <|endoftext|>), so the bpe + * path appends it. A config that disables append_terminal_token would need + * to override this; the released models all enable it. */ + int sep = tok_bpe_token_id(t->bpe, "<|endoftext|>"); + t->context_separator_id = sep >= 0 ? sep : FFWD_CONTEXT_SEPARATOR_TOKEN_ID; + t->terminal_id = t->context_separator_id; + /* Off until the caller sets it from config->append_terminal_token; it is + * model-specific (pplx-embed does not append, for example). */ + t->append_terminal = 0; + + if (is_late) { + int id = tok_bpe_token_id(t->bpe, "[MASK]"); + t->late_mask_id = id >= 0 ? id : FFWD_LATE_MASK_TOKEN_ID; + id = tok_bpe_token_id(t->bpe, "[Q]"); + t->late_query_prefix_id = id >= 0 ? id : FFWD_LATE_QUERY_PREFIX_ID; + id = tok_bpe_token_id(t->bpe, "[D]"); + t->late_document_prefix_id = id >= 0 ? id : FFWD_LATE_DOCUMENT_PREFIX_ID; + + const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"; + for (const char *p = punct; + *p && t->n_late_skip_ids < (int)(sizeof(t->late_skip_ids) / sizeof(t->late_skip_ids[0])); + p++) { + char text[2] = {*p, '\0'}; + int n_ids = 0; + int *ids = tok_bpe_encode(t->bpe, text, &n_ids); + if (ids && n_ids == 1) + t->late_skip_ids[t->n_late_skip_ids++] = ids[0]; + free(ids); + } + } + return t; + +fail: + ffwd_tok_free(t); + return NULL; +} + +void ffwd_tok_free(ffwd_tok_t *t) { + if (!t) + return; + if (t->bpe_ws) + tok_bpe_workspace_free(t->bpe_ws); + if (t->bpe) + tok_bpe_free(t->bpe); + if (t->wp_ws) + tok_wp_workspace_free(t->wp_ws); + if (t->wp) + tok_wp_free(t->wp); + if (t->spm_ws) + tok_spm_workspace_free(t->spm_ws); + if (t->spm) + tok_spm_free(t->spm); + free(t); +} + +int ffwd_tok_context_separator_id(const ffwd_tok_t *t) { + return t ? t->context_separator_id : -1; +} + +void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on) { + if (t) + t->append_terminal = on ? 1 : 0; +} + +/* Encode text and apply the model's special-token layout. Returns malloc'd ids + * and sets *n_out; NULL on error. */ +static int *tokenize_core(ffwd_tok_t *t, const char *text, int *n_out) { + *n_out = 0; + + if (t->kind == TOK_KIND_WP || t->kind == TOK_KIND_SPM) { + int n = 0; + int *core = (t->kind == TOK_KIND_WP) + ? tok_wp_encode_with_workspace(t->wp, t->wp_ws, text, &n) + : tok_spm_encode_with_workspace(t->spm, t->spm_ws, text, &n); + /* Wrap [CLS] core [SEP]. A whitespace-only input cleans to zero core + * tokens (core == NULL, n == 0); the bare [CLS][SEP] pair is what + * Hugging Face produces for it too. Grow in place so only the one + * allocation is touched. */ + if (n < 0 || n > INT_MAX - 2) { + free(core); + return NULL; + } + int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); + if (!ids) { + free(core); + return NULL; + } + memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); + ids[0] = t->cls_id; + ids[n + 1] = t->sep_id; + *n_out = n + 2; + return ids; + } + + /* BPE. */ + int n = 0; + int *ids = tok_bpe_encode_with_workspace(t->bpe, t->bpe_ws, text, &n); + if (!ids || n <= 0) { + free(ids); + return NULL; + } + if (t->append_terminal) { + if (n == INT_MAX) { + free(ids); + return NULL; + } + int *grown = realloc(ids, (size_t)(n + 1) * sizeof(*ids)); + if (!grown) { + free(ids); + return NULL; + } + ids = grown; + ids[n++] = t->terminal_id; + } + *n_out = n; + return ids; +} + +int *ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out) { + if (n_out) + *n_out = 0; + if (!t || !text) + return NULL; + + int n = 0; + int *ids; + if (!query_instruct) { + ids = tokenize_core(t, text, &n); + } else { + /* Tokenize the instruction and the input in one pass. */ + size_t plen = strlen(query_instruct); + size_t tlen = strlen(text); + char *buf = malloc(plen + tlen + 1); + if (!buf) + return NULL; + memcpy(buf, query_instruct, plen); + memcpy(buf + plen, text, tlen + 1); + ids = tokenize_core(t, buf, &n); + free(buf); + } + if (!ids) + return NULL; + if (n_out) + *n_out = n; + return ids; +} + +static int late_id_is_skipped(const ffwd_tok_t *t, int id) { + for (int i = 0; i < t->n_late_skip_ids; i++) { + if (t->late_skip_ids[i] == id) + return 1; + } + return 0; +} + +int ffwd_tokenize_late(ffwd_tok_t *t, + const char *text, + int is_query, + ffwd_late_tokens_t *out) { + memset(out, 0, sizeof(*out)); + if (!t || !text) + return -1; + + int raw_n = 0; + int *raw = tokenize_core(t, text, &raw_n); + if (!raw) + return -1; + + int target = raw_n + 1; + if (is_query && target < FFWD_LATE_QUERY_TOKENS) + target = FFWD_LATE_QUERY_TOKENS; + + out->ids = malloc((size_t)target * sizeof(*out->ids)); + if (!out->ids) { + free(raw); + return -1; + } + out->ids[0] = raw[0]; + out->ids[1] = is_query ? t->late_query_prefix_id : t->late_document_prefix_id; + if (raw_n > 1) + memcpy(out->ids + 2, raw + 1, (size_t)(raw_n - 1) * sizeof(*out->ids)); + out->n_tokens = raw_n + 1; + free(raw); + + if (is_query) { + while (out->n_tokens < FFWD_LATE_QUERY_TOKENS) + out->ids[out->n_tokens++] = t->late_mask_id; + out->n_keep = out->n_tokens; + } else { + out->keep = malloc((size_t)out->n_tokens * sizeof(*out->keep)); + if (!out->keep) { + ffwd_late_tokens_free(out); + return -1; + } + for (int i = 0; i < out->n_tokens; i++) { + if (!late_id_is_skipped(t, out->ids[i])) + out->keep[out->n_keep++] = i; + } + } + + if (out->n_keep <= 0) { + ffwd_late_tokens_free(out); + return -1; + } + return 0; +} + +void ffwd_late_tokens_free(ffwd_late_tokens_t *out) { + if (!out) + return; + free(out->ids); + free(out->keep); + memset(out, 0, sizeof(*out)); +} diff --git a/src/tokenizer_unicode.h b/libffwd/unicode.h index 8ca2458..bce8967 100644 --- a/src/tokenizer_unicode.h +++ b/libffwd/unicode.h @@ -31,54 +31,61 @@ typedef struct { } wp_fold_t; static const wp_range_t WP_PUNCT_RANGES[] = { - {0x21, 0x2F}, {0x3A, 0x40}, {0x5B, 0x60}, {0x7B, 0x7E}, {0xA1, 0xA1}, {0xA7, 0xA7}, - {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387}, - {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3}, - {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D}, {0x61B, 0x61B}, - {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9}, - {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD}, - {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4}, - {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D}, - {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB}, - {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED}, - {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, {0x1944, 0x1945}, - {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF}, - {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027}, - {0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E}, - {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF}, - {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF}, - {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003}, - {0x3008, 0x3011}, {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0}, - {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E}, - {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC}, - {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F}, - {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19}, - {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B}, - {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20}, - {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65}, - {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F}, - {0x10857, 0x10857}, {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58}, - {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C}, - {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC}, - {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8}, - {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D}, - {0x112A9, 0x112A9}, {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D}, - {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C}, - {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2}, - {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45}, - {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474}, - {0x16A6E, 0x16A6F}, {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44}, - {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B}, - {0x1E95E, 0x1E95F}, + {0x21, 0x2F}, {0x3A, 0x40}, {0x5B, 0x60}, {0x7B, 0x7E}, + {0xA1, 0xA1}, {0xA7, 0xA7}, {0xAB, 0xAB}, {0xB6, 0xB7}, + {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387}, + {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, + {0x5C3, 0x5C3}, {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, + {0x60C, 0x60D}, {0x61B, 0x61B}, {0x61E, 0x61F}, {0x66A, 0x66D}, + {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9}, {0x830, 0x83E}, + {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD}, + {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, + {0xDF4, 0xDF4}, {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, + {0xF14, 0xF14}, {0xF3A, 0xF3D}, {0xF85, 0xF85}, {0xFD0, 0xFD4}, + {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB}, {0x1360, 0x1368}, + {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED}, + {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, + {0x1944, 0x1945}, {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, + {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF}, {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, + {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027}, {0x2030, 0x2043}, + {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E}, + {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, + {0x27E6, 0x27EF}, {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD}, + {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF}, {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, + {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003}, {0x3008, 0x3011}, + {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0}, + {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, + {0xA67E, 0xA67E}, {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, + {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC}, {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, + {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F}, {0xAADE, 0xAADF}, + {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19}, + {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, + {0xFE6A, 0xFE6B}, {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, + {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20}, {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, + {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65}, {0x10100, 0x10102}, + {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F}, {0x10857, 0x10857}, + {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58}, {0x10A7F, 0x10A7F}, + {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C}, {0x10EAD, 0x10EAD}, + {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC}, {0x110BE, 0x110C1}, + {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8}, {0x111CD, 0x111CD}, + {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D}, {0x112A9, 0x112A9}, + {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D}, {0x114C6, 0x114C6}, + {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C}, {0x1173C, 0x1173E}, + {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2}, {0x11A3F, 0x11A46}, + {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45}, {0x11C70, 0x11C71}, + {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474}, {0x16A6E, 0x16A6F}, + {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44}, {0x16E97, 0x16E9A}, + {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B}, {0x1E95E, 0x1E95F}, }; enum { WP_PUNCT_RANGES_N = (int)(sizeof(WP_PUNCT_RANGES) / sizeof(WP_PUNCT_RANGES[0])) }; static const wp_range_t WP_CONTROL_RANGES[] = { - {0x0, 0x8}, {0xB, 0xC}, {0xE, 0x1F}, {0x7F, 0x9F}, {0xAD, 0xAD}, {0x600, 0x605}, - {0x61C, 0x61C}, {0x6DD, 0x6DD}, {0x70F, 0x70F}, {0x8E2, 0x8E2}, {0x180E, 0x180E}, - {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x2064}, {0x2066, 0x206F}, {0xE000, 0xF8FF}, - {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0x110BD, 0x110BD}, {0x110CD, 0x110CD}, - {0x13430, 0x13438}, {0x1BCA0, 0x1BCA3}, {0x1D173, 0x1D17A}, {0xE0001, 0xE0001}, + {0x0, 0x8}, {0xB, 0xC}, {0xE, 0x1F}, {0x7F, 0x9F}, + {0xAD, 0xAD}, {0x600, 0x605}, {0x61C, 0x61C}, {0x6DD, 0x6DD}, + {0x70F, 0x70F}, {0x8E2, 0x8E2}, {0x180E, 0x180E}, {0x200B, 0x200F}, + {0x202A, 0x202E}, {0x2060, 0x2064}, {0x2066, 0x206F}, {0xE000, 0xF8FF}, + {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0x110BD, 0x110BD}, {0x110CD, 0x110CD}, + {0x13430, 0x13438}, {0x1BCA0, 0x1BCA3}, {0x1D173, 0x1D17A}, {0xE0001, 0xE0001}, {0xE0020, 0xE007F}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}, }; enum { WP_CONTROL_RANGES_N = (int)(sizeof(WP_CONTROL_RANGES) / sizeof(WP_CONTROL_RANGES[0])) }; diff --git a/src/vec.c b/libffwd/vec.c index cea1576..f848be0 100644 --- a/src/vec.c +++ b/libffwd/vec.c @@ -1,4 +1,4 @@ -#include "embed.h" +#include "ffwd.h" #include <math.h> @@ -10,7 +10,7 @@ * similarity. They touch no model or tensor internals, only the public API. * ======================================================================== */ -int embed_l2_normalize(float *vec, int dim) { +int ffwd_l2_normalize(float *vec, int dim) { if (!vec || dim <= 0) return -1; @@ -27,7 +27,7 @@ int embed_l2_normalize(float *vec, int dim) { return 0; } -float embed_cosine_similarity(const float *a, const float *b, int dim) { +float ffwd_cosine_similarity(const float *a, const float *b, int dim) { if (!a || !b || dim <= 0) return 0.0f; @@ -42,8 +42,8 @@ float embed_cosine_similarity(const float *a, const float *b, int dim) { norm_b_sq += bv * bv; } - if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) || - !isfinite(norm_b_sq) || !isfinite(dot)) + if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) || !isfinite(norm_b_sq) || + !isfinite(dot)) return 0.0f; double denom = sqrt((double)norm_a_sq) * sqrt((double)norm_b_sq); diff --git a/src/tokenizer_wordpiece.c b/libffwd/wordpiece.c index d8acbbe..d3f62dd 100644 --- a/src/tokenizer_wordpiece.c +++ b/libffwd/wordpiece.c @@ -1,5 +1,5 @@ /* - * wordpiece_tokenizer.c - BERT-family WordPiece tokenizer. + * tok_wp_tokenizer.c - BERT-family WordPiece tokenizer. * * Pipeline (matching transformers' BertTokenizer): * BasicTokenizer: clean control/whitespace -> space CJK ideographs -> @@ -8,7 +8,7 @@ * pieces marked "##", whole token -> [UNK] when any piece fails to match. * * Unicode handling is full and exact against transformers, driven by generated - * tables (wordpiece_unicode.h): punctuation and control by Unicode category, + * tables (tok_wp_unicode.h): punctuation and control by Unicode category, * whitespace by the complete Zs set, the CJK split by the BERT ideograph ranges, * and the do_lower_case fold by a lowercase+NFD+strip-marks table (with Hangul * decomposed by formula). Cased and uncased multilingual encoders (e.g. @@ -18,9 +18,9 @@ * Greek final sigma). */ -#include "tokenizer_wordpiece.h" +#include "wordpiece.h" -#include "tokenizer_unicode.h" +#include "unicode.h" #include <stdint.h> #include <stdio.h> @@ -237,7 +237,7 @@ static void wp_fold(uint32_t cp, uint32_t out[3], int *n_out) { /* =============================== workspace ================================== */ -struct wordpiece_workspace { +struct tok_wp_workspace { uint32_t *cps; /* cleaned + CJK-spaced codepoints of the whole input */ int cps_cap; uint32_t *norm; /* one word's normalized codepoints */ @@ -294,11 +294,11 @@ static int grow_char(char **buf, int *cap, int need) { return 0; } -wordpiece_workspace_t *wordpiece_workspace_new(void) { - return (wordpiece_workspace_t *)calloc(1, sizeof(wordpiece_workspace_t)); +tok_wp_workspace_t *tok_wp_workspace_new(void) { + return (tok_wp_workspace_t *)calloc(1, sizeof(tok_wp_workspace_t)); } -void wordpiece_workspace_free(wordpiece_workspace_t *ws) { +void tok_wp_workspace_free(tok_wp_workspace_t *ws) { if (!ws) return; free(ws->cps); @@ -315,11 +315,8 @@ void wordpiece_workspace_free(wordpiece_workspace_t *ws) { /* Append the WordPiece ids of one BasicTokenizer sub-word (codepoints * word[0..n)) to ws->ids[*n_out]. The whole sub-word maps to [UNK] when it is * too long or any greedy piece misses the vocab. Returns 0 / -1 on OOM. */ -static int wordpiece_subword(const wordpiece_tokenizer_t *tok, - wordpiece_workspace_t *ws, - const uint32_t *word, - int n, - int *n_out) { +static int +tok_wp_subword(const tok_wp_t *tok, tok_wp_workspace_t *ws, const uint32_t *word, int n, int *n_out) { if (n == 0) return 0; if (n > tok->max_chars_per_word) { @@ -386,8 +383,7 @@ static int wordpiece_subword(const wordpiece_tokenizer_t *tok, /* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. Returns the * token count, or -1 on allocation failure. */ -static int -wordpiece_run(const wordpiece_tokenizer_t *tok, wordpiece_workspace_t *ws, const char *text) { +static int tok_wp_run(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text) { /* clean_text + tokenize_chinese_chars, in one pass over the input. */ const unsigned char *p = (const unsigned char *)text; const unsigned char *end = p + strlen(text); @@ -449,16 +445,16 @@ wordpiece_run(const wordpiece_tokenizer_t *tok, wordpiece_workspace_t *ws, const int run = 0; for (int k = 0; k < m; k++) { if (wp_is_punct(ws->norm[k])) { - if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (k - run), run, &n_out) != 0) + if (run > 0 && tok_wp_subword(tok, ws, ws->norm + (k - run), run, &n_out) != 0) return -1; run = 0; - if (wordpiece_subword(tok, ws, ws->norm + k, 1, &n_out) != 0) + if (tok_wp_subword(tok, ws, ws->norm + k, 1, &n_out) != 0) return -1; } else { run++; } } - if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (m - run), run, &n_out) != 0) + if (run > 0 && tok_wp_subword(tok, ws, ws->norm + (m - run), run, &n_out) != 0) return -1; } return n_out; @@ -496,7 +492,7 @@ static char *read_file(const char *path, long *out_len) { return buf; } -/* do_lower_case from tokenizer_config.json: default true (BERT uncased). A +/* do_lower_case from config.json: default true (BERT uncased). A * lightweight scan is enough - we only need one boolean and avoid a JSON dep * here, mirroring how the config layer reads small flags. */ static int read_do_lower_case(const char *model_dir) { @@ -521,7 +517,7 @@ static int read_do_lower_case(const char *model_dir) { return result; } -wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) { +tok_wp_t *tok_wp_load(const char *model_dir) { if (!model_dir) return NULL; char path[2048]; @@ -541,7 +537,7 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) { if (len > 0 && buf[len - 1] != '\n') count++; - wordpiece_tokenizer_t *tok = (wordpiece_tokenizer_t *)calloc(1, sizeof(*tok)); + tok_wp_t *tok = (tok_wp_t *)calloc(1, sizeof(*tok)); char **id_to_token = (char **)calloc((size_t)(count > 0 ? count : 1), sizeof(char *)); if (!tok || !id_to_token) { free(buf); @@ -582,7 +578,7 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) { int cap = next_pow2(n * 2 < 16 ? 16 : n * 2); wp_entry_t *map = (wp_entry_t *)calloc((size_t)cap, sizeof(wp_entry_t)); if (!map) { - wordpiece_tokenizer_free(tok); + tok_wp_free(tok); return NULL; } for (int i = 0; i < n; i++) @@ -593,7 +589,7 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) { tok->unk_id = map_get(map, cap, "[UNK]"); if (tok->unk_id < 0) { fprintf(stderr, "wordpiece: vocab.txt has no [UNK] token\n"); - wordpiece_tokenizer_free(tok); + tok_wp_free(tok); return NULL; } return tok; @@ -601,29 +597,29 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) { /* ============================== public API ================================= */ -const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id) { +const char *tok_wp_decode(const tok_wp_t *tok, int id) { if (!tok || id < 0 || id >= tok->vocab_size) return NULL; return tok->id_to_token[id]; } -int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token) { +int tok_wp_token_id(const tok_wp_t *tok, const char *token) { if (!tok || !token) return -1; return map_get((const wp_entry_t *)tok->vocab_map, tok->vocab_map_cap, token); } -int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok, - wordpiece_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens) { +int tok_wp_encode_into(const tok_wp_t *tok, + tok_wp_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !ws || !text) return -1; - int n = wordpiece_run(tok, ws, text); + int n = tok_wp_run(tok, ws, text); if (n < 0) return -1; if (out_n_tokens) @@ -634,15 +630,15 @@ int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok, return 0; } -int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok, - wordpiece_workspace_t *ws, - const char *text, - int *out_n_tokens) { +int *tok_wp_encode_with_workspace(const tok_wp_t *tok, + tok_wp_workspace_t *ws, + const char *text, + int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !ws || !text) return NULL; - int n = wordpiece_run(tok, ws, text); + int n = tok_wp_run(tok, ws, text); if (n < 0) return NULL; int *ids = (int *)malloc((size_t)(n > 0 ? n : 1) * sizeof(int)); @@ -654,18 +650,18 @@ int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok, return ids; } -int *wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n) { +int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n) { if (out_n) *out_n = 0; - wordpiece_workspace_t *ws = wordpiece_workspace_new(); + tok_wp_workspace_t *ws = tok_wp_workspace_new(); if (!ws) return NULL; - int *ids = wordpiece_tokenizer_encode_with_workspace(tok, ws, text, out_n); - wordpiece_workspace_free(ws); + int *ids = tok_wp_encode_with_workspace(tok, ws, text, out_n); + tok_wp_workspace_free(ws); return ids; } -void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok) { +void tok_wp_free(tok_wp_t *tok) { if (!tok) return; if (tok->id_to_token) { diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h new file mode 100644 index 0000000..66c62a0 --- /dev/null +++ b/libffwd/wordpiece.h @@ -0,0 +1,70 @@ +/* + * wordpiece.h - BERT-family WordPiece tokenizer. + * + * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional + * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting) + * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes + * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the + * byte-level BPE backend leaves terminal/separator tokens to the frontend. + * + * The public surface mirrors bpe.h (load / encode with a reusable + * workspace / decode / resolve a named special token / free) so both can sit + * behind one tokenizer interface later. + */ + +#ifndef TOK_WP_H +#define TOK_WP_H + +typedef struct { + char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */ + int vocab_size; + int unk_id; /* [UNK] id, resolved at load */ + int do_lower_case; /* lowercase + strip accents before WordPiece */ + int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */ + + /* Opaque token-string -> id hash map. */ + void *vocab_map; + int vocab_map_cap; +} tok_wp_t; + +typedef struct tok_wp_workspace tok_wp_workspace_t; + +/* Load from <model_dir>/vocab.txt, reading do_lower_case from + * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on + * error (missing vocab.txt, missing [UNK], allocation failure). */ +tok_wp_t *tok_wp_load(const char *model_dir); + +/* Decode a single id to its vocab token string (internal pointer), or NULL. */ +const char *tok_wp_decode(const tok_wp_t *tok, int id); + +/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */ +int tok_wp_token_id(const tok_wp_t *tok, const char *token); + +/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on + * error (and *out_n_tokens = 0). */ +int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n_tokens); + +/* Reusable scratch for allocation-light repeated encoding. */ +tok_wp_workspace_t *tok_wp_workspace_new(void); +void tok_wp_workspace_free(tok_wp_workspace_t *ws); + +/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1 + * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to + * the number of tokens needed/emitted in all cases. */ +int tok_wp_encode_into(const tok_wp_t *tok, + tok_wp_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); + +/* Same tokenization as tok_wp_encode(), reusing ws internally and + * performing only the final returned int[] allocation. */ +int *tok_wp_encode_with_workspace(const tok_wp_t *tok, + tok_wp_workspace_t *ws, + const char *text, + int *out_n_tokens); + +void tok_wp_free(tok_wp_t *tok); + +#endif /* TOK_WP_H */ diff --git a/src/build.h b/src/build.h deleted file mode 100644 index dc68405..0000000 --- a/src/build.h +++ /dev/null @@ -1,64 +0,0 @@ -/* embed_build.h - build identity for the embed and embed-server front-ends. - * - * The real values come from the Makefile's VERSION_CFLAGS at compile time; the - * fallbacks below apply when those flags are absent. Front-end only: the - * inference library does not use these. */ -#ifndef EMBED_BUILD_H -#define EMBED_BUILD_H - -#include <stdio.h> -#include <string.h> - -#ifndef EMBED_VERSION -#define EMBED_VERSION "development" -#endif -#ifndef EMBED_BUILD_COMMIT -#define EMBED_BUILD_COMMIT "unknown" -#endif -#ifndef EMBED_BUILD_DATE -#define EMBED_BUILD_DATE "unknown" -#endif -#ifndef EMBED_BUILD_OS -#define EMBED_BUILD_OS "unknown" -#endif -#ifndef EMBED_BUILD_ARCH -#define EMBED_BUILD_ARCH "unknown" -#endif - -/* Backend label for --build-info, selected by the same macro that picks the - * compiled inference path. */ -#if defined(USE_MLX) -#define EMBED_BUILD_BACKENDS "accelerate,mlx" -#elif defined(USE_CUDA) -#define EMBED_BUILD_BACKENDS "openblas,cuda" -#elif defined(__APPLE__) -#define EMBED_BUILD_BACKENDS "accelerate" -#else -#define EMBED_BUILD_BACKENDS "openblas" -#endif - -/* Program name with any leading directory removed. */ -static inline const char *embed_prog_name(const char *argv0) -{ - const char *slash = strrchr(argv0, '/'); - return slash ? slash + 1 : argv0; -} - -static inline void embed_print_version(const char *prog) -{ - printf("%s %s (%s/%s)\n", prog, EMBED_VERSION, - EMBED_BUILD_OS, EMBED_BUILD_ARCH); -} - -static inline void embed_print_build_info(const char *prog) -{ - embed_print_version(prog); - printf("backend: %s\n" - "compiler: %s\n" - "built: %s\n" - "commit: %s\n", - EMBED_BUILD_BACKENDS, __VERSION__, - EMBED_BUILD_DATE, EMBED_BUILD_COMMIT); -} - -#endif /* EMBED_BUILD_H */ diff --git a/src/cli.c b/src/cli.c deleted file mode 100644 index 0e3ed7d..0000000 --- a/src/cli.c +++ /dev/null @@ -1,911 +0,0 @@ -/* embed_cli.c - embed command-line tool */ - -#include "embed.h" -#include "build.h" -#include "kernels.h" -#include "tokenizer_bpe.h" -#include "tokenizer_sentencepiece.h" -#include "tokenizer_wordpiece.h" - -#ifdef USE_MLX -#include "mlx.h" -#endif -#ifdef USE_CUDA -#include "cuda.h" -#endif - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <time.h> -#include <limits.h> -#include <unistd.h> - -static double now_ms(void) { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6; -} - -static void print_usage(const char *prog) { - fprintf(stderr, - "Usage: %s -d <model_dir> [options] [text...]\n" - "\n" - "Options:\n" - " -d <dir> Model directory (required)\n" -#ifdef USE_MLX - " --mlx Use Apple MLX GPU backend\n" -#endif -#ifdef USE_CUDA - " --cuda Use CUDA/cuBLAS GPU backend\n" -#endif - " --stream Read lines from stdin, write one JSON embedding per line\n" - " --json Emit JSON instead of plain text (embedding or matrix)\n" -#ifdef USE_MLX - " --mlx-quant-bits N\n" - " Quantize MLX linear weights to 8 bits at load time\n" - " --mlx-quant-group-size N\n" - " MLX quantization group size (default: 64)\n" -#endif -#ifdef USE_CUDA - " --cuda-gemm-mode MODE\n" - " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" - " --cuda-weight-dtype DTYPE\n" - " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n" - " weight memory and uses BF16 tensor cores\n" -#endif - " -b, --batch-size N\n" - " Max texts per engine batch (default: all; --stream: 1)\n" - " -t, --threads N\n" - " CPU threads (default: available cores, cgroup-aware)\n" - " -e, --embeddings\n" - " Print raw embeddings (with multiple texts)\n" - " -v, --verbose\n" - " Verbose (-vv for debug)\n" - " -V, --version\n" - " Print version and exit\n" - " --build-info Print build details and exit\n" - " -h Show this help\n" - "\n" - "Modes:\n" - " 1 text arg Embedding as space-separated floats (JSON with --json)\n" - " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n" - " no args Batch: read all stdin lines, then similarity matrix\n" - " --stream Streaming: read stdin lines, write JSON per line\n" - "\n" - "Examples:\n" - " %s -d ./model \"what is AI?\"\n" -#ifdef USE_MLX - " %s -d ./model --mlx --stream < texts.txt\n", - prog, prog, prog); -#else - , - prog, prog); -#endif -} - -/* ======================================================================== - * Embed one text: tokenize > forward > return float[dim] - * ======================================================================== */ - -typedef struct { - embed_model_t *model; - embed_workspace_t *workspace; -#ifdef USE_MLX - embed_mlx_ctx_t *mlx_ctx; -#endif -#ifdef USE_CUDA - embed_cuda_ctx_t *cuda_ctx; -#endif - embed_tokenizer_t *tok; - embed_tokenizer_workspace_t *tok_ws; - /* BERT-family WordPiece tokenizer, selected by a vocab.txt probe at load. - * When wp_tok is set, tokenize_text wraps the ids with [CLS]/[SEP] and the - * Qwen byte-level BPE fields above stay NULL. */ - wordpiece_tokenizer_t *wp_tok; - wordpiece_workspace_t *wp_tok_ws; - /* SentencePiece/XLM-R tokenizer. When sp_tok is set, tokenize_text wraps - * core ids with <s> ... </s>. */ - sentencepiece_tokenizer_t *sp_tok; - sentencepiece_workspace_t *sp_tok_ws; - int cls_id; - int sep_id; - int dim; - int terminal_token_id; - int append_terminal_token; -} engine_t; - -static void engine_free_tokenizers(engine_t *e) { - embed_tokenizer_workspace_free(e->tok_ws); - embed_tokenizer_free(e->tok); - wordpiece_workspace_free(e->wp_tok_ws); - wordpiece_tokenizer_free(e->wp_tok); - sentencepiece_workspace_free(e->sp_tok_ws); - sentencepiece_tokenizer_free(e->sp_tok); -} - -typedef struct { - int *ids; - int n_tokens; -} token_buf_t; - -static int -engine_embed_batch(engine_t *e, const embed_input_t *inputs, int batch, float *out_embeddings) { -#ifdef USE_MLX - if (e->mlx_ctx) - return embed_mlx_encode_batch(e->mlx_ctx, inputs, batch, out_embeddings); -#endif -#ifdef USE_CUDA - if (e->cuda_ctx) - return embed_cuda_encode_batch(e->cuda_ctx, inputs, batch, out_embeddings); -#endif - return embed_model_encode_batch(e->model, e->workspace, inputs, batch, out_embeddings); -} - -static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) { - memset(out, 0, sizeof(*out)); - if (e->wp_tok) { - /* BERT: WordPiece encode, then wrap [CLS] ... [SEP] (the same contract - * as the BPE terminal-token append below). An empty clean yields the - * bare pair, matching Hugging Face. Grow the buffer in place. */ - int n = 0; - int *core = wordpiece_tokenizer_encode_with_workspace(e->wp_tok, e->wp_tok_ws, text, &n); - if (n < 0 || n > INT_MAX - 2) { - free(core); - return -1; - } - int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - return -1; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = e->cls_id; - ids[n + 1] = e->sep_id; - out->ids = ids; - out->n_tokens = n + 2; - } else if (e->sp_tok) { - int n = 0; - int *core = - sentencepiece_tokenizer_encode_with_workspace(e->sp_tok, e->sp_tok_ws, text, &n); - if (n < 0 || n > INT_MAX - 2) { - free(core); - return -1; - } - int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - return -1; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = e->cls_id; - ids[n + 1] = e->sep_id; - out->ids = ids; - out->n_tokens = n + 2; - } else { - out->ids = embed_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens); - if (!out->ids || out->n_tokens == 0) { - fprintf(stderr, "tokenization failed: %s\n", text); - free(out->ids); - out->ids = NULL; - out->n_tokens = 0; - return -1; - } - if (e->append_terminal_token) { - if (out->n_tokens == INT_MAX) { - free(out->ids); - memset(out, 0, sizeof(*out)); - return -1; - } - int *ids = (int *)realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); - if (!ids) { - free(out->ids); - memset(out, 0, sizeof(*out)); - return -1; - } - out->ids = ids; - out->ids[out->n_tokens++] = e->terminal_token_id; - } - } - - if (embed_verbose >= 1) { - fprintf(stderr, "tokens (%d): ", out->n_tokens); - for (int i = 0; i < out->n_tokens && i < 20; i++) - fprintf(stderr, "%d ", out->ids[i]); - if (out->n_tokens > 20) - fprintf(stderr, "..."); - fprintf(stderr, "\n"); - } - return 0; -} - -static void free_tokens(token_buf_t *tokens, int n) { - if (!tokens) - return; - for (int i = 0; i < n; i++) - free(tokens[i].ids); -} - -/* ======================================================================== - * Output helpers - * ======================================================================== */ - -static void print_embedding_raw(const float *emb, int dim) { - for (int i = 0; i < dim; i++) { - if (i > 0) - putchar(' '); - printf("%.8f", (double)emb[i]); - } - putchar('\n'); -} - -/* Single embedding as a JSON array of floats. */ -static void print_embedding_array(const float *emb, int dim) { - putchar('['); - for (int i = 0; i < dim; i++) { - if (i) - putchar(','); - printf("%.8f", (double)emb[i]); - } - puts("]"); -} - -static size_t engine_workspace_nbytes(const engine_t *e) { - return (e && e->workspace) ? embed_workspace_nbytes(e->workspace) : 0; -} - -static void -print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) { - printf("{\"embedding\":["); - for (int i = 0; i < dim; i++) { - if (i > 0) - putchar(','); - printf("%.8f", (double)emb[i]); - } - printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms, - workspace_bytes); - fflush(stdout); -} - -/* ======================================================================== - * Stdin mode: read lines from stdin, write JSON to stdout - * ======================================================================== */ - -static int process_stdin_batch(engine_t *e, char **lines, int n_lines) { - if (n_lines <= 0) - return 0; - - token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t)); - embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_lines * sizeof(embed_input_t)); - int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int)); - float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float)); - if (!tokens || !inputs || !input_to_line || !embs) { - fprintf(stderr, "OOM\n"); - free(tokens); - free(inputs); - free(input_to_line); - free(embs); - for (int i = 0; i < n_lines; i++) - printf("{\"error\":\"embedding failed\"}\n"); - fflush(stdout); - return 1; - } - - int n_inputs = 0; - for (int i = 0; i < n_lines; i++) { - if (tokenize_text(e, lines[i], &tokens[i]) == 0) { - inputs[n_inputs].ids = tokens[i].ids; - inputs[n_inputs].n_tokens = tokens[i].n_tokens; - input_to_line[n_inputs] = i; - n_inputs++; - } - } - - double ms = 0.0; - int rc = 0; - if (n_inputs > 0) { - double t0 = now_ms(); - rc = engine_embed_batch(e, inputs, n_inputs, embs); - ms = now_ms() - t0; - if (embed_verbose >= 1) - fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms); - } - - int next_valid = 0; - for (int i = 0; i < n_lines; i++) { - if (!tokens[i].ids) { - printf("{\"error\":\"tokenization failed\"}\n"); - continue; - } - if (rc != 0) { - printf("{\"error\":\"embedding failed\"}\n"); - continue; - } - while (next_valid < n_inputs && input_to_line[next_valid] != i) - next_valid++; - if (next_valid >= n_inputs) { - printf("{\"error\":\"embedding failed\"}\n"); - continue; - } - print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms, - engine_workspace_nbytes(e)); - next_valid++; - } - fflush(stdout); - - free_tokens(tokens, n_lines); - free(tokens); - free(inputs); - free(input_to_line); - free(embs); - return rc == 0 ? 0 : 1; -} - -static int run_stdin(engine_t *e, int batch_size) { - char line[65536]; - if (batch_size <= 0) - batch_size = 1; - - if (embed_verbose >= 1) - fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size); - - char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *)); - if (!batch_lines) { - fprintf(stderr, "OOM\n"); - return 1; - } - int n_batch = 0; - int rc = 0; - - while (fgets(line, sizeof(line), stdin)) { - /* Strip newline */ - size_t len = strlen(line); - while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) - line[--len] = '\0'; - if (len == 0) - continue; - - if (embed_verbose >= 1) - fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line, - len > 60 ? "..." : ""); - - batch_lines[n_batch] = strdup(line); - if (!batch_lines[n_batch]) { - fprintf(stderr, "OOM\n"); - rc = 1; - break; - } - n_batch++; - - if (n_batch == batch_size) { - if (process_stdin_batch(e, batch_lines, n_batch) != 0) - rc = 1; - for (int i = 0; i < n_batch; i++) { - free(batch_lines[i]); - batch_lines[i] = NULL; - } - n_batch = 0; - } - } - - if (n_batch > 0) { - if (process_stdin_batch(e, batch_lines, n_batch) != 0) - rc = 1; - for (int i = 0; i < n_batch; i++) - free(batch_lines[i]); - } - free(batch_lines); - - if (embed_verbose >= 1) - fprintf(stderr, "stdin: EOF\n"); - - return rc; -} - -/* ======================================================================== - * Batch mode: embed args or stdin lines, then print similarity or vectors - * ======================================================================== */ - -static int append_text(char ***texts, int *n_texts, int *cap, const char *s) { - if (*n_texts == *cap) { - int new_cap = *cap ? *cap * 2 : 8; - char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *)); - if (!new_texts) - return -1; - *texts = new_texts; - *cap = new_cap; - } - - (*texts)[*n_texts] = strdup(s); - if (!(*texts)[*n_texts]) - return -1; - (*n_texts)++; - return 0; -} - -/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one - * row of space-separated values per line. With print_embs the raw embeddings - * follow, one per line after a blank line (plain-text output only). */ -static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) { - if (json) { - putchar('['); - for (int i = 0; i < n; i++) { - if (i) - putchar(','); - putchar('['); - for (int j = 0; j < n; j++) { - if (j) - putchar(','); - printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim, - embs + (size_t)j * dim, dim)); - } - putchar(']'); - } - puts("]"); - return; - } - - for (int i = 0; i < n; i++) { - for (int j = 0; j < n; j++) { - if (j) - putchar(' '); - printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim, - embs + (size_t)j * dim, dim)); - } - putchar('\n'); - } - if (print_embs) { - putchar('\n'); - for (int i = 0; i < n; i++) - print_embedding_raw(embs + (size_t)i * dim, dim); - } -} - -static int run_batch( - engine_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) { - int n_texts = 0, cap = 0; - char **texts = NULL; - int rc = 0; - - if (arg_start < argc) { - for (int i = arg_start; i < argc; i++) { - if (append_text(&texts, &n_texts, &cap, argv[i]) != 0) - goto oom_texts; - } - } else { - char line[65536]; - while (fgets(line, sizeof(line), stdin)) { - size_t l = strlen(line); - while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r')) - line[--l] = '\0'; - if (l == 0) - continue; - if (append_text(&texts, &n_texts, &cap, line) != 0) - goto oom_texts; - } - } - - if (n_texts == 0) { - fprintf(stderr, "no input texts\n"); - free(texts); - return 1; - } - - int dim = e->dim; - token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t)); - embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_texts * sizeof(embed_input_t)); - float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float)); - if (!tokens || !inputs || !embs) { - fprintf(stderr, "OOM\n"); - free(tokens); - free(inputs); - free(embs); - for (int i = 0; i < n_texts; i++) - free(texts[i]); - free(texts); - return 1; - } - - for (int i = 0; i < n_texts; i++) { - if (embed_verbose >= 1) - fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]); - if (tokenize_text(e, texts[i], &tokens[i]) != 0) { - rc = 1; - goto done; - } - inputs[i].ids = tokens[i].ids; - inputs[i].n_tokens = tokens[i].n_tokens; - } - - int max_batch = batch_size > 0 ? batch_size : n_texts; - for (int start = 0; start < n_texts; start += max_batch) { - int cur = n_texts - start; - if (cur > max_batch) - cur = max_batch; - - double t0 = now_ms(); - if (engine_embed_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) { - fprintf(stderr, "forward pass failed\n"); - rc = 1; - goto done; - } - if (embed_verbose >= 1) - fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1, - cur, now_ms() - t0); - } - - if (n_texts == 1) { - if (json) - print_embedding_array(embs, dim); - else - print_embedding_raw(embs, dim); - } else { - print_matrix(embs, n_texts, dim, json, print_embs); - } - -done: - free_tokens(tokens, n_texts); - for (int i = 0; i < n_texts; i++) - free(texts[i]); - free(texts); - free(tokens); - free(inputs); - free(embs); - return rc; - -oom_texts: - fprintf(stderr, "OOM\n"); - for (int i = 0; i < n_texts; i++) - free(texts[i]); - free(texts); - return 1; -} - -/* ======================================================================== - * Main - * ======================================================================== */ - -int main(int argc, char *argv[]) { - const char *model_dir = NULL; - int n_threads = 0; - int print_embs = 0; - int verbose = 0; - int stdin_mode = 0; - int batch_size = 0; - int json = 0; - const char *prog = embed_prog_name(argv[0]); -#ifdef USE_MLX - int use_mlx = 0; - int mlx_quantize_bits = 0; - int mlx_quantize_group_size = 64; -#endif -#ifdef USE_CUDA - int use_cuda = 0; - const char *cuda_fast_gemm = NULL; - const char *cuda_weights = NULL; -#endif - - int arg_start = 1; - while (arg_start < argc && argv[arg_start][0] == '-') { - const char *f = argv[arg_start]; - if (!strcmp(f, "-d")) { - model_dir = argv[++arg_start]; - } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) { - embed_print_version(prog); - return 0; - } else if (!strcmp(f, "--build-info")) { - embed_print_build_info(prog); - return 0; - } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { - n_threads = atoi(argv[++arg_start]); - } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { - batch_size = atoi(argv[++arg_start]); - } else if (!strcmp(f, "--json")) { - json = 1; - } -#ifdef USE_MLX - else if (!strcmp(f, "--mlx-quant-bits")) { - mlx_quantize_bits = atoi(argv[++arg_start]); - if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); - return 1; - } - } else if (!strcmp(f, "--mlx-quant-group-size")) { - mlx_quantize_group_size = atoi(argv[++arg_start]); - if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); - return 1; - } - } -#endif -#ifdef USE_CUDA - else if (!strcmp(f, "--cuda-gemm-mode")) { - cuda_fast_gemm = argv[++arg_start]; - } else if (!strcmp(f, "--cuda-weight-dtype")) { - cuda_weights = argv[++arg_start]; - } -#endif - else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { - print_embs = 1; - } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { - verbose++; - } else if (!strcmp(f, "-vv")) { - verbose = 2; - } else if (!strcmp(f, "--stream")) { - stdin_mode = 1; - } -#ifdef USE_MLX - else if (!strcmp(f, "--mlx")) { - use_mlx = 1; - } -#endif -#ifdef USE_CUDA - else if (!strcmp(f, "--cuda")) { - use_cuda = 1; - } -#endif - else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { - print_usage(prog); - return 0; - } else if (!strcmp(f, "--")) { - /* End of options: everything after is positional text, even if it - * starts with a dash. */ - arg_start++; - break; - } else if (f[0] == '-' && f[1] == '-') { - /* An unrecognized --option is a hard error, not silently-embedded - * text. Catches typos and stale flags (e.g. the removed --backend) - * loudly instead of running the default path or embedding the flag. */ - fprintf(stderr, "unknown option: %s\n", f); - print_usage(prog); - return 1; - } else - break; - arg_start++; - } - - embed_verbose = verbose; - -#ifdef USE_CUDA - if (cuda_fast_gemm) { - if (!use_cuda) { - fprintf(stderr, "--cuda-gemm-mode requires --cuda\n"); - return 1; - } - if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { - fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); - return 1; - } - } - if (cuda_weights) { - if (!use_cuda) { - fprintf(stderr, "--cuda-weight-dtype requires --cuda\n"); - return 1; - } - if (!strcmp(cuda_weights, "bf16")) { - embed_cuda_set_weights_bf16(1); - } else if (!strcmp(cuda_weights, "f32")) { - embed_cuda_set_weights_bf16(0); - } else { - fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); - return 1; - } - } -#endif -#ifdef USE_MLX - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quant-bits requires --mlx\n"); - return 1; - } -#endif - - if (!model_dir || !model_dir[0]) { - fprintf(stderr, "model directory required (-d <dir>)\n"); - print_usage(prog); - return 1; - } - -#ifdef USE_MLX - if (use_mlx) { - if (verbose >= 1) - fprintf(stderr, "Using MLX GPU backend\n"); - } else -#endif -#ifdef USE_CUDA - if (use_cuda) { - if (verbose >= 1) - fprintf(stderr, "Using CUDA GPU backend\n"); - } else -#endif - { - if (n_threads <= 0) - n_threads = embed_get_num_cpus(); - embed_set_threads(n_threads); - if (verbose >= 1) - fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } - - /* Tokenizer selection by files present: WordPiece vocab.txt for BERT, - * SentencePiece .model for XLM-R/RoBERTa, otherwise Qwen byte-level BPE. */ - engine_t e = {0}; - e.cls_id = -1; - e.sep_id = -1; - double t0 = now_ms(); - char wp_path[1024]; - snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", model_dir); - if (access(wp_path, R_OK) == 0) { - e.wp_tok = wordpiece_tokenizer_load(model_dir); - if (e.wp_tok) { - e.cls_id = wordpiece_tokenizer_token_id(e.wp_tok, "[CLS]"); - e.sep_id = wordpiece_tokenizer_token_id(e.wp_tok, "[SEP]"); - } - if (!e.wp_tok || e.cls_id < 0 || e.sep_id < 0) { - fprintf(stderr, "failed to load WordPiece tokenizer: %s\n", wp_path); - wordpiece_tokenizer_free(e.wp_tok); - return 1; - } - } else { - char sp_path[1024]; - snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", model_dir); - if (access(sp_path, R_OK) != 0) { - snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", model_dir); - if (access(sp_path, R_OK) != 0) - snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", model_dir); - } - if (access(sp_path, R_OK) == 0) { - e.sp_tok = sentencepiece_tokenizer_load(model_dir); - if (e.sp_tok) { - e.cls_id = sentencepiece_tokenizer_token_id(e.sp_tok, "<s>"); - e.sep_id = sentencepiece_tokenizer_token_id(e.sp_tok, "</s>"); - } - if (!e.sp_tok || e.cls_id < 0 || e.sep_id < 0) { - fprintf(stderr, "failed to load SentencePiece tokenizer: %s\n", sp_path); - sentencepiece_tokenizer_free(e.sp_tok); - return 1; - } - } - } - if (!e.wp_tok && !e.sp_tok) { - char vocab_path[1024]; - snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); - e.tok = embed_tokenizer_load(vocab_path); - if (!e.tok) { - fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); - return 1; - } - } - if (verbose >= 1) - fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0); - - /* Model */ - t0 = now_ms(); - const embed_config_t *config = NULL; - -#ifdef USE_MLX - if (use_mlx) { - embed_mlx_options_t mlx_opts = { - .quantize_bits = mlx_quantize_bits, - .quantize_group_size = mlx_quantize_group_size, - }; - e.mlx_ctx = embed_mlx_load_with_options(model_dir, &mlx_opts); - if (!e.mlx_ctx) { - fprintf(stderr, "failed to load model\n"); - engine_free_tokenizers(&e); - return 1; - } - config = embed_mlx_config(e.mlx_ctx); - e.dim = config->hidden_size; - } else -#endif -#ifdef USE_CUDA - if (use_cuda) { - e.cuda_ctx = embed_cuda_load(model_dir); - if (!e.cuda_ctx) { - fprintf(stderr, "failed to load CUDA model\n"); - engine_free_tokenizers(&e); - return 1; - } - config = embed_cuda_config(e.cuda_ctx); - e.dim = config->hidden_size; - } else -#endif - { - e.model = embed_model_load(model_dir); - if (!e.model) { - fprintf(stderr, "failed to load model\n"); - engine_free_tokenizers(&e); - return 1; - } - e.workspace = embed_workspace_new(e.model); - if (!e.workspace) { - fprintf(stderr, "failed to allocate workspace\n"); - embed_model_free(e.model); - engine_free_tokenizers(&e); - return 1; - } - config = embed_model_config(e.model); - e.dim = config->hidden_size; - } - e.append_terminal_token = config->append_terminal_token; - /* The appended terminal token is the tokenizer's <|endoftext|>, resolved - * like the contextual separator: a vocab that defines it (test models) - * wins; otherwise the released-model family constant applies. */ - if (e.append_terminal_token) { - if (!e.tok) { - fprintf(stderr, "terminal-token appending requires the BPE tokenizer\n"); - if (e.workspace) - embed_workspace_free(e.workspace); - if (e.model) - embed_model_free(e.model); -#ifdef USE_MLX - if (e.mlx_ctx) - embed_mlx_free(e.mlx_ctx); -#endif -#ifdef USE_CUDA - if (e.cuda_ctx) - embed_cuda_free(e.cuda_ctx); -#endif - engine_free_tokenizers(&e); - return 1; - } - int eot = embed_tokenizer_token_id(e.tok, "<|endoftext|>"); - e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; - } - if (verbose >= 1) - fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0, -#ifdef USE_MLX - use_mlx ? " (MLX)" : -#endif -#ifdef USE_CUDA - use_cuda ? " (CUDA)" - : -#endif - ""); - - if (e.wp_tok) - e.wp_tok_ws = wordpiece_workspace_new(); - else if (e.sp_tok) - e.sp_tok_ws = sentencepiece_workspace_new(); - else - e.tok_ws = embed_tokenizer_workspace_new(); - if (!e.tok_ws && !e.wp_tok_ws && !e.sp_tok_ws) { - fprintf(stderr, "failed to allocate tokenizer workspace\n"); - if (e.workspace) - embed_workspace_free(e.workspace); - if (e.model) - embed_model_free(e.model); -#ifdef USE_MLX - if (e.mlx_ctx) - embed_mlx_free(e.mlx_ctx); -#endif -#ifdef USE_CUDA - if (e.cuda_ctx) - embed_cuda_free(e.cuda_ctx); -#endif - engine_free_tokenizers(&e); - return 1; - } - - /* Run */ - int rc; - if (stdin_mode) - rc = run_stdin(&e, batch_size > 0 ? batch_size : 1); - else - rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json); - - /* Cleanup */ - if (e.workspace) - embed_workspace_free(e.workspace); - if (e.model) - embed_model_free(e.model); -#ifdef USE_MLX - if (e.mlx_ctx) - embed_mlx_free(e.mlx_ctx); -#endif -#ifdef USE_CUDA - if (e.cuda_ctx) - embed_cuda_free(e.cuda_ctx); -#endif - engine_free_tokenizers(&e); - return rc; -} diff --git a/src/config.h b/src/config.h deleted file mode 100644 index e8353d3..0000000 --- a/src/config.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef EMBED_CONFIG_H -#define EMBED_CONFIG_H - -#include "embed.h" - -int embed_config_parse(embed_config_t *cfg, const char *model_dir); - -#endif /* EMBED_CONFIG_H */ diff --git a/src/embed.h b/src/embed.h deleted file mode 100644 index 5c4f2f3..0000000 --- a/src/embed.h +++ /dev/null @@ -1,357 +0,0 @@ -/* - * embed.h - embed API - */ - -#ifndef EMBED_H -#define EMBED_H - -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying EMBED_API are exported - * from the shared library; everything else stays internal. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - -#include <stddef.h> -#include <stdint.h> - -/* ======================================================================== - * Shared constants and family defaults - * ======================================================================== */ - -#define EMBED_VOCAB_SIZE 151936 -#define EMBED_HEAD_DIM 128 -#define EMBED_MAX_LAYERS 64 /* upper bound for stack arrays */ -#define EMBED_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */ - -/* ======================================================================== - * Model Configuration (populated from config.json at load time) - * ======================================================================== */ - -typedef enum { - EMBED_ATTENTION_BIDIRECTIONAL = 0, - EMBED_ATTENTION_CAUSAL = 1, -} embed_attention_mode_t; - -typedef enum { - EMBED_POOL_MEAN = 0, - EMBED_POOL_LAST_TOKEN = 1, - EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */ -} embed_pooling_mode_t; - -typedef enum { - EMBED_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (pplx, Qwen2/3) */ - EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */ -} embed_family_t; - -/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf - * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or - * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */ -typedef enum { - EMBED_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */ - EMBED_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */ -} embed_act_t; - -typedef struct { - embed_family_t family; - embed_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */ - int hidden_size; - int n_layers; - int n_heads; - int n_kv_heads; - int head_dim; - int q_dim; /* n_heads * head_dim */ - int kv_dim; /* n_kv_heads * head_dim */ - int intermediate_size; - int vocab_size; - int max_position_embeddings; /* BERT learned position table size */ - int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */ - int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */ - float rms_norm_eps; - float layer_norm_eps; /* BERT LayerNorm epsilon */ - float rope_theta; - int qk_norm; - int qkv_bias; - embed_attention_mode_t attention_mode; - embed_pooling_mode_t pooling_mode; - int normalize_embeddings; - /* Qwen3-Embedding pools the last token, which is the tokenizer's - * <|endoftext|> suffix added by its post-processor - not the model's chat - * eos_token_id (<|im_end|>). Text frontends append it and resolve the id - * from the tokenizer exactly like the contextual separator - * (EMBED_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The - * model forward consumes token ids exactly and never appends. */ - int append_terminal_token; -} embed_config_t; - -/* ======================================================================== - * Model + workspace ownership - * ======================================================================== */ - -typedef struct embed_model embed_model_t; -typedef struct embed_workspace embed_workspace_t; -typedef struct embed_late_model embed_late_model_t; -typedef struct embed_late_workspace embed_late_workspace_t; - -typedef struct { - const int *ids; - int n_tokens; -} embed_input_t; - -typedef struct { - int start; - int n_tokens; -} embed_span_t; - -typedef struct { - embed_input_t input; - const embed_span_t *spans; - int n_spans; -} embed_context_input_t; - -/* ======================================================================== - * API - * ======================================================================== */ - -/* - * Load immutable model data from a directory containing: - * config.json - model hyperparameters - * model.safetensors (or sharded) - F32 or BF16 weights - * vocab.json + merges.txt - BPE tokenizer - * - * Reads config.json to determine model size (0.6B vs 4B). - * Returns NULL on error. - */ -EMBED_API embed_model_t *embed_model_load(const char *model_dir); - -EMBED_API void embed_model_free(embed_model_t *model); - -/* - * Allocate/free mutable scratch buffers for a model. The model must outlive - * workspaces created from it. - */ -EMBED_API embed_workspace_t *embed_workspace_new(const embed_model_t *model); -EMBED_API void embed_workspace_free(embed_workspace_t *ws); - -/* Config accessor for opaque model ownership. */ -EMBED_API const embed_config_t *embed_model_config(const embed_model_t *model); - -/* - * Current mutable workspace memory owned by this workspace, in bytes. - * - * This includes the workspace struct, scratch buffers, and RoPE cache capacity. - * It does not include immutable mmap'd model weights. - */ -EMBED_API size_t embed_workspace_nbytes(const embed_workspace_t *ws); - -/* - * Compute embedding for a token sequence. - * - * 1. Token embedding lookup - * 2. N transformer layers (GQA + RoPE + SwiGLU) - * 3. Final RMSNorm - * 4. Model-defined pooling and optional L2 normalization - * - * pplx-embed-v1 uses bidirectional attention and unnormalized mean pooling. - * Qwen3-Embedding uses causal attention and normalized last-token pooling. - * Token IDs are consumed exactly as provided; text frontends append any - * model-required terminal token before calling this API. - * - * Returns malloc'd float[hidden_size] (caller frees). NULL on error. - */ -EMBED_API float *embed_model_encode(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens); - -/* - * Compute one embedding into caller-provided out[hidden_size]. - * Returns 0 on success, -1 on error. - */ -EMBED_API int embed_model_encode_into(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding); - -/* - * Compute embeddings for a true packed/ragged batch. - * - * out_embeddings is caller-provided [batch, hidden_size]. - * Each sequence attends only to tokens from the same input, and RoPE - * positions restart at zero for every input. - * - * Returns 0 on success, -1 on error. - */ -EMBED_API int embed_model_encode_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_input_t *inputs, - int batch, - float *out_embeddings); - -/* - * Apply the config's pooling mode to packed final hidden states. - * - * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive - * token count per sequence. The states must already include final RMSNorm. - */ -EMBED_API int embed_pool_batch(const embed_config_t *cfg, - const float *states, - const int *seq_lengths, - int batch, - float *out_embeddings); - -/* - * Run the transformer forward pass WITHOUT pooling. - * Returns the full [n_tokens * hidden_size] output after final RMSNorm. - * Caller frees the returned buffer. NULL on error. - * - * Useful for contextual (late-chunking) models where you need - * per-token embeddings before splitting by separator positions. - */ -EMBED_API float *embed_model_forward(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens); - -/* - * Run the transformer forward pass into caller-provided - * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error. - */ -EMBED_API int embed_model_forward_into(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_states); - -/* - * Pool embeddings from final hidden states. - * - * states is [n_tokens, hidden_size], normally produced by - * embed_model_forward_into(). Each span is mean pooled into - * out_embeddings[n_spans, hidden_size] without L2 normalization. - */ -EMBED_API int embed_pool_spans(const embed_config_t *cfg, - const float *states, - int n_tokens, - const embed_span_t *spans, - int n_spans, - float *out_embeddings); - -/* - * Run one contextual sequence and pool selected token spans. - * Returns 0 on success, -1 on error. - */ -EMBED_API int embed_model_encode_spans(const embed_model_t *model, - embed_workspace_t *ws, - const int *token_ids, - int n_tokens, - const embed_span_t *spans, - int n_spans, - float *out_embeddings); - -/* - * Run a packed/ragged contextual document batch and pool every selected span. - * - * Documents attend independently, RoPE positions restart for each document, - * and out_embeddings contains spans in document order. - */ -EMBED_API int embed_model_encode_spans_batch(const embed_model_t *model, - embed_workspace_t *ws, - const embed_context_input_t *inputs, - int batch, - float *out_embeddings); - -/* - * L2-normalize vec[dim] in place. - * - * Returns 0 on success. Returns -1 for invalid arguments or a zero-length - * vector, which cannot be normalized. - */ -EMBED_API int embed_l2_normalize(float *vec, int dim); - -/* - * Cosine similarity between two vectors. - * Returns 0 for invalid arguments or if either vector has zero length. - */ -EMBED_API float embed_cosine_similarity(const float *a, const float *b, int dim); - -/* - * Late-interaction model API. - * - * A late model produces one 128D vector per retained token. The vectors are - * projected by the snapshot's 1_Dense head and L2-normalized per token by - * default, matching ColBERT/MaxSim scoring semantics. - */ -EMBED_API embed_late_model_t *embed_late_model_load(const char *model_dir); -EMBED_API void embed_late_model_free(embed_late_model_t *model); - -EMBED_API embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model); -EMBED_API void embed_late_workspace_free(embed_late_workspace_t *ws); - -EMBED_API const embed_config_t *embed_late_model_config(const embed_late_model_t *model); -EMBED_API int embed_late_model_token_dim(const embed_late_model_t *model); - -/* - * Encode token_ids into out_vectors[n_tokens, token_dim]. - * - * If normalize is non-zero, every non-zero token vector is L2-normalized. - */ -EMBED_API int embed_late_model_encode_tokens(const embed_late_model_t *model, - embed_late_workspace_t *ws, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors); - -/* - * Encode n_docs documents in one block-diagonal forward and pack each - * document's kept token vectors back-to-back into out_vectors[total_keep, - * token_dim] in document order, where total_keep is the sum of n_keep. - * out_offsets[n_docs + 1] receives the prefix sum of kept counts, the layout - * embed_late_maxsim_batch consumes. keep[d] lists the n_keep[d] token indices to - * retain from document d. Returns 0 on success, -1 on error. - */ -EMBED_API int embed_late_model_encode_docs(const embed_late_model_t *model, - embed_late_workspace_t *ws, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - float *out_vectors, - int *out_offsets); - -/* - * Sum over query tokens of max dot product against document tokens. - */ -EMBED_API float embed_late_maxsim(const float *query_vectors, - int query_tokens, - const float *doc_vectors, - int doc_tokens, - int dim); - -/* - * Score one query against a packed/ragged batch of documents. - * - * doc_offsets has docs + 1 entries. Document i is stored in - * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim). - * scores is caller-provided [docs]. - */ -EMBED_API int embed_late_maxsim_batch(const float *query_vectors, - int query_tokens, - const float *doc_vectors, - const int *doc_offsets, - int docs, - int dim, - float *scores); - -/* Verbose level: 0=quiet, 1=info, 2=debug */ -EMBED_API extern int embed_verbose; - -#endif /* EMBED_H */ diff --git a/src/kernels_attn.c b/src/kernels_attn.c deleted file mode 100644 index 66f5ca5..0000000 --- a/src/kernels_attn.c +++ /dev/null @@ -1,469 +0,0 @@ -#include "kernels.h" -#include "kernels_impl.h" - -#include <math.h> -#include <stdlib.h> -#include <string.h> - -#if (defined(__AVX512F__) || defined(__AVX2__)) && \ - (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) -#include <immintrin.h> -#endif -#if defined(__ARM_NEON) || defined(__ARM_NEON__) -#include <arm_neon.h> -#endif - -#ifdef USE_BLAS -#ifdef __APPLE__ -#include <Accelerate/Accelerate.h> -#else -#include <cblas.h> -#endif -#endif - -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif - -/* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE) - * GQA, each with an online-softmax and a BLAS-GEMM path. */ - -/* ======================================================================== - * Attention Operations - * ======================================================================== */ - -float embed_dot_f32(const float *a, const float *b, int n) { return embed_dot_f32_impl(a, b, n); } - -static inline float embed_dot_f32_fast(const float *a, const float *b, int n) { - return embed_dot_f32_impl(a, b, n); -} - -/* dst = dst * scale */ -static inline void embed_vec_scale_inplace(float *dst, float scale, int n) { - embed_vec_scale_inplace_impl(dst, scale, n); -} - -/* dst += alpha * src */ -static inline void embed_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - embed_vec_axpy_inplace_impl(dst, src, alpha, n); -} - -/* dst = dst * correction + src */ -static inline void embed_vec_scale_add(float *dst, const float *src, float correction, int n) { - embed_vec_scale_add_impl(dst, src, correction, n); -} - -#define EMBED_PACKED_ATTN_QUERY_TILE 32 -#define EMBED_PACKED_ATTN_BLAS_MIN_SEQ 128 - -static void embed_bidirectional_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - - for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) - o_row[d] = 0.0f; - - for (int j = start; j < end; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - - float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - embed_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) { - float inv_sum = 1.0f / sum_exp; - embed_vec_scale_inplace(o_row, inv_sum, head_dim); - } - } -} - -#ifdef USE_BLAS -static void embed_bidirectional_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - int rows = query_end - query_start; - int keys = end - start; - - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, - k, kv_hidden, 0.0f, scores, keys); - embed_softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); -} -#endif - -typedef struct { - float *out; - const float *Q; - const float *K; - const float *V; - const int *offsets; - int batch; - int n_heads, n_kv_heads; - int head_dim; - float scale; - float *scores; - int max_seq; -} packed_gqa_attn_task_t; - -static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; - int task_id = 0; -#ifdef USE_BLAS - float *scores = - t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; -#endif - - for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; -#ifdef USE_BLAS - if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { - embed_bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, - t->head_dim, t->scale, h, q0, q1, scores); - continue; - } -#endif - embed_bidirectional_gqa_attention_packed_online_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); - } - } - } -} - -size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { -#ifdef USE_BLAS - if (!offsets || batch <= 0) - return 0; - - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) - max_seq = len; - } - if (max_seq < EMBED_PACKED_ATTN_BLAS_MIN_SEQ) - return 0; - if ((size_t)max_seq > - SIZE_MAX / (sizeof(float) * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)embed_num_threads())) - return 0; - return (size_t)embed_num_threads() * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); -#else - (void)offsets; - (void)batch; - return 0; -#endif -} - -void embed_bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * len * n_heads; - if (len > max_seq) - max_seq = len; - } - - float *scores = NULL; -#ifdef USE_BLAS - size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) - scores = scratch; -#else - (void)scratch; - (void)scratch_bytes; -#endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; - if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { - parallel_for(packed_gqa_attn_worker, &task); - } else { - packed_gqa_attn_worker(0, 1, &task); - } -} - -void embed_bidirectional_gqa_attention_packed(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale) { - size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - embed_bidirectional_gqa_attention_packed_with_scratch( - out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes); - free(scores); -} - -static void embed_causal_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - - for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; - - float max_score = -1e30f; - float sum_exp = 0.0f; - for (int d = 0; d < head_dim; d++) - o_row[d] = 0.0f; - - for (int j = start; j <= i; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; - - if (score > max_score) { - float correction = expf(max_score - score); - sum_exp = sum_exp * correction + 1.0f; - embed_vec_scale_add(o_row, v_row, correction, head_dim); - max_score = score; - } else { - float wt = expf(score - max_score); - sum_exp += wt; - embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); - } - } - - if (sum_exp > 0.0f) - embed_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); - } -} - -#ifdef USE_BLAS -static void embed_causal_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; - int rows = query_end - query_start; - int keys = query_end - start; - - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, - k, kv_hidden, 0.0f, scores, keys); - for (int r = 0; r < rows; r++) { - int allowed = query_start + r - start + 1; - float *row = scores + (size_t)r * keys; - for (int j = allowed; j < keys; j++) - row[j] = -INFINITY; - } - embed_softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); -} -#endif - -static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; - int task_id = 0; -#ifdef USE_BLAS - float *scores = - t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; -#endif - - for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; -#ifdef USE_BLAS - if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { - embed_causal_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1, scores); - continue; - } -#endif - embed_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, - t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); - } - } - } -} - -void embed_causal_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * (len + 1) / 2 * n_heads; - if (len > max_seq) - max_seq = len; - } - - float *scores = NULL; -#ifdef USE_BLAS - size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - if (required != 0 && scratch && scratch_bytes >= required) - scores = scratch; -#else - (void)scratch; - (void)scratch_bytes; -#endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; - if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) - parallel_for(packed_causal_gqa_attn_worker, &task); - else - packed_causal_gqa_attn_worker(0, 1, &task); -} - -void embed_causal_gqa_attention_packed(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale) { - size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); - float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - embed_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, - head_dim, scale, scores, scratch_bytes); - free(scores); -} diff --git a/src/kernels_impl.h b/src/kernels_impl.h deleted file mode 100644 index 4ee7d74..0000000 --- a/src/kernels_impl.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * embed_kernels_impl.h - internal architecture dispatch for hot kernels - */ - -#ifndef EMBED_KERNELS_IMPL_H -#define EMBED_KERNELS_IMPL_H - -#include <stdint.h> - -void embed_bf16_matvec_fused_generic( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -float embed_dot_f32_generic(const float *a, const float *b, int n); -void embed_vec_scale_inplace_generic(float *dst, float scale, int n); -void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); -void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n); - -#ifdef __ARM_NEON -void embed_bf16_matvec_fused_neon( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -float embed_dot_f32_neon(const float *a, const float *b, int n); -void embed_vec_scale_inplace_neon(float *dst, float scale, int n); -void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); -void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n); - -#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_neon -#define embed_dot_f32_impl embed_dot_f32_neon -#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_neon -#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_neon -#define embed_vec_scale_add_impl embed_vec_scale_add_neon - -#elif defined(__AVX2__) && defined(__FMA__) -void embed_bf16_matvec_fused_avx( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -float embed_dot_f32_avx(const float *a, const float *b, int n); -void embed_vec_scale_inplace_avx(float *dst, float scale, int n); -void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); -void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n); - -#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_avx -#define embed_dot_f32_impl embed_dot_f32_avx -#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_avx -#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_avx -#define embed_vec_scale_add_impl embed_vec_scale_add_avx - -#else -#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_generic -#define embed_dot_f32_impl embed_dot_f32_generic -#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_generic -#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_generic -#define embed_vec_scale_add_impl embed_vec_scale_add_generic -#endif - -#endif /* EMBED_KERNELS_IMPL_H */ diff --git a/src/kernels_norm.c b/src/kernels_norm.c deleted file mode 100644 index 6bae71d..0000000 --- a/src/kernels_norm.c +++ /dev/null @@ -1,245 +0,0 @@ -#include "kernels.h" -#include "kernels_impl.h" - -#include <math.h> -#include <stdlib.h> -#include <string.h> - -#if (defined(__AVX512F__) || defined(__AVX2__)) && \ - (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) -#include <immintrin.h> -#endif -#if defined(__ARM_NEON) || defined(__ARM_NEON__) -#include <arm_neon.h> -#endif - -#ifdef USE_BLAS -#ifdef __APPLE__ -#include <Accelerate/Accelerate.h> -#else -#include <cblas.h> -#endif -#endif - -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif - -/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the - * per-head RMSNorm used for QK-norm. */ - -/* ======================================================================== - * Normalization - * ======================================================================== */ - -/* Rows split across the pool only when the tensor is big enough to pay - * for the dispatch. */ -#define EMBED_RMS_NORM_PARALLEL_ELEMS 262144 - -static void embed_rms_norm_range( - float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) { - for (int s = start; s < end; s++) { - const float *x_row = x + (size_t)s * hidden; - float *out_row = out + (size_t)s * hidden; - -#if defined(__AVX512F__) && defined(__FMA__) - __m512 accv = _mm512_setzero_ps(); - int i = 0; - for (; i + 16 <= hidden; i += 16) { - __m512 v = _mm512_loadu_ps(x_row + i); - accv = _mm512_fmadd_ps(v, v, accv); - } - float sum_sq = _mm512_reduce_add_ps(accv); - for (; i < hidden; i++) - sum_sq += x_row[i] * x_row[i]; -#elif defined(__AVX2__) && defined(__FMA__) - __m256 accv = _mm256_setzero_ps(); - int i = 0; - for (; i + 8 <= hidden; i += 8) { - __m256 v = _mm256_loadu_ps(x_row + i); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float sum_sq = _mm_cvtss_f32(acc128); - for (; i < hidden; i++) - sum_sq += x_row[i] * x_row[i]; -#else - float sum_sq = 0.0f; - for (int i = 0; i < hidden; i++) { - sum_sq += x_row[i] * x_row[i]; - } -#endif - float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps); - -#if defined(__AVX512F__) - __m512 scale = _mm512_set1_ps(rms_inv); - int j = 0; - for (; j + 16 <= hidden; j += 16) { - __m512 vx = _mm512_loadu_ps(x_row + j); - __m512 vw = _mm512_loadu_ps(weight + j); - _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale)); - } - for (; j < hidden; j++) - out_row[j] = x_row[j] * rms_inv * weight[j]; -#elif defined(__AVX2__) - __m256 scale = _mm256_set1_ps(rms_inv); - int j = 0; - for (; j + 8 <= hidden; j += 8) { - __m256 vx = _mm256_loadu_ps(x_row + j); - __m256 vw = _mm256_loadu_ps(weight + j); - _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale)); - } - for (; j < hidden; j++) - out_row[j] = x_row[j] * rms_inv * weight[j]; -#else - for (int i = 0; i < hidden; i++) { - out_row[i] = x_row[i] * rms_inv * weight[i]; - } -#endif - } -} - -typedef struct { - float *out; - const float *x; - const float *weight; - int seq_len; - int hidden; - float eps; -} rms_norm_task_t; - -static void rms_norm_worker(int tid, int n_threads, void *arg) { - rms_norm_task_t *t = (rms_norm_task_t *)arg; - int chunk = (t->seq_len + n_threads - 1) / n_threads; - int start = tid * chunk; - int end = start + chunk; - if (end > t->seq_len) - end = t->seq_len; - if (start >= end) - return; - embed_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps); -} - -void embed_rms_norm( - float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) { - if (seq_len <= 0 || hidden <= 0) - return; - - long long elems = (long long)seq_len * hidden; - if (embed_num_threads() > 1 && elems >= EMBED_RMS_NORM_PARALLEL_ELEMS) { - rms_norm_task_t task = { - .out = out, - .x = x, - .weight = weight, - .seq_len = seq_len, - .hidden = hidden, - .eps = eps, - }; - parallel_for(rms_norm_worker, &task); - } else { - embed_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); - } -} - -void embed_rms_norm_per_head( - float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) { - /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ - int hidden = n_heads * head_dim; - for (int s = 0; s < seq_len; s++) { - for (int h = 0; h < n_heads; h++) { - float *vec = x + s * hidden + h * head_dim; - -#if defined(__AVX512F__) && defined(__FMA__) - __m512 accv = _mm512_setzero_ps(); - int d = 0; - for (; d + 16 <= head_dim; d += 16) { - __m512 v = _mm512_loadu_ps(vec + d); - accv = _mm512_fmadd_ps(v, v, accv); - } - float sum_sq = _mm512_reduce_add_ps(accv); - for (; d < head_dim; d++) - sum_sq += vec[d] * vec[d]; -#elif defined(__AVX2__) && defined(__FMA__) - __m256 accv = _mm256_setzero_ps(); - int d = 0; - for (; d + 8 <= head_dim; d += 8) { - __m256 v = _mm256_loadu_ps(vec + d); - accv = _mm256_fmadd_ps(v, v, accv); - } - __m128 acc128 = - _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1)); - acc128 = _mm_hadd_ps(acc128, acc128); - acc128 = _mm_hadd_ps(acc128, acc128); - float sum_sq = _mm_cvtss_f32(acc128); - for (; d < head_dim; d++) - sum_sq += vec[d] * vec[d]; -#else - float sum_sq = 0.0f; - for (int d = 0; d < head_dim; d++) { - sum_sq += vec[d] * vec[d]; - } -#endif - float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps); - -#if defined(__AVX512F__) - __m512 scale = _mm512_set1_ps(rms_inv); - int j = 0; - for (; j + 16 <= head_dim; j += 16) { - __m512 v = _mm512_loadu_ps(vec + j); - __m512 w = _mm512_loadu_ps(weight + j); - _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale)); - } - for (; j < head_dim; j++) - vec[j] = vec[j] * rms_inv * weight[j]; -#elif defined(__AVX2__) - __m256 scale = _mm256_set1_ps(rms_inv); - int j = 0; - for (; j + 8 <= head_dim; j += 8) { - __m256 v = _mm256_loadu_ps(vec + j); - __m256 w = _mm256_loadu_ps(weight + j); - _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale)); - } - for (; j < head_dim; j++) - vec[j] = vec[j] * rms_inv * weight[j]; -#else - for (int d = 0; d < head_dim; d++) { - vec[d] = vec[d] * rms_inv * weight[d]; - } -#endif - } - } -} - -/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each - * row's mean and variance are reduced before the row is rewritten. Scalar for - * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */ -void embed_layer_norm(float *out, - const float *x, - const float *gamma, - const float *beta, - int seq_len, - int hidden, - float eps) { - if (seq_len <= 0 || hidden <= 0) - return; - float inv_h = 1.0f / (float)hidden; - for (int s = 0; s < seq_len; s++) { - const float *row = x + (size_t)s * hidden; - float *dst = out + (size_t)s * hidden; - float mean = 0.0f; - for (int d = 0; d < hidden; d++) - mean += row[d]; - mean *= inv_h; - float var = 0.0f; - for (int d = 0; d < hidden; d++) { - float diff = row[d] - mean; - var += diff * diff; - } - var *= inv_h; - float inv_std = 1.0f / sqrtf(var + eps); - for (int d = 0; d < hidden; d++) - dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d]; - } -} diff --git a/src/late.c b/src/late.c deleted file mode 100644 index 6ba52cc..0000000 --- a/src/late.c +++ /dev/null @@ -1,383 +0,0 @@ -#include "embed.h" -#include "model_internal.h" -#include "kernels.h" - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <limits.h> -#include <float.h> - -/* ======================================================================== - * Late-interaction model - * - * A late-interaction model is a pooled base model plus a 1_Dense linear - * projection from hidden states to token vectors. Instead of pooling to one - * vector, it keeps one projected vector per kept token and scores query/doc - * pairs with MaxSim. It reuses the base model's packed forward pass and - * workspace; only the projection, the token-vector buffer, and the scoring - * live here. - * ======================================================================== */ - -struct embed_late_model { - embed_model_t *base; - safetensors_file_t *dense_sf; - embed_weight_ref_t projection; /* [token_dim, hidden] */ - int token_dim; -}; - -struct embed_late_workspace { - const embed_late_model_t *model; - embed_workspace_t *base_ws; - float *states; - int states_seq_cap; -}; - -static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) { - if (!ws || !ws->model || n_tokens <= 0) - return -1; - if (n_tokens <= ws->states_seq_cap) - return 0; - int hidden = ws->model->base->config.hidden_size; - if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0) - return -1; - ws->states_seq_cap = n_tokens; - return 0; -} - -static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) { - if (!sf || !name) - return NULL; - for (int i = 0; i < sf->num_tensors; i++) { - if (strcmp(sf->tensors[i].name, name) == 0) - return &sf->tensors[i]; - } - return NULL; -} - -embed_late_model_t *embed_late_model_load(const char *model_dir) { - if (!model_dir_has_late_projection(model_dir)) { - fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n"); - return NULL; - } - - embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late)); - if (!late) - return NULL; - - late->base = model_load_range_ex(model_dir, 0, -1, 1); - if (!late->base) - goto fail; - - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir); - if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "embed_late_model_load: model path too long\n"); - goto fail; - } - - late->dense_sf = safetensors_open(path); - if (!late->dense_sf) { - fprintf(stderr, "embed_late_model_load: cannot open %s\n", path); - goto fail; - } - - const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight"); - if (!t) { - fprintf(stderr, "embed_late_model_load: missing linear.weight\n"); - goto fail; - } - if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) { - fprintf(stderr, "embed_late_model_load: bad projection shape\n"); - goto fail; - } - - int64_t shape[2] = {t->shape[0], late->base->config.hidden_size}; - if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1)) - goto fail; - - /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */ - late->projection = weight_ref_from_tensor(late->dense_sf, t); - if (!late->projection.data) - goto fail; - late->token_dim = (int)t->shape[0]; - return late; - -fail: - embed_late_model_free(late); - return NULL; -} - -void embed_late_model_free(embed_late_model_t *model) { - if (!model) - return; - embed_model_free(model->base); - /* Free the projection before the mmap: only an F16-widened ref owns its - * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */ - if (model->projection.owned) - free((void *)model->projection.data); - safetensors_close(model->dense_sf); - free(model); -} - -embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) { - if (!model || !model->base) - return NULL; - embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws)); - if (!ws) - return NULL; - ws->model = model; - ws->base_ws = embed_workspace_new(model->base); - if (!ws->base_ws) { - free(ws); - return NULL; - } - return ws; -} - -void embed_late_workspace_free(embed_late_workspace_t *ws) { - if (!ws) - return; - embed_workspace_free(ws->base_ws); - free(ws->states); - free(ws); -} - -const embed_config_t *embed_late_model_config(const embed_late_model_t *model) { - return model && model->base ? &model->base->config : NULL; -} - -int embed_late_model_token_dim(const embed_late_model_t *model) { - return model ? model->token_dim : 0; -} - -/* Internal (embed_internal.h): expose the base model and projection so GPU - * backends can upload them to device without seeing struct embed_late_model. */ -embed_model_t *embed_late_model_base(const embed_late_model_t *model) { - return model ? model->base : NULL; -} - -const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) { - return model ? &model->projection : NULL; -} - -int embed_late_model_encode_tokens(const embed_late_model_t *model, - embed_late_workspace_t *ws, - const int *token_ids, - int n_tokens, - int normalize, - float *out_vectors) { - if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors || - model->token_dim <= 0) - return -1; - - int hidden = model->base->config.hidden_size; - int dim = model->token_dim; - if (ensure_late_state_buffer(ws, n_tokens) != 0) - return -1; - if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0) - return -1; - - linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden, - dim); - if (normalize) { - for (int i = 0; i < n_tokens; i++) { - if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) - return -1; - } - } - return 0; -} - -float embed_late_maxsim(const float *query_vectors, - int query_tokens, - const float *doc_vectors, - int doc_tokens, - int dim) { - if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0) - return 0.0f; - - float score = 0.0f; - for (int qi = 0; qi < query_tokens; qi++) { - const float *q = query_vectors + (size_t)qi * dim; - float best = -FLT_MAX; - for (int di = 0; di < doc_tokens; di++) { - const float *d = doc_vectors + (size_t)di * dim; - float dot = embed_dot_f32(q, d, dim); - if (dot > best) - best = dot; - } - score += best; - } - return score; -} - -int embed_late_maxsim_batch(const float *query_vectors, - int query_tokens, - const float *doc_vectors, - const int *doc_offsets, - int docs, - int dim, - float *scores) { - if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || - docs <= 0 || dim <= 0 || doc_offsets[0] != 0) - return -1; - - for (int i = 0; i < docs; i++) { - if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i]) - return -1; - } - -#ifdef USE_BLAS - /* One similarity GEMM per group of whole candidates: - * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document - * row-max plus sum. The group token budget bounds the scratch matrix; - * one oversized document still runs alone in a group of its own. */ - enum { LATE_GROUP_TOKEN_BUDGET = 4096 }; - int total = doc_offsets[docs]; - int budget = LATE_GROUP_TOKEN_BUDGET; - size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget); - int largest = 0; - for (int i = 0; i < docs; i++) { - int len = doc_offsets[i + 1] - doc_offsets[i]; - if (len > largest) - largest = len; - } - if ((size_t)query_tokens * (size_t)largest > cap) - cap = (size_t)query_tokens * (size_t)largest; - float *sim = (float *)malloc(cap * sizeof(float)); - if (sim) { - int doc = 0; - while (doc < docs) { - int first = doc; - int start = doc_offsets[first]; - doc++; - while (doc < docs && doc_offsets[doc + 1] - start <= budget && - doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens)) - doc++; - int group_tokens = doc_offsets[doc] - start; - - embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, - group_tokens); - - for (int i = first; i < doc; i++) { - int s0 = doc_offsets[i] - start; - int s1 = doc_offsets[i + 1] - start; - float score = 0.0f; - for (int qi = 0; qi < query_tokens; qi++) { - const float *row = sim + (size_t)qi * group_tokens; - float best = row[s0]; - for (int t = s0 + 1; t < s1; t++) - if (row[t] > best) - best = row[t]; - score += best; - } - scores[i] = score; - } - } - free(sim); - return 0; - } - /* Allocation failure: fall through to the scalar path. */ -#endif - - for (int i = 0; i < docs; i++) { - int start = doc_offsets[i]; - int end = doc_offsets[i + 1]; - scores[i] = embed_late_maxsim(query_vectors, query_tokens, - doc_vectors + (size_t)start * dim, end - start, dim); - } - return 0; -} - -/* Late-interaction batch encoder. Lives here, beside the per-document encoder, - * and reuses the base model's packed/ragged batch forward pass. */ -int embed_late_model_encode_docs(const embed_late_model_t *model, - embed_late_workspace_t *ws, - const int *const *doc_ids, - const int *n_tokens, - const int *const *keep, - const int *n_keep, - int n_docs, - int normalize, - float *out_vectors, - int *out_offsets) { - if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep || - n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0) - return -1; - - int hidden = model->base->config.hidden_size; - int dim = model->token_dim; - - /* Pack all candidates for one block-diagonal forward. inputs/offsets index - * the full token stream; out_offsets is the prefix sum of kept tokens. */ - embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs)); - int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets)); - if (!inputs || !offsets) { - free(inputs); - free(offsets); - return -1; - } - - int total_keep = 0; - out_offsets[0] = 0; - for (int d = 0; d < n_docs; d++) { - if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 || - n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) { - free(inputs); - free(offsets); - return -1; - } - for (int k = 0; k < n_keep[d]; k++) { - if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) { - free(inputs); - free(offsets); - return -1; - } - } - inputs[d].ids = doc_ids[d]; - inputs[d].n_tokens = n_tokens[d]; - total_keep += n_keep[d]; - out_offsets[d + 1] = total_keep; - } - - int total_seq = 0, max_seq = 0; - int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq); - if (rc == 0) - rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq, - max_seq, 1); - free(inputs); - if (rc != 0) { - free(offsets); - return -1; - } - - /* Gather each document's kept hidden rows into ws->states (its natural - * [seq, hidden] shape), then project the packed [total_keep, hidden] block - * to [total_keep, dim] in one GEMM, matching the per-document path. */ - if (ensure_late_state_buffer(ws, total_keep) != 0) { - free(offsets); - return -1; - } - const float *states = ws->base_ws->x; - int pos = 0; - for (int d = 0; d < n_docs; d++) { - int start = offsets[d]; - for (int k = 0; k < n_keep[d]; k++) { - memcpy(ws->states + (size_t)pos * hidden, - states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float)); - pos++; - } - } - free(offsets); - - linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep, - hidden, dim); - if (normalize) { - for (int i = 0; i < total_keep; i++) { - if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0) - return -1; - } - } - return 0; -} diff --git a/src/server.c b/src/server.c deleted file mode 100644 index 808c6f8..0000000 --- a/src/server.c +++ /dev/null @@ -1,1042 +0,0 @@ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include "build.h" -#include "server.h" -#include "server_internal.h" -#include "server_util.h" -#include "sbuf.h" -#include "embed.h" -#include "safetensors.h" -#include "kernels.h" -#include "tokenizer_bpe.h" -#include "tokenizer_wordpiece.h" - -#ifdef USE_MLX -#include "mlx.h" -#endif -#ifdef USE_CUDA -#include "cuda.h" -#endif - -#include "deps/ae/ae.h" -#include "deps/ae/anet.h" - -#include <arpa/inet.h> -#include <ctype.h> -#include <errno.h> -#include <fcntl.h> -#include <limits.h> -#include <netinet/in.h> -#include <pthread.h> -#include <signal.h> -#include <stdbool.h> -#include <stdint.h> -#include <stdarg.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <strings.h> -#include <sys/socket.h> -#include <sys/time.h> -#include <sys/types.h> -#include <time.h> -#include <unistd.h> - -#ifdef __APPLE__ -#include <sys/sysctl.h> -#endif - -#include <cjson/cJSON.h> - -static volatile sig_atomic_t g_stop_requested = 0; -static volatile sig_atomic_t g_signal_wfd = -1; - -static void stop_signal_handler(int sig) { - (void)sig; - if (g_stop_requested) - _exit(130); - int saved = errno; - g_stop_requested = 1; - int fd = (int)g_signal_wfd; - if (fd >= 0) { - const unsigned char byte = 1; - if (write(fd, &byte, 1) < 0) { - /* best-effort stop wakeup; a failed self-pipe write is ignored */ - } - } - errno = saved; -} - -static bool auth_ok(http_server *s, const char *auth) { - if (!s->api_key || !s->api_key[0]) - return true; - if (!auth) - return false; - const char prefix[] = "Bearer "; - return !strncmp(auth, prefix, sizeof(prefix) - 1) && - !strcmp(auth + sizeof(prefix) - 1, s->api_key); -} - -void dispatch_request(client *c) { - aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); - - if (!strcmp(c->req.method, "OPTIONS")) { - append_http_response(c, 204, NULL, "", 0); - if (queue_write(c) != AE_OK) - close_client(c); - return; - } - - if (!route_is_inference(c->req.method, c->req.path)) { - size_t len; - char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); - append_http_response(c, 404, "application/json", body, len); - free(body); - if (queue_write(c) != AE_OK) - close_client(c); - return; - } - - if (!auth_ok(c->srv, c->req.auth)) { - size_t len; - char *body = - json_error_body("missing or invalid bearer token", "authentication_error", &len); - append_http_response(c, 401, "application/json", body, len); - free(body); - if (queue_write(c) != AE_OK) - close_client(c); - return; - } - - job *j = xcalloc(1, sizeof(*j)); - j->srv = c->srv; - j->c = c; - snprintf(j->method, sizeof(j->method), "%s", c->req.method); - snprintf(j->path, sizeof(j->path), "%s", c->req.path); - j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); - j->body_len = c->req.body_len; - j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; - j->created_ns = nstime(); - client_incref(c); - enqueue_job(j); -} - -static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)loop; - (void)mask; - http_server *s = clientData; - char cip[64]; - int cport = 0; - for (;;) { - int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); - if (cfd == ANET_ERR) { - if (errno == EAGAIN || errno == EWOULDBLOCK) - return; - if (errno == EINTR) - continue; - return; - } - set_nonblock(cfd); - client *c = xcalloc(1, sizeof(*c)); - c->srv = s; - c->fd = cfd; - c->refcount = 1; - c->last_active_ms = mstime(); - c->linked = 1; - c->next = s->clients; - if (s->clients) - s->clients->prev = c; - s->clients = c; - s->n_clients++; - if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) - close_client(c); - } -} - -static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { - (void)loop; - (void)id; - http_server *s = clientData; - uint64_t now = mstime(); - client *c = s->clients; - while (c) { - client *next = c->next; - /* refcount > 1 means a request from this client is queued or being - * processed by the worker; long batch queues (e.g. 4B long-document - * batches) must not be reaped as idle. */ - if (!c->cancelled && c->refcount <= 1 && - now - c->last_active_ms > EMBED_HTTP_CLIENT_TIMEOUT_MS) - close_client(c); - c = next; - } - return EMBED_HTTP_SWEEP_MS; -} - -static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { - (void)fd; - (void)mask; - http_server *s = clientData; - unsigned char tmp[128]; - while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) { - } - aeStop(loop); -} - -static void *worker_main(void *arg) { - http_server *s = arg; -#ifdef USE_MLX - if (s->use_mlx) { - int rc = 0; - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - embed_mlx_options_t mlx_opts = { - .quantize_bits = s->mlx_quantize_bits, - .quantize_group_size = s->mlx_quantize_group_size, - }; - if (m->info->kind == MODEL_KIND_LATE) - m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts); - else - m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts); - if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) || - (!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) { - server_log("embed-server: failed to load MLX model on worker: %s", m->path); - rc = 1; - break; - } - const embed_config_t *config = m->info->kind == MODEL_KIND_LATE - ? embed_mlx_late_config(m->mlx_late_ctx) - : embed_mlx_config(m->mlx_ctx); - int token_dim = - m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0; - if (configure_loaded_model(m, config, token_dim) != 0 || - (m->info->kind == MODEL_KIND_LATE && - (m->late_mask_id >= config->vocab_size || - m->late_query_prefix_id >= config->vocab_size || - m->late_document_prefix_id >= config->vocab_size))) { - rc = 1; - break; - } - } - pthread_mutex_lock(&s->mu); - s->worker_init_rc = rc; - s->worker_ready = 1; - if (rc) - s->stopping = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - if (rc) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->mlx_ctx) { - embed_mlx_free(m->mlx_ctx); - m->mlx_ctx = NULL; - } - if (m->mlx_late_ctx) { - embed_mlx_late_free(m->mlx_late_ctx); - m->mlx_late_ctx = NULL; - } - } - return NULL; - } - } else -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - int rc = 0; - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->info->kind == MODEL_KIND_LATE) - m->cuda_late_ctx = embed_cuda_late_load(m->path); - else - m->cuda_ctx = embed_cuda_load(m->path); - if ((!m->cuda_ctx && m->info->kind != MODEL_KIND_LATE) || - (!m->cuda_late_ctx && m->info->kind == MODEL_KIND_LATE)) { - server_log("embed-server: failed to load CUDA model on worker: %s", m->path); - rc = 1; - break; - } - const embed_config_t *config = m->info->kind == MODEL_KIND_LATE - ? embed_cuda_late_config(m->cuda_late_ctx) - : embed_cuda_config(m->cuda_ctx); - int token_dim = - m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_token_dim(m->cuda_late_ctx) : 0; - if (configure_loaded_model(m, config, token_dim) != 0 || - (m->info->kind == MODEL_KIND_LATE && - (m->late_mask_id >= config->vocab_size || - m->late_query_prefix_id >= config->vocab_size || - m->late_document_prefix_id >= config->vocab_size))) { - rc = 1; - break; - } - } - pthread_mutex_lock(&s->mu); - s->worker_init_rc = rc; - s->worker_ready = 1; - if (rc) - s->stopping = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - if (rc) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->cuda_ctx) { - embed_cuda_free(m->cuda_ctx); - m->cuda_ctx = NULL; - } - if (m->cuda_late_ctx) { - embed_cuda_late_free(m->cuda_late_ctx); - m->cuda_late_ctx = NULL; - } - } - return NULL; - } - } else -#endif - { - pthread_mutex_lock(&s->mu); - s->worker_ready = 1; - pthread_cond_broadcast(&s->cv); - pthread_mutex_unlock(&s->mu); - } - - for (;;) { - job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS]; - int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS); - if (n_jobs == 0) - break; - process_job_group(s, jobs, n_jobs); - } -#ifdef USE_MLX - if (s->use_mlx) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->mlx_ctx) { - embed_mlx_free(m->mlx_ctx); - m->mlx_ctx = NULL; - } - if (m->mlx_late_ctx) { - embed_mlx_late_free(m->mlx_late_ctx); - m->mlx_late_ctx = NULL; - } - } - } -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - if (m->cuda_ctx) { - embed_cuda_free(m->cuda_ctx); - m->cuda_ctx = NULL; - } - if (m->cuda_late_ctx) { - embed_cuda_late_free(m->cuda_late_ctx); - m->cuda_late_ctx = NULL; - } - } - } -#endif - return NULL; -} - -static int listen_on(const char *host, int port) { - char portbuf[32]; - snprintf(portbuf, sizeof(portbuf), "%d", port); - char err[ANET_ERR_LEN]; - int fd = anetTcpServer(err, port, (char *)host, EMBED_HTTP_BACKLOG); - (void)portbuf; - if (fd == ANET_ERR) { - server_log("embed-server: listen failed on %s:%d: %s", host, port, err); - return -1; - } - set_nonblock(fd); - return fd; -} - -static uint64_t physical_memory_nbytes(void) { -#ifdef __APPLE__ - uint64_t bytes = 0; - size_t len = sizeof(bytes); - if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0) - return bytes; -#endif - long pages = sysconf(_SC_PHYS_PAGES); - long page_size = sysconf(_SC_PAGESIZE); - if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size) - return 0; - return (uint64_t)pages * (uint64_t)page_size; -} - -static int validate_model_specs(const embed_server_config_t *cfg) { - for (int i = 0; i < cfg->n_models; i++) { - const embed_server_model_spec_t *spec = &cfg->models[i]; - if (!spec->id || !spec->id[0]) { - fprintf(stderr, "embed-server: model label must not be empty\n"); - return -1; - } - if (!spec->path || !spec->path[0]) { - fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id); - return -1; - } - if (spec->kind < EMBED_SERVER_MODEL_STANDARD || spec->kind > EMBED_SERVER_MODEL_LATE) { - fprintf(stderr, "embed-server: model %s has an invalid kind\n", spec->id); - return -1; - } - if (spec->api < EMBED_SERVER_API_DEFAULT || spec->api > EMBED_SERVER_API_PERPLEXITY) { - fprintf(stderr, "embed-server: model %s has an invalid api\n", spec->id); - return -1; - } - if (spec->min_dim < 0) { - fprintf(stderr, "embed-server: model %s has an invalid min_dim\n", spec->id); - return -1; - } - for (int j = 0; j < i; j++) { - if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) { - fprintf(stderr, "embed-server: duplicate model label: %s\n", spec->id); - return -1; - } - } - } - return 0; -} - -static int mlx_memory_preflight(const embed_server_config_t *cfg) { - if (!cfg->use_mlx) - return 0; - - uint64_t payload = 0; - for (int i = 0; i < cfg->n_models; i++) { - multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path); - size_t model_payload = 0; - if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) { - multi_safetensors_close(ms); - fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path); - return -1; - } - multi_safetensors_close(ms); - if ((uint64_t)model_payload > UINT64_MAX - payload) { - fprintf(stderr, "embed-server: MLX model payload size overflow\n"); - return -1; - } - payload += (uint64_t)model_payload; - } - - if (payload > UINT64_MAX / EMBED_MLX_RESIDENT_MULTIPLIER) { - fprintf(stderr, "embed-server: MLX resident-memory estimate overflow\n"); - return -1; - } - uint64_t estimated = payload * EMBED_MLX_RESIDENT_MULTIPLIER; - uint64_t physical = physical_memory_nbytes(); - if (physical == 0) { - server_log("embed-server: warning: could not determine physical memory; " - "skipping MLX memory preflight"); - return 0; - } - double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization - : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0; - uint64_t budget = (uint64_t)((double)physical * utilization); - const double gib = 1024.0 * 1024.0 * 1024.0; - server_log("embed-server: MLX memory preflight: %.1f GiB tensors, " - "%.1f GiB estimated resident, %.1f GiB budget " - "(%.2f of physical memory)", - (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization); - if (cfg->mlx_quantize_bits) { - server_log("embed-server: MLX %d-bit quantization enabled; preflight " - "uses source payload as a conservative peak estimate", - cfg->mlx_quantize_bits); - } - if (estimated <= budget) - return 0; - server_log("embed-server: refusing MLX model set above the host-memory " - "budget"); - server_log("embed-server: use BF16 artifacts, load fewer models, or raise " - "--memory-utilization (values above 1.0 overcommit)"); - return -1; -} - -int embed_run_server(const embed_server_config_t *cfg) { - if (!cfg || !cfg->models || cfg->n_models <= 0) { - fprintf(stderr, "embed-server: at least one model label and path is required\n"); - return 1; - } - if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0) - return 1; - - http_server s; - memset(&s, 0, sizeof(s)); - s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; - s.port = cfg->port > 0 ? cfg->port : 8000; - s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE; - s.max_batch_tokens = - cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS; - s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us - : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US - : EMBED_SERVER_BATCH_WAIT_US); - s.use_mlx = cfg->use_mlx; - s.use_cuda = cfg->use_cuda; - s.mlx_quantize_bits = cfg->mlx_quantize_bits; - s.mlx_quantize_group_size = - cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64; - if (cfg->api_key && cfg->api_key[0]) - s.api_key = xstrdup(cfg->api_key); - else { - const char *env_key = getenv("EMBED_API_KEY"); - if (env_key && env_key[0]) - s.api_key = xstrdup(env_key); - } - - pthread_mutex_init(&s.mu, NULL); - pthread_cond_init(&s.cv, NULL); - - for (int i = 0; i < cfg->n_models; i++) { - if (load_one_model(&s, &cfg->models[i]) != 0) { - free_models(&s); - return 1; - } - } - - if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { - perror("pipe"); - free_models(&s); - return 1; - } - set_nonblock(s.completion_pipe[0]); - set_nonblock(s.completion_pipe[1]); - set_nonblock(s.signal_pipe[0]); - set_nonblock(s.signal_pipe[1]); - - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = stop_signal_handler; - sigemptyset(&sa.sa_mask); - sigaction(SIGINT, &sa, NULL); - sigaction(SIGTERM, &sa, NULL); - signal(SIGPIPE, SIG_IGN); - g_signal_wfd = s.signal_pipe[1]; - - s.loop = aeCreateEventLoop(EMBED_HTTP_SETSIZE); - if (!s.loop) { - fprintf(stderr, "embed-server: failed to create event loop\n"); - free_models(&s); - return 1; - } - - if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { - fprintf(stderr, "embed-server: failed to start worker\n"); - aeDeleteEventLoop(s.loop); - free_models(&s); - return 1; - } - pthread_mutex_lock(&s.mu); - while (!s.worker_ready) - pthread_cond_wait(&s.cv, &s.mu); - int worker_init_rc = s.worker_init_rc; - pthread_mutex_unlock(&s.mu); - if (worker_init_rc) { - pthread_join(s.worker, NULL); - aeDeleteEventLoop(s.loop); - free_models(&s); - free(s.api_key); - pthread_cond_destroy(&s.cv); - pthread_mutex_destroy(&s.mu); - return 1; - } - - s.listen_fd = listen_on(s.host, s.port); - if (s.listen_fd < 0) { - pthread_mutex_lock(&s.mu); - s.stopping = 1; - pthread_cond_signal(&s.cv); - pthread_mutex_unlock(&s.mu); - pthread_join(s.worker, NULL); - aeDeleteEventLoop(s.loop); - free_models(&s); - return 1; - } - - aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); - aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); - aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); - aeCreateTimeEvent(s.loop, EMBED_HTTP_SWEEP_MS, timeout_cb, &s, NULL); - - server_log("embed-server: listening on http://%s:%d", s.host, s.port); - aeMain(s.loop); - - server_log("embed-server: shutdown requested"); - if (s.listen_fd >= 0) - close(s.listen_fd); - client *c = s.clients; - while (c) { - client *next = c->next; - close_client(c); - c = next; - } - - pthread_mutex_lock(&s.mu); - s.stopping = 1; - pthread_cond_signal(&s.cv); - pthread_mutex_unlock(&s.mu); - pthread_join(s.worker, NULL); - - aeDeleteEventLoop(s.loop); - close(s.completion_pipe[0]); - close(s.completion_pipe[1]); - close(s.signal_pipe[0]); - close(s.signal_pipe[1]); - free_models(&s); - free(s.api_key); - pthread_cond_destroy(&s.cv); - pthread_mutex_destroy(&s.mu); - return 0; -} - -/* ==================================================================== - * embed-server entry point - * ==================================================================== */ -#ifndef EMBED_SERVER_TEST - -static void print_usage(const char *prog) { - fprintf(stderr, - "Usage: %s --model LABEL=DIR [options]\n" - "\n" - "Options:\n" - " --model LABEL=DIR[:key=val]\n" - " Standard embedding model (repeatable)\n" - " --contextual-model LABEL=DIR[:key=val]\n" - " Contextual embedding model (repeatable)\n" - " --late-model LABEL=DIR[:key=val]\n" - " Late-interaction rerank model (repeatable)\n" - " keys: api=openai|perplexity, min_dim=N,\n" - " query=TEXT (last key; \\n escapes newline)\n" -#ifdef USE_MLX - " --mlx Use Apple MLX GPU backend\n" -#endif -#ifdef USE_CUDA - " --cuda Use NVIDIA CUDA GPU backend\n" -#endif - " --host HOST Bind host (default: 127.0.0.1)\n" - " --port N Bind port (default: 8000)\n" - " --api-key K Require Authorization: Bearer K\n" - " -b, --batch-size N Max texts or documents per inference batch\n" - " (default: 32)\n" - " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" - " --batch-wait-us N First-arrival micro-batch deadline in us\n" -#ifdef USE_CUDA - " (default: CUDA 1000; CPU 0)\n" -#else - " (default: 0)\n" -#endif -#ifdef USE_MLX - " --memory-utilization F Fraction of physical memory the MLX model-set\n" - " preflight may plan for (default: 0.90; values\n" - " above 1.0 overcommit)\n" - " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n" - " --mlx-quant-group-size N MLX quantization group size (default: 64)\n" -#endif -#ifdef USE_CUDA - " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n" - " (default: f32, exact)\n" - " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n" - " f32); bf16 halves weight memory\n" -#endif - " -t, --threads N CPU threads (default: available cores, cgroup-aware)\n" - " -v, --verbose Verbose (-vv for debug)\n" - " -V, --version Print version and exit\n" - " --build-info Print build details and exit\n" - " -h, --help Show this help\n" - "\n" - "Examples:\n" - " %s --model docs=./model --port 8000\n" - " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n" -#ifdef USE_MLX - " %s --mlx --mlx-quant-bits 8 \\\n" - " --model pplx=./model-4b-bf16:api=perplexity:min_dim=128\n", - prog, prog, prog, prog); -#else - , - prog, prog, prog); -#endif -} - -typedef struct { - embed_server_model_spec_t *v; - int n; - int cap; -} model_specs_t; - -static char *dup_range(const char *p, size_t n) { - char *s = (char *)malloc(n + 1); - if (!s) - return NULL; - memcpy(s, p, n); - s[n] = '\0'; - return s; -} - -static char *unescape_model_option(const char *s) { - size_t n = strlen(s); - char *out = (char *)malloc(n + 1); - if (!out) - return NULL; - size_t w = 0; - for (size_t r = 0; r < n; r++) { - if (s[r] == '\\' && r + 1 < n) { - r++; - if (s[r] == 'n') - out[w++] = '\n'; - else if (s[r] == 't') - out[w++] = '\t'; - else if (s[r] == 'r') - out[w++] = '\r'; - else - out[w++] = s[r]; - } else { - out[w++] = s[r]; - } - } - out[w] = '\0'; - return out; -} - -static int parse_model_option(embed_server_model_spec_t *spec, const char *opt, const char *flag) { - const char *eq = strchr(opt, '='); - if (!eq || eq == opt || !eq[1]) { - fprintf(stderr, "%s option expects key=value: %s\n", flag, opt); - return -1; - } - size_t key_len = (size_t)(eq - opt); - const char *val = eq + 1; - if (key_len == 3 && strncmp(opt, "api", key_len) == 0) { - if (!strcmp(val, "openai")) { - spec->api = EMBED_SERVER_API_OPENAI; - } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) { - spec->api = EMBED_SERVER_API_PERPLEXITY; - } else if (!strcmp(val, "default")) { - spec->api = EMBED_SERVER_API_DEFAULT; - } else { - fprintf(stderr, "%s api must be openai or perplexity\n", flag); - return -1; - } - return 0; - } - if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) { - char *end = NULL; - long v = strtol(val, &end, 10); - if (!end || *end || v <= 0 || v > INT_MAX) { - fprintf(stderr, "%s min_dim must be a positive integer\n", flag); - return -1; - } - spec->min_dim = (int)v; - return 0; - } - fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt); - return -1; -} - -static int append_model_spec(model_specs_t *specs, - const char *arg, - embed_server_model_kind_t kind, - const char *flag) { - const char *eq = strchr(arg, '='); - if (!eq || eq == arg || !eq[1]) { - fprintf(stderr, "%s expects LABEL=DIR\n", flag); - return -1; - } - if (specs->n == specs->cap) { - int new_cap = specs->cap ? specs->cap * 2 : 4; - void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); - if (!p) - return -1; - specs->v = p; - specs->cap = new_cap; - } - size_t id_len = (size_t)(eq - arg); - const char *path_start = eq + 1; - const char *opts = strchr(path_start, ':'); - size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start); - if (path_len == 0) { - fprintf(stderr, "%s expects a non-empty DIR\n", flag); - return -1; - } - embed_server_model_spec_t spec; - memset(&spec, 0, sizeof(spec)); - spec.id = dup_range(arg, id_len); - spec.path = dup_range(path_start, path_len); - spec.kind = kind; - if (!spec.id || !spec.path) { - free((char *)spec.id); - free((char *)spec.path); - return -1; - } - const char *p = opts ? opts + 1 : NULL; - while (p && *p) { - if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) { - const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15); - spec.query_instruct = unescape_model_option(val); - if (!spec.query_instruct) - goto fail; - break; - } - const char *next = strchr(p, ':'); - size_t opt_len = next ? (size_t)(next - p) : strlen(p); - char *opt = dup_range(p, opt_len); - if (!opt) - goto fail; - int rc = parse_model_option(&spec, opt, flag); - free(opt); - if (rc != 0) - goto fail; - p = next ? next + 1 : NULL; - } - specs->v[specs->n] = spec; - specs->n++; - return 0; - -fail: - free((char *)spec.id); - free((char *)spec.path); - free((char *)spec.query_instruct); - return -1; -} - -static void free_model_specs(model_specs_t *specs) { - if (!specs) - return; - for (int i = 0; i < specs->n; i++) { - free((char *)specs->v[i].id); - free((char *)specs->v[i].path); - free((char *)specs->v[i].query_instruct); - } - free(specs->v); -} - -int main(int argc, char *argv[]) { - int n_threads = 0; - int verbose = 0; - int batch_size = 0; - int max_batch_tokens = 0; - int batch_wait_us = -1; -#ifdef USE_MLX - int use_mlx = 0; - int mlx_quantize_bits = 0; - int mlx_quantize_group_size = 64; - double memory_utilization = 0.0; -#endif -#ifdef USE_CUDA - int use_cuda = 0; - const char *cuda_fast_gemm = NULL; - const char *cuda_weights = NULL; -#endif - const char *host = "127.0.0.1"; - const char *api_key = NULL; - int port = 8000; - model_specs_t model_specs = {0}; - - const char *prog = embed_prog_name(argv[0]); - int arg = 1; - while (arg < argc && argv[arg][0] == '-') { - const char *f = argv[arg]; - if (!strcmp(f, "-V") || !strcmp(f, "--version")) { - embed_print_version(prog); - free_model_specs(&model_specs); - return 0; - } else if (!strcmp(f, "--build-info")) { - embed_print_build_info(prog); - free_model_specs(&model_specs); - return 0; - } else if (!strcmp(f, "--model")) { - if (arg + 1 >= argc || - append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_STANDARD, f) != - 0) { - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--contextual-model")) { - if (arg + 1 >= argc || - append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_CONTEXTUAL, f) != - 0) { - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--late-model")) { - if (arg + 1 >= argc || - append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_LATE, f) != 0) { - free_model_specs(&model_specs); - return 1; - } - } -#ifdef USE_MLX - else if (!strcmp(f, "--mlx")) { - use_mlx = 1; - } else if (!strcmp(f, "--mlx-quant-bits")) { - mlx_quantize_bits = atoi(argv[++arg]); - if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) { - fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--mlx-quant-group-size")) { - mlx_quantize_group_size = atoi(argv[++arg]); - if (mlx_quantize_group_size <= 0) { - fprintf(stderr, "--mlx-quant-group-size must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--memory-utilization")) { - memory_utilization = atof(argv[++arg]); - if (memory_utilization <= 0.0) { - fprintf(stderr, "--memory-utilization must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } -#endif -#ifdef USE_CUDA - else if (!strcmp(f, "--cuda")) { - use_cuda = 1; - } else if (!strcmp(f, "--cuda-gemm-mode")) { - cuda_fast_gemm = argv[++arg]; - } else if (!strcmp(f, "--cuda-weight-dtype")) { - cuda_weights = argv[++arg]; - } -#endif - else if (!strcmp(f, "--host")) { - host = argv[++arg]; - } else if (!strcmp(f, "--port")) { - port = atoi(argv[++arg]); - } else if (!strcmp(f, "--api-key")) { - api_key = argv[++arg]; - } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { - batch_size = atoi(argv[++arg]); - } else if (!strcmp(f, "--max-batch-tokens")) { - max_batch_tokens = atoi(argv[++arg]); - if (max_batch_tokens <= 0) { - fprintf(stderr, "--max-batch-tokens must be > 0\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "--batch-wait-us")) { - batch_wait_us = atoi(argv[++arg]); - if (batch_wait_us < 0) { - fprintf(stderr, "--batch-wait-us must be >= 0\n"); - free_model_specs(&model_specs); - return 1; - } - } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { - n_threads = atoi(argv[++arg]); - } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { - verbose++; - } else if (!strcmp(f, "-vv")) { - verbose = 2; - } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { - print_usage(prog); - free_model_specs(&model_specs); - return 0; - } else { - fprintf(stderr, "unknown option: %s\n", f); - print_usage(prog); - free_model_specs(&model_specs); - return 1; - } - arg++; - } - - embed_verbose = verbose; - - if (arg < argc) { - fprintf(stderr, "unexpected argument: %s\n", argv[arg]); - print_usage(prog); - free_model_specs(&model_specs); - return 1; - } - if (model_specs.n == 0) { - fprintf(stderr, "at least one --model LABEL=DIR is required\n"); - print_usage(prog); - free_model_specs(&model_specs); - return 1; - } -#ifdef USE_MLX - if (mlx_quantize_bits && !use_mlx) { - fprintf(stderr, "--mlx-quant-bits requires --mlx\n"); - free_model_specs(&model_specs); - return 1; - } -#endif -#ifdef USE_CUDA - if (cuda_fast_gemm) { - if (!use_cuda) { - fprintf(stderr, "--cuda-gemm-mode requires --cuda\n"); - free_model_specs(&model_specs); - return 1; - } - if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) { - fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n"); - free_model_specs(&model_specs); - return 1; - } - } - if (cuda_weights) { - if (!use_cuda) { - fprintf(stderr, "--cuda-weight-dtype requires --cuda\n"); - free_model_specs(&model_specs); - return 1; - } - if (!strcmp(cuda_weights, "bf16")) { - embed_cuda_set_weights_bf16(1); - } else if (!strcmp(cuda_weights, "f32")) { - embed_cuda_set_weights_bf16(0); - } else { - fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n"); - free_model_specs(&model_specs); - return 1; - } - } -#endif - -#ifdef USE_MLX - if (use_mlx) { - if (verbose >= 1) - fprintf(stderr, "Using MLX GPU backend\n"); - } else -#endif -#ifdef USE_CUDA - if (use_cuda) { - if (verbose >= 1) - fprintf(stderr, "Using CUDA GPU backend\n"); - } else -#endif - { - if (n_threads <= 0) - n_threads = embed_get_num_cpus(); - embed_set_threads(n_threads); - if (verbose >= 1) - fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); - } - - embed_server_config_t scfg = { - .models = model_specs.v, - .n_models = model_specs.n, - .host = host, - .port = port, - .batch_size = batch_size > 0 ? batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE, - .max_batch_tokens = max_batch_tokens, - .batch_wait_us = batch_wait_us, -#ifdef USE_MLX - .use_mlx = use_mlx, - .mlx_quantize_bits = mlx_quantize_bits, - .mlx_quantize_group_size = mlx_quantize_group_size, - .memory_utilization = memory_utilization, -#endif -#ifdef USE_CUDA - .use_cuda = use_cuda, -#endif - .api_key = api_key, - }; - int rc = embed_run_server(&scfg); - free_model_specs(&model_specs); - return rc; -} - -#endif /* EMBED_SERVER_TEST */ diff --git a/src/server.h b/src/server.h deleted file mode 100644 index 5ad2b12..0000000 --- a/src/server.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef EMBED_SERVER_H -#define EMBED_SERVER_H - -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying EMBED_API are exported - * from the shared library; everything else stays internal. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - -#include <stdbool.h> - -typedef enum { - EMBED_SERVER_MODEL_STANDARD = 0, - EMBED_SERVER_MODEL_CONTEXTUAL = 1, - EMBED_SERVER_MODEL_LATE = 2, -} embed_server_model_kind_t; - -typedef enum { - EMBED_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */ - EMBED_SERVER_API_OPENAI = 1, - EMBED_SERVER_API_PERPLEXITY = 2, -} embed_server_embedding_api_t; - -typedef struct { - const char *id; /* served label; requests match this string exactly */ - const char *path; - embed_server_model_kind_t kind; - embed_server_embedding_api_t api; - int min_dim; /* 0 means the loaded model's full hidden size */ - const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */ -} embed_server_model_spec_t; - -typedef struct { - const embed_server_model_spec_t *models; - int n_models; - const char *host; - int port; - int batch_size; - int max_batch_tokens; - int batch_wait_us; - int use_mlx; - int use_cuda; - int mlx_quantize_bits; - int mlx_quantize_group_size; - double memory_utilization; /* fraction of physical memory the MLX - preflight may plan for; 0 = default 0.90, - values above 1.0 allow overcommit */ - const char *api_key; -} embed_server_config_t; - -EMBED_API int embed_run_server(const embed_server_config_t *cfg); - -#endif diff --git a/src/server_models.c b/src/server_models.c deleted file mode 100644 index 320840b..0000000 --- a/src/server_models.c +++ /dev/null @@ -1,627 +0,0 @@ -#include "server_internal.h" -#include "server_util.h" - -#include <errno.h> -#include <limits.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -/* ======================================================================== - * Model load/lifecycle, and tokenize/inference dispatch - * - * Each loaded model owns resolved serving metadata for the operator-chosen - * label. Transformer behavior comes from the model files through embed_config; - * server-only facts absent from those files, such as output API family and - * Matryoshka floor, come from the load spec. The tokenize_* and model_embed_* - * helpers are the single dispatch point the request handlers call, hiding the - * CPU/MLX/CUDA backend choice behind one signature. - * ======================================================================== */ - -loaded_model *loaded_model_for_label(http_server *s, const char *label) { - if (!label) - return NULL; - for (int i = 0; i < s->n_models; i++) { - if (s->models[i].info && !strcmp(label, s->models[i].info->id)) - return &s->models[i]; - } - return NULL; -} - -static model_kind model_kind_from_public(embed_server_model_kind_t kind) { - switch (kind) { - case EMBED_SERVER_MODEL_CONTEXTUAL: - return MODEL_KIND_CONTEXTUAL; - case EMBED_SERVER_MODEL_LATE: - return MODEL_KIND_LATE; - case EMBED_SERVER_MODEL_STANDARD: - default: - return MODEL_KIND_STANDARD; - } -} - -static embedding_api_t embedding_api_from_public(embed_server_embedding_api_t api) { - return api == EMBED_SERVER_API_PERPLEXITY ? EMBED_API_PERPLEXITY : EMBED_API_OPENAI; -} - -static int read_optional_text_file(const char *model_dir, const char *relative, char **out) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative); - if (n < 0 || (size_t)n >= sizeof(path)) { - server_log("embed-server: model path too long"); - return -1; - } - FILE *f = fopen(path, "rb"); - if (!f && errno == ENOENT) - return 0; - if (!f) { - server_log("embed-server: failed to open %s", path); - return -1; - } - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return -1; - } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - char *buf = (char *)malloc((size_t)sz + 1); - if (!buf) { - fclose(f); - return -1; - } - if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { - free(buf); - fclose(f); - return -1; - } - fclose(f); - buf[sz] = '\0'; - *out = buf; - return 1; -} - -static int read_query_instruct_from_files(const char *model_dir, char **out) { - char *buf = NULL; - int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf); - if (rc <= 0) - return rc; - cJSON *root = cJSON_Parse(buf); - free(buf); - if (!root) { - server_log("embed-server: invalid config_sentence_transformers.json: %s", model_dir); - return -1; - } - cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts"); - cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL; - if (query && !cJSON_IsString(query)) { - cJSON_Delete(root); - server_log("embed-server: prompts.query must be a string: %s", model_dir); - return -1; - } - if (query && query->valuestring && query->valuestring[0]) { - *out = xstrdup(query->valuestring); - cJSON_Delete(root); - return *out ? 1 : -1; - } - cJSON_Delete(root); - return 0; -} - -static int init_model_info(model_info **out, const embed_server_model_spec_t *spec) { - model_info *info = xcalloc(1, sizeof(*info)); - info->id = xstrdup(spec->id); - info->kind = model_kind_from_public(spec->kind); - info->api = embedding_api_from_public(spec->api); - info->min_dim = spec->min_dim; - if (spec->query_instruct) { - info->query_instruct = xstrdup(spec->query_instruct); - } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) { - free(info->id); - free(info); - return -1; - } - if (!info->id || (spec->query_instruct && !info->query_instruct)) { - free(info->query_instruct); - free(info->id); - free(info); - return -1; - } - *out = info; - return 0; -} - -static void free_model_info(model_info *info) { - if (!info) - return; - free(info->id); - free(info->query_instruct); - free(info); -} - -void free_token_bufs(token_buf *t, int n) { - if (!t) - return; - for (int i = 0; i < n; i++) - free(t[i].ids); -} - -void embedding_request_free(embedding_request *r) { - if (!r) - return; - free_token_bufs(r->tokens, r->n_inputs); - free(r->tokens); - free(r->inputs); - if (r->root) - cJSON_Delete(r->root); - memset(r, 0, sizeof(*r)); -} - -void contextual_request_free(contextual_request *r) { - if (!r) - return; - for (int i = 0; i < r->n_docs; i++) { - free(r->docs[i].ids); - free(r->docs[i].spans); - } - free(r->docs); - if (r->root) - cJSON_Delete(r->root); - memset(r, 0, sizeof(*r)); -} - -void late_tokens_free(late_tokens *t) { - if (!t) - return; - free(t->ids); - free(t->keep); - memset(t, 0, sizeof(*t)); -} - -void rerank_request_free(rerank_request *r) { - if (!r) - return; - late_tokens_free(&r->query); - for (int i = 0; i < r->n_documents; i++) - late_tokens_free(&r->documents[i]); - free(r->documents); - if (r->root) - cJSON_Delete(r->root); - memset(r, 0, sizeof(*r)); -} - -int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) { - memset(out, 0, sizeof(*out)); - uint64_t t0 = nstime(); - if (m->wp_tok) { - int n = 0; - int *core = wordpiece_tokenizer_encode_with_workspace(m->wp_tok, m->wp_tok_ws, text, &n); - if (j) - j->tokenize_ns += nstime() - t0; - /* BERT wraps the WordPiece ids with [CLS] ... [SEP], exactly as the BPE - * branch below appends its terminal token. A whitespace-only input - * cleans to zero core tokens (core == NULL, n == 0); the bare - * [CLS][SEP] pair is what Hugging Face produces for it too. Grow the - * returned buffer in place so only one allocation is touched. */ - if (n < 0 || n > INT_MAX - 2) { - free(core); - memset(out, 0, sizeof(*out)); - return -1; - } - int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - memset(out, 0, sizeof(*out)); - return -1; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = m->cls_id; - ids[n + 1] = m->sep_id; - out->ids = ids; - out->n_tokens = n + 2; - return 0; - } - if (m->sp_tok) { - int n = 0; - int *core = - sentencepiece_tokenizer_encode_with_workspace(m->sp_tok, m->sp_tok_ws, text, &n); - if (j) - j->tokenize_ns += nstime() - t0; - if (n < 0 || n > INT_MAX - 2) { - free(core); - memset(out, 0, sizeof(*out)); - return -1; - } - int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); - if (!ids) { - free(core); - memset(out, 0, sizeof(*out)); - return -1; - } - memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = m->cls_id; - ids[n + 1] = m->sep_id; - out->ids = ids; - out->n_tokens = n + 2; - return 0; - } - out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens); - if (j) - j->tokenize_ns += nstime() - t0; - if (!out->ids || out->n_tokens <= 0) { - free(out->ids); - memset(out, 0, sizeof(*out)); - return -1; - } - if (m->append_terminal_token) { - if (out->n_tokens == INT_MAX) { - free(out->ids); - memset(out, 0, sizeof(*out)); - return -1; - } - int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids)); - if (!ids) { - free(out->ids); - memset(out, 0, sizeof(*out)); - return -1; - } - out->ids = ids; - out->ids[out->n_tokens++] = m->terminal_token_id; - } - return 0; -} - -/* Tokenize one embedding input, optionally prepending the model's query - * instruction. The instruction and input are tokenized in one pass. */ -int tokenize_input( - loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) { - if (!query_instruct) - return tokenize_one(m, j, text, out); - size_t plen = strlen(query_instruct); - size_t tlen = strlen(text); - char *buf = (char *)malloc(plen + tlen + 1); - if (!buf) { - memset(out, 0, sizeof(*out)); - return -1; - } - memcpy(buf, query_instruct, plen); - memcpy(buf + plen, text, tlen + 1); - int rc = tokenize_one(m, j, buf, out); - free(buf); - return rc; -} - -static int late_id_is_skipped(const loaded_model *m, int id) { - for (int i = 0; i < m->n_late_skip_ids; i++) { - if (m->late_skip_ids[i] == id) - return 1; - } - return 0; -} - -int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) { - token_buf raw = {0}; - if (tokenize_one(m, j, text, &raw) != 0) - return -1; - - int raw_tokens = raw.n_tokens; - int target = raw_tokens + 1; - if (is_query && target < EMBED_LATE_QUERY_TOKENS) - target = EMBED_LATE_QUERY_TOKENS; - - out->ids = xmalloc((size_t)target * sizeof(*out->ids)); - out->ids[0] = raw.ids[0]; - out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id; - if (raw_tokens > 1) { - memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids)); - } - out->n_tokens = raw_tokens + 1; - if (is_query) { - while (out->n_tokens < EMBED_LATE_QUERY_TOKENS) - out->ids[out->n_tokens++] = m->late_mask_id; - out->n_keep = out->n_tokens; - } else { - out->keep = xmalloc((size_t)out->n_tokens * sizeof(*out->keep)); - for (int i = 0; i < out->n_tokens; i++) { - if (!late_id_is_skipped(m, out->ids[i])) - out->keep[out->n_keep++] = i; - } - } - free(raw.ids); - return out->n_keep > 0 ? 0 : -1; -} - -int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) { -#ifdef USE_MLX - if (m->mlx_ctx) - return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out); -#endif -#ifdef USE_CUDA - if (m->cuda_ctx) - return embed_cuda_encode_batch(m->cuda_ctx, inputs, batch, out); -#endif - return embed_model_encode_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); -} - -int model_embed_spans_batch(loaded_model *m, - const embed_context_input_t *inputs, - int batch, - float *out) { -#ifdef USE_MLX - if (m->mlx_ctx) - return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out); -#endif -#ifdef USE_CUDA - if (m->cuda_ctx) - return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out); -#endif - return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out); -} - -static int model_uses_dense_batches(const loaded_model *m) { -#ifdef USE_MLX - return m->mlx_ctx != NULL; -#else - (void)m; - return 0; -#endif -} - -/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX - * pads each dense row to the longest input in the chunk. */ -int inference_batch_accepts_input( - const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) { - if (batch == 0) - return 1; - if (model_uses_dense_batches(m)) - return next_tokens <= max_batch_tokens / (batch + 1); - return next_tokens <= max_batch_tokens - packed_tokens; -} - -int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) { - if (!m || !m->info || !config || config->hidden_size <= 0) { - if (m && m->info) - server_log("embed-server: model %s has invalid config", m->info->id); - return -1; - } - if (m->info->kind == MODEL_KIND_LATE) { - if (token_dim <= 0) { - server_log("embed-server: late model %s is missing token projection metadata", - m->info->id); - return -1; - } - } else if (token_dim != 0) { - server_log("embed-server: non-late model %s unexpectedly has token projection metadata", - m->info->id); - return -1; - } - - int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size; - if (min_dim <= 0 || min_dim > config->hidden_size) { - server_log("embed-server: model %s has invalid min_dim=%d for dim=%d", m->info->id, - min_dim, config->hidden_size); - return -1; - } - m->info->dim = config->hidden_size; - m->info->min_dim = min_dim; - m->info->token_dim = token_dim; - m->info->attention_mode = config->attention_mode; - m->info->pooling_mode = config->pooling_mode; - m->info->normalize_embeddings = config->normalize_embeddings; - m->append_terminal_token = config->append_terminal_token; - /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */ - m->renormalize_truncated = - config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN; - return 0; -} - -int load_one_model(http_server *s, const embed_server_model_spec_t *spec) { - /* Grow the model list by one entry. */ - loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models)); - if (!nm) - return -1; - s->models = nm; - loaded_model *m = &s->models[s->n_models]; - memset(m, 0, sizeof(*m)); - s->n_models++; - - if (init_model_info(&m->info, spec) != 0) - return -1; - const char *path = spec->path; - m->path = xstrdup(path); - m->terminal_token_id = -1; - m->cls_id = -1; - m->sep_id = -1; - /* Pick the tokenizer by the files present: WordPiece vocab.txt, then a - * SentencePiece .model, otherwise the Qwen byte-level BPE vocab.json. */ - char wp_path[1024]; - snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path); - if (access(wp_path, R_OK) == 0) { - m->wp_tok = wordpiece_tokenizer_load(path); - if (!m->wp_tok) { - server_log("embed-server: failed to load WordPiece tokenizer: %s", wp_path); - return -1; - } - m->wp_tok_ws = wordpiece_workspace_new(); - if (!m->wp_tok_ws) { - server_log("embed-server: failed to allocate WordPiece workspace: %s", path); - return -1; - } - /* [CLS]/[SEP] wrap every input; resolve their ids once from the vocab. */ - m->cls_id = wordpiece_tokenizer_token_id(m->wp_tok, "[CLS]"); - m->sep_id = wordpiece_tokenizer_token_id(m->wp_tok, "[SEP]"); - if (m->cls_id < 0 || m->sep_id < 0) { - server_log("embed-server: WordPiece vocab missing [CLS]/[SEP]: %s", path); - return -1; - } - } else { - char sp_path[1024]; - snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", path); - if (access(sp_path, R_OK) != 0) { - snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", path); - if (access(sp_path, R_OK) != 0) - snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", path); - } - if (access(sp_path, R_OK) == 0) { - m->sp_tok = sentencepiece_tokenizer_load(path); - if (!m->sp_tok) { - server_log("embed-server: failed to load SentencePiece tokenizer: %s", sp_path); - return -1; - } - m->sp_tok_ws = sentencepiece_workspace_new(); - if (!m->sp_tok_ws) { - server_log("embed-server: failed to allocate SentencePiece workspace: %s", path); - return -1; - } - m->cls_id = sentencepiece_tokenizer_token_id(m->sp_tok, "<s>"); - m->sep_id = sentencepiece_tokenizer_token_id(m->sp_tok, "</s>"); - if (m->cls_id < 0 || m->sep_id < 0) { - server_log("embed-server: SentencePiece vocab missing <s>/</s>: %s", path); - return -1; - } - } - } - if (!m->wp_tok && !m->sp_tok) { - char vocab_path[1024]; - snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); - m->tok = embed_tokenizer_load(vocab_path); - if (!m->tok) { - server_log("embed-server: failed to load tokenizer: %s", vocab_path); - return -1; - } - m->tok_ws = embed_tokenizer_workspace_new(); - if (!m->tok_ws) { - server_log("embed-server: failed to allocate tokenizer workspace: %s", path); - return -1; - } - /* The contextual chunk separator is the tokenizer's <|endoftext|>. The - * released models keep added special tokens out of vocab.json, so the - * family constant is the normal case; a vocab that does define the - * token (e.g. small test models) takes precedence so the id stays - * inside that model's embedding table. */ - int sep_id = embed_tokenizer_token_id(m->tok, "<|endoftext|>"); - m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; - /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|> - * suffix - the same token as the separator, not the model's chat - * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */ - m->terminal_token_id = m->context_separator_id; - if (m->info->kind == MODEL_KIND_LATE) { - int id = embed_tokenizer_token_id(m->tok, "[MASK]"); - m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID; - id = embed_tokenizer_token_id(m->tok, "[Q]"); - m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID; - id = embed_tokenizer_token_id(m->tok, "[D]"); - m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID; - - const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"; - for (const char *p = punct; - *p && - m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / sizeof(m->late_skip_ids[0])); - p++) { - char text[2] = {*p, '\0'}; - int n_ids = 0; - int *ids = embed_tokenizer_encode(m->tok, text, &n_ids); - if (ids && n_ids == 1) - m->late_skip_ids[m->n_late_skip_ids++] = ids[0]; - free(ids); - } - } - } -#ifdef USE_MLX - if (s->use_mlx) { - /* MLX streams are thread-local. The inference worker loads the MLX - * model so all MLX arrays/streams are created, used, and freed by the - * same thread. */ - return 0; - } -#endif -#ifdef USE_CUDA - if (s->use_cuda) { - return 0; - } -#endif - const embed_config_t *config = NULL; - int token_dim = 0; - if (m->info->kind == MODEL_KIND_LATE) { - m->cpu_late_model = embed_late_model_load(path); - if (!m->cpu_late_model) { - server_log("embed-server: failed to load late model: %s", path); - return -1; - } - m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model); - if (!m->cpu_late_ws) { - server_log("embed-server: failed to allocate late workspace: %s", path); - return -1; - } - config = embed_late_model_config(m->cpu_late_model); - token_dim = embed_late_model_token_dim(m->cpu_late_model); - } else { - m->cpu_model = embed_model_load(path); - if (!m->cpu_model) { - server_log("embed-server: failed to load model: %s", path); - return -1; - } - m->cpu_ws = embed_workspace_new(m->cpu_model); - if (!m->cpu_ws) { - server_log("embed-server: failed to allocate workspace: %s", path); - return -1; - } - config = embed_model_config(m->cpu_model); - } - if (configure_loaded_model(m, config, token_dim) != 0) { - return -1; - } - if (m->info->kind == MODEL_KIND_LATE && - (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size || - m->late_document_prefix_id >= config->vocab_size)) { - server_log("embed-server: late special-token ids exceed model vocab"); - return -1; - } - return 0; -} - -void free_models(http_server *s) { - for (int i = 0; i < s->n_models; i++) { - loaded_model *m = &s->models[i]; - free_model_info(m->info); - free(m->path); - if (m->tok_ws) - embed_tokenizer_workspace_free(m->tok_ws); - if (m->tok) - embed_tokenizer_free(m->tok); - if (m->wp_tok_ws) - wordpiece_workspace_free(m->wp_tok_ws); - if (m->wp_tok) - wordpiece_tokenizer_free(m->wp_tok); - if (m->sp_tok_ws) - sentencepiece_workspace_free(m->sp_tok_ws); - if (m->sp_tok) - sentencepiece_tokenizer_free(m->sp_tok); - if (m->cpu_ws) - embed_workspace_free(m->cpu_ws); - if (m->cpu_model) - embed_model_free(m->cpu_model); - if (m->cpu_late_ws) - embed_late_workspace_free(m->cpu_late_ws); - if (m->cpu_late_model) - embed_late_model_free(m->cpu_late_model); -#ifdef USE_MLX - if (m->mlx_ctx) - embed_mlx_free(m->mlx_ctx); - if (m->mlx_late_ctx) - embed_mlx_late_free(m->mlx_late_ctx); -#endif -#ifdef USE_CUDA - if (m->cuda_ctx) - embed_cuda_free(m->cuda_ctx); - if (m->cuda_late_ctx) - embed_cuda_late_free(m->cuda_late_ctx); -#endif - } - free(s->models); - s->models = NULL; - s->n_models = 0; -} diff --git a/src/tokenizer_bpe.h b/src/tokenizer_bpe.h deleted file mode 100644 index 8d13d47..0000000 --- a/src/tokenizer_bpe.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * embed_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) - * - * Supports: - * - decode token ID -> UTF-8 text - * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) - */ - -#ifndef EMBED_TOKENIZER_H -#define EMBED_TOKENIZER_H - -/* Public-API export annotation. The libraries are built with - * -fvisibility=hidden, so only declarations carrying EMBED_API are exported - * from the shared library; everything else stays internal. */ -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - -typedef struct { - char **id_to_text; /* [vocab_size] decoded text strings */ - char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */ - int vocab_size; - - /* Internal hash maps (opaque to callers) */ - void *vocab_map; - int vocab_map_cap; - void *merge_map; - int merge_map_cap; - /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id). - * NULL when the vocab could not resolve every merge; encoding then falls - * back to the string-based merge loop. */ - void *int_merges; - int int_merges_cap; - /* Byte-level unicode codepoint -> single-character token id (512 slots), - * or NULL when any byte token is missing from the vocab. */ - int *cp_to_id; -} embed_tokenizer_t; - -typedef struct embed_tokenizer_workspace embed_tokenizer_workspace_t; - -/* Load tokenizer from vocab.json in model directory */ -EMBED_API embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path); - -/* Decode a single token ID to text. Returns pointer to internal string. */ -EMBED_API const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id); - -/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>"). - * Returns -1 when the token is not in the vocab. */ -EMBED_API int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token); - -/* Encode UTF-8 text into token IDs using BPE. - * Returns malloc'd array of token IDs and sets *out_n_tokens. - * Returns NULL on error (and sets *out_n_tokens to 0). */ -EMBED_API int * -embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens); - -/* Reusable scratch for allocation-light repeated encoding. */ -EMBED_API embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void); -EMBED_API void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws); - -/* Encode into caller-provided out_ids[0..out_cap-1]. - * Returns 0 on success, -1 on error, -2 if out_cap is too small. - * In all cases *out_n_tokens is set to the number of tokens needed/emitted - * when out_n_tokens is non-NULL. */ -EMBED_API int embed_tokenizer_encode_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens); - -/* Same tokenization as embed_tokenizer_encode(), but reuses ws internally and - * performs only the final returned int[] allocation. */ -EMBED_API int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens); - -/* Free tokenizer */ -EMBED_API void embed_tokenizer_free(embed_tokenizer_t *tok); - -#endif /* EMBED_TOKENIZER_H */ diff --git a/src/tokenizer_sentencepiece.h b/src/tokenizer_sentencepiece.h deleted file mode 100644 index b4ce48c..0000000 --- a/src/tokenizer_sentencepiece.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * tokenizer_sentencepiece.h - SentencePiece tokenizer for XLM-R style models. - * - * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`, - * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R - * and multilingual E5. It encodes text only; frontends add the model's special - * token layout, e.g. <s> ... </s> for XLM-R. - */ - -#ifndef SENTENCEPIECE_TOKENIZER_H -#define SENTENCEPIECE_TOKENIZER_H - -#include <stddef.h> -#include <stdint.h> - -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - -typedef enum { - SENTENCEPIECE_IDENTITY = 0, - SENTENCEPIECE_XLM_ROBERTA = 1, -} sentencepiece_id_map_t; - -typedef struct { - char **id_to_piece; /* [piece_count] raw SentencePiece ids */ - uint16_t *piece_lens; /* byte length for id_to_piece entries */ - float *scores; /* SentencePiece score per raw id */ - uint8_t *types; /* SentencePiece piece type per raw id */ - int piece_count; /* raw SentencePiece vocabulary size */ - int vocab_size; /* model embedding vocab size after id mapping */ - int unk_id; /* model-space unknown-token id */ - int bos_id; /* model-space beginning-token id, or -1 */ - int eos_id; /* model-space end-token id, or -1 */ - int pad_id; /* model-space pad-token id, or -1 */ - int mask_id; /* model-space mask-token id, or -1 */ - int max_piece_len; /* max normal/user/unused piece byte length */ - double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ - int add_dummy_prefix; - int remove_extra_whitespaces; - int escape_whitespaces; - sentencepiece_id_map_t id_map; - - void *piece_map; /* piece bytes -> raw SentencePiece id */ - int piece_map_cap; - - uint32_t *xcda; - size_t xcda_count; - char *prefix_replacements; - size_t prefix_replacements_len; -} sentencepiece_tokenizer_t; - -typedef struct sentencepiece_workspace sentencepiece_workspace_t; - -/* Load the first supported SentencePiece model found in model_dir. Returns NULL - * on missing model file, unsupported model type, malformed protobuf, or OOM. */ -EMBED_API sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir); - -/* Decode one model-space id to an internal token string, or NULL when invalid. */ -EMBED_API const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id); - -/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a - * model-space id, or -1 when absent. */ -EMBED_API int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, - const char *token); - -/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns - * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an - * error. */ -EMBED_API int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok, - const char *text, - int *out_n_tokens); - -EMBED_API sentencepiece_workspace_t *sentencepiece_workspace_new(void); -EMBED_API void sentencepiece_workspace_free(sentencepiece_workspace_t *ws); - -/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if - * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */ -EMBED_API int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok, - sentencepiece_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens); - -/* Same tokenization as sentencepiece_tokenizer_encode(), reusing ws internally - * and performing only the final returned int[] allocation. */ -EMBED_API int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok, - sentencepiece_workspace_t *ws, - const char *text, - int *out_n_tokens); - -EMBED_API void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok); - -#endif /* SENTENCEPIECE_TOKENIZER_H */ diff --git a/src/tokenizer_wordpiece.h b/src/tokenizer_wordpiece.h deleted file mode 100644 index 59cb2bb..0000000 --- a/src/tokenizer_wordpiece.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * wordpiece_tokenizer.h - BERT-family WordPiece tokenizer. - * - * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional - * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting) - * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes - * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the - * byte-level BPE backend leaves terminal/separator tokens to the frontend. - * - * The public surface mirrors embed_tokenizer.h (load / encode with a reusable - * workspace / decode / resolve a named special token / free) so both can sit - * behind one tokenizer interface later. - */ - -#ifndef WORDPIECE_TOKENIZER_H -#define WORDPIECE_TOKENIZER_H - -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - -typedef struct { - char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */ - int vocab_size; - int unk_id; /* [UNK] id, resolved at load */ - int do_lower_case; /* lowercase + strip accents before WordPiece */ - int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */ - - /* Opaque token-string -> id hash map. */ - void *vocab_map; - int vocab_map_cap; -} wordpiece_tokenizer_t; - -typedef struct wordpiece_workspace wordpiece_workspace_t; - -/* Load from <model_dir>/vocab.txt, reading do_lower_case from - * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on - * error (missing vocab.txt, missing [UNK], allocation failure). */ -EMBED_API wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir); - -/* Decode a single id to its vocab token string (internal pointer), or NULL. */ -EMBED_API const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id); - -/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */ -EMBED_API int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token); - -/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on - * error (and *out_n_tokens = 0). */ -EMBED_API int * -wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n_tokens); - -/* Reusable scratch for allocation-light repeated encoding. */ -EMBED_API wordpiece_workspace_t *wordpiece_workspace_new(void); -EMBED_API void wordpiece_workspace_free(wordpiece_workspace_t *ws); - -/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1 - * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to - * the number of tokens needed/emitted in all cases. */ -EMBED_API int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok, - wordpiece_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens); - -/* Same tokenization as wordpiece_tokenizer_encode(), reusing ws internally and - * performing only the final returned int[] allocation. */ -EMBED_API int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok, - wordpiece_workspace_t *ws, - const char *text, - int *out_n_tokens); - -EMBED_API void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok); - -#endif /* WORDPIECE_TOKENIZER_H */ diff --git a/tools/cli/Makefile b/tools/cli/Makefile new file mode 100644 index 0000000..cc076f1 --- /dev/null +++ b/tools/cli/Makefile @@ -0,0 +1,22 @@ +ROOT := ../.. +COMPONENT := cli +include $(ROOT)/build.mk + +.DEFAULT_GOAL := all + +LIBFFWD := $(ROOT)/libffwd + +BIN := $(OUTDIR)/ffwd-cli +SRCS := main.c +LIBS := $(OUTDIR)/libffwd.a +LDLIBS := $(BACKEND_LDFLAGS) -lm + +CFLAGS += -std=gnu11 -pthread +CPPFLAGS += -I. -I$(LIBFFWD) + +$(OBJDIR)/main.o: CFLAGS += $(VERSION_CFLAGS) + +$(OUTDIR)/libffwd.a: + $(MAKE) -C $(LIBFFWD) + +$(eval $(program)) diff --git a/tools/cli/main.c b/tools/cli/main.c new file mode 100644 index 0000000..d9516b8 --- /dev/null +++ b/tools/cli/main.c @@ -0,0 +1,556 @@ +/* main.c - ffwd-cli command-line tool */ + +#include "ffwd.h" +#include "internal.h" +#include "build.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <limits.h> +#include <unistd.h> + +static double now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6; +} + +static void print_usage(const char *prog) { + fprintf(stderr, + "Usage: %s -d <model_dir> [options] [text...]\n" + "\n" + "Options:\n" + " -d <dir> Model directory (required)\n" + " --stream Read lines from stdin, write one JSON embedding per line\n" + " --json Emit JSON instead of plain text (embedding or matrix)\n" + " -b, --batch-size N\n" + " Max texts per inference batch (default: all; --stream: 1)\n", + prog); + ffwd_cli_help(stderr); + fprintf(stderr, + " -e, --embeddings\n" + " Print raw embeddings (with multiple texts)\n" + " -v, --verbose\n" + " Verbose (-vv for debug)\n" + " -V, --version\n" + " Print version and exit\n" + " --build-info Print build details and exit\n" + " -h Show this help\n" + "\n" + "Modes:\n" + " 1 text arg Embedding as space-separated floats (JSON with --json)\n" + " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n" + " no args Batch: read all stdin lines, then similarity matrix\n" + " --stream Streaming: read stdin lines, write JSON per line\n" + "\n" + "Examples:\n" + " %s -d ./model \"what is AI?\"\n" + " %s -d ./model --stream < texts.txt\n", + prog, prog); +} + +/* ======================================================================== + * Embed one text: tokenize > forward > return float[dim] + * ======================================================================== */ + +typedef struct { + ffwd_t *backend; + ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */ + int dim; +} runner_t; + +static void runner_free_tokenizers(runner_t *e) { ffwd_tok_free(e->tok); } + +typedef struct { + int *ids; + int n_tokens; +} token_buf_t; + +static int +runner_ffwd_batch(runner_t *e, const ffwd_input_t *inputs, int batch, float *out_embeddings) { + return ffwd_encode_batch(e->backend, inputs, batch, out_embeddings); +} + +static int tokenize_text(runner_t *e, const char *text, token_buf_t *out) { + memset(out, 0, sizeof(*out)); + out->ids = ffwd_tokenize(e->tok, text, NULL, &out->n_tokens); + if (!out->ids) { + fprintf(stderr, "tokenization failed: %s\n", text); + return -1; + } + + if (ffwd_verbose >= 1) { + fprintf(stderr, "tokens (%d): ", out->n_tokens); + for (int i = 0; i < out->n_tokens && i < 20; i++) + fprintf(stderr, "%d ", out->ids[i]); + if (out->n_tokens > 20) + fprintf(stderr, "..."); + fprintf(stderr, "\n"); + } + return 0; +} + +static void free_tokens(token_buf_t *tokens, int n) { + if (!tokens) + return; + for (int i = 0; i < n; i++) + free(tokens[i].ids); +} + +/* ======================================================================== + * Output helpers + * ======================================================================== */ + +static void print_embedding_raw(const float *emb, int dim) { + for (int i = 0; i < dim; i++) { + if (i > 0) + putchar(' '); + printf("%.8f", (double)emb[i]); + } + putchar('\n'); +} + +/* Single embedding as a JSON array of floats. */ +static void print_embedding_array(const float *emb, int dim) { + putchar('['); + for (int i = 0; i < dim; i++) { + if (i) + putchar(','); + printf("%.8f", (double)emb[i]); + } + puts("]"); +} + +static size_t runner_workspace_nbytes(const runner_t *e) { + return e ? ffwd_scratch_nbytes(e->backend) : 0; +} + +static void +print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) { + printf("{\"embedding\":["); + for (int i = 0; i < dim; i++) { + if (i > 0) + putchar(','); + printf("%.8f", (double)emb[i]); + } + printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms, + workspace_bytes); + fflush(stdout); +} + +/* ======================================================================== + * Stdin mode: read lines from stdin, write JSON to stdout + * ======================================================================== */ + +static int process_stdin_batch(runner_t *e, char **lines, int n_lines) { + if (n_lines <= 0) + return 0; + + token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t)); + ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_lines * sizeof(ffwd_input_t)); + int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int)); + float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float)); + if (!tokens || !inputs || !input_to_line || !embs) { + fprintf(stderr, "OOM\n"); + free(tokens); + free(inputs); + free(input_to_line); + free(embs); + for (int i = 0; i < n_lines; i++) + printf("{\"error\":\"embedding failed\"}\n"); + fflush(stdout); + return 1; + } + + int n_inputs = 0; + for (int i = 0; i < n_lines; i++) { + if (tokenize_text(e, lines[i], &tokens[i]) == 0) { + inputs[n_inputs].ids = tokens[i].ids; + inputs[n_inputs].n_tokens = tokens[i].n_tokens; + input_to_line[n_inputs] = i; + n_inputs++; + } + } + + double ms = 0.0; + int rc = 0; + if (n_inputs > 0) { + double t0 = now_ms(); + rc = runner_ffwd_batch(e, inputs, n_inputs, embs); + ms = now_ms() - t0; + if (ffwd_verbose >= 1) + fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms); + } + + int next_valid = 0; + for (int i = 0; i < n_lines; i++) { + if (!tokens[i].ids) { + printf("{\"error\":\"tokenization failed\"}\n"); + continue; + } + if (rc != 0) { + printf("{\"error\":\"embedding failed\"}\n"); + continue; + } + while (next_valid < n_inputs && input_to_line[next_valid] != i) + next_valid++; + if (next_valid >= n_inputs) { + printf("{\"error\":\"embedding failed\"}\n"); + continue; + } + print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms, + runner_workspace_nbytes(e)); + next_valid++; + } + fflush(stdout); + + free_tokens(tokens, n_lines); + free(tokens); + free(inputs); + free(input_to_line); + free(embs); + return rc == 0 ? 0 : 1; +} + +static int run_stdin(runner_t *e, int batch_size) { + char line[65536]; + if (batch_size <= 0) + batch_size = 1; + + if (ffwd_verbose >= 1) + fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size); + + char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *)); + if (!batch_lines) { + fprintf(stderr, "OOM\n"); + return 1; + } + int n_batch = 0; + int rc = 0; + + while (fgets(line, sizeof(line), stdin)) { + /* Strip newline */ + size_t len = strlen(line); + while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) + line[--len] = '\0'; + if (len == 0) + continue; + + if (ffwd_verbose >= 1) + fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line, + len > 60 ? "..." : ""); + + batch_lines[n_batch] = strdup(line); + if (!batch_lines[n_batch]) { + fprintf(stderr, "OOM\n"); + rc = 1; + break; + } + n_batch++; + + if (n_batch == batch_size) { + if (process_stdin_batch(e, batch_lines, n_batch) != 0) + rc = 1; + for (int i = 0; i < n_batch; i++) { + free(batch_lines[i]); + batch_lines[i] = NULL; + } + n_batch = 0; + } + } + + if (n_batch > 0) { + if (process_stdin_batch(e, batch_lines, n_batch) != 0) + rc = 1; + for (int i = 0; i < n_batch; i++) + free(batch_lines[i]); + } + free(batch_lines); + + if (ffwd_verbose >= 1) + fprintf(stderr, "stdin: EOF\n"); + + return rc; +} + +/* ======================================================================== + * Batch mode: embed args or stdin lines, then print similarity or vectors + * ======================================================================== */ + +static int append_text(char ***texts, int *n_texts, int *cap, const char *s) { + if (*n_texts == *cap) { + int new_cap = *cap ? *cap * 2 : 8; + char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *)); + if (!new_texts) + return -1; + *texts = new_texts; + *cap = new_cap; + } + + (*texts)[*n_texts] = strdup(s); + if (!(*texts)[*n_texts]) + return -1; + (*n_texts)++; + return 0; +} + +/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one + * row of space-separated values per line. With print_embs the raw embeddings + * follow, one per line after a blank line (plain-text output only). */ +static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) { + if (json) { + putchar('['); + for (int i = 0; i < n; i++) { + if (i) + putchar(','); + putchar('['); + for (int j = 0; j < n; j++) { + if (j) + putchar(','); + printf("%.6f", (double)ffwd_cosine_similarity(embs + (size_t)i * dim, + embs + (size_t)j * dim, dim)); + } + putchar(']'); + } + puts("]"); + return; + } + + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + if (j) + putchar(' '); + printf("%.6f", (double)ffwd_cosine_similarity(embs + (size_t)i * dim, + embs + (size_t)j * dim, dim)); + } + putchar('\n'); + } + if (print_embs) { + putchar('\n'); + for (int i = 0; i < n; i++) + print_embedding_raw(embs + (size_t)i * dim, dim); + } +} + +static int run_batch( + runner_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) { + int n_texts = 0, cap = 0; + char **texts = NULL; + int rc = 0; + + if (arg_start < argc) { + for (int i = arg_start; i < argc; i++) { + if (append_text(&texts, &n_texts, &cap, argv[i]) != 0) + goto oom_texts; + } + } else { + char line[65536]; + while (fgets(line, sizeof(line), stdin)) { + size_t l = strlen(line); + while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r')) + line[--l] = '\0'; + if (l == 0) + continue; + if (append_text(&texts, &n_texts, &cap, line) != 0) + goto oom_texts; + } + } + + if (n_texts == 0) { + fprintf(stderr, "no input texts\n"); + free(texts); + return 1; + } + + int dim = e->dim; + token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t)); + ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_texts * sizeof(ffwd_input_t)); + float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float)); + if (!tokens || !inputs || !embs) { + fprintf(stderr, "OOM\n"); + free(tokens); + free(inputs); + free(embs); + for (int i = 0; i < n_texts; i++) + free(texts[i]); + free(texts); + return 1; + } + + for (int i = 0; i < n_texts; i++) { + if (ffwd_verbose >= 1) + fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]); + if (tokenize_text(e, texts[i], &tokens[i]) != 0) { + rc = 1; + goto done; + } + inputs[i].ids = tokens[i].ids; + inputs[i].n_tokens = tokens[i].n_tokens; + } + + int max_batch = batch_size > 0 ? batch_size : n_texts; + for (int start = 0; start < n_texts; start += max_batch) { + int cur = n_texts - start; + if (cur > max_batch) + cur = max_batch; + + double t0 = now_ms(); + if (runner_ffwd_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) { + fprintf(stderr, "forward pass failed\n"); + rc = 1; + goto done; + } + if (ffwd_verbose >= 1) + fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1, + cur, now_ms() - t0); + } + + if (n_texts == 1) { + if (json) + print_embedding_array(embs, dim); + else + print_embedding_raw(embs, dim); + } else { + print_matrix(embs, n_texts, dim, json, print_embs); + } + +done: + free_tokens(tokens, n_texts); + for (int i = 0; i < n_texts; i++) + free(texts[i]); + free(texts); + free(tokens); + free(inputs); + free(embs); + return rc; + +oom_texts: + fprintf(stderr, "OOM\n"); + for (int i = 0; i < n_texts; i++) + free(texts[i]); + free(texts); + return 1; +} + +/* ======================================================================== + * Main + * ======================================================================== */ + +int main(int argc, char *argv[]) { + const char *model_dir = NULL; + int print_embs = 0; + int verbose = 0; + int stdin_mode = 0; + int batch_size = 0; + int json = 0; + const char *prog = ffwd_prog_name(argv[0]); + /* All backend tuning flags parse unconditionally into opts; each backend + * uses the fields it understands and ignores the rest. */ + ffwd_options_t opts = {.gpu_quant_group_size = 64}; + + int arg_start = 1; + while (arg_start < argc && argv[arg_start][0] == '-') { + const char *f = argv[arg_start]; + if (!strcmp(f, "-d")) { + model_dir = argv[++arg_start]; + } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) { + ffwd_print_version(prog); + return 0; + } else if (!strcmp(f, "--build-info")) { + ffwd_print_build_info(prog); + return 0; + } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + opts.n_threads = atoi(argv[++arg_start]); + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + batch_size = atoi(argv[++arg_start]); + } else if (!strcmp(f, "--json")) { + json = 1; + } else if (!strcmp(f, "--gpu-quant-bits")) { + opts.gpu_quant_bits = atoi(argv[++arg_start]); + } else if (!strcmp(f, "--gpu-quant-group-size")) { + opts.gpu_quant_group_size = atoi(argv[++arg_start]); + } else if (!strcmp(f, "--gpu-gemm-mode")) { + opts.gpu_gemm_mode = argv[++arg_start]; + } else if (!strcmp(f, "--gpu-weight-dtype")) { + opts.gpu_weight_dtype = argv[++arg_start]; + } else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) { + print_embs = 1; + } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { + verbose++; + } else if (!strcmp(f, "-vv")) { + verbose = 2; + } else if (!strcmp(f, "--stream")) { + stdin_mode = 1; + } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + print_usage(prog); + return 0; + } else if (!strcmp(f, "--")) { + /* End of options: everything after is positional text, even if it + * starts with a dash. */ + arg_start++; + break; + } else if (f[0] == '-' && f[1] == '-') { + /* An unrecognized --option is a hard error, not silently-embedded + * text. Catches typos and stale flags (e.g. the removed --backend) + * loudly instead of running the default path or embedding the flag. */ + fprintf(stderr, "unknown option: %s\n", f); + print_usage(prog); + return 1; + } else + break; + arg_start++; + } + + ffwd_verbose = verbose; + + if (!model_dir || !model_dir[0]) { + fprintf(stderr, "model directory required (-d <dir>)\n"); + print_usage(prog); + return 1; + } + + char err[256]; + if (ffwd_init(&opts, err, sizeof(err)) != 0) { + fprintf(stderr, "%s\n", err); + return 1; + } + + /* Tokenizer (file probe + special-token layout) is owned by libffwd */ + runner_t e = {0}; + double t0 = now_ms(); + e.tok = ffwd_tok_open(model_dir, 0, err, sizeof(err)); + if (!e.tok) { + fprintf(stderr, "%s\n", err); + return 1; + } + if (verbose >= 1) + fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0); + + /* Model */ + t0 = now_ms(); + e.backend = ffwd_open(model_dir, 0, &opts, err, sizeof(err)); + if (!e.backend || ffwd_activate(e.backend, err, sizeof(err)) != 0) { + fprintf(stderr, "%s\n", err); + ffwd_free(e.backend); + runner_free_tokenizers(&e); + return 1; + } + e.dim = ffwd_config(e.backend)->hidden_size; + ffwd_tok_set_append_terminal(e.tok, ffwd_config(e.backend)->append_terminal_token); + if (verbose >= 1) + fprintf(stderr, "Model: %d-dim, %.0f ms\n", e.dim, now_ms() - t0); + + /* Run */ + int rc; + if (stdin_mode) + rc = run_stdin(&e, batch_size > 0 ? batch_size : 1); + else + rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json); + + /* Cleanup */ + ffwd_free(e.backend); + runner_free_tokenizers(&e); + return rc; +} diff --git a/tools/server/Makefile b/tools/server/Makefile new file mode 100644 index 0000000..4a11b0d --- /dev/null +++ b/tools/server/Makefile @@ -0,0 +1,49 @@ +ROOT := ../.. +COMPONENT := server +include $(ROOT)/build.mk + +LIBFFWD := $(ROOT)/libffwd +AE := $(ROOT)/deps/ae + +BIN := $(OUTDIR)/ffwd-server +SRCS := base64.c encode.c handlers.c http.c json.c main.c models.c sbuf.c \ + schedule.c server.c util.c +OBJS := $(call objs,$(SRCS)) +DEPS := $(call deps,$(OBJS)) + +TEST_BINS := test_base64 test_sbuf test_encode test_json test_schedule test_models +TEST_OBJS := $(filter-out $(OBJDIR)/main.o,$(OBJS)) + +LIBS := $(OUTDIR)/libffwd.a $(OUTDIR)/libae.a +LDLIBS := $(CJSON_LDFLAGS) $(BACKEND_LDFLAGS) -lm + +CFLAGS += -std=gnu11 -pthread +CPPFLAGS += -I. -I$(LIBFFWD) -I$(AE) $(CJSON_CFLAGS) + +$(OBJDIR)/main.o: CFLAGS += $(VERSION_CFLAGS) + +.PHONY: all clean test + +all: $(BIN) + +$(eval $(call link_program,$(BIN),$(OBJS),$(LIBS),$(LDLIBS))) + +test: $(TEST_BINS) + @rc=0; for t in $(TEST_BINS); do ./$$t || rc=1; done; \ + if [ $$rc -eq 0 ]; then echo "server ok"; else echo "server not ok"; exit 1; fi + +$(TEST_BINS): %: %.c test_util.h $(TEST_OBJS) $(LIBS) + $(CC) $(CFLAGS) $(CPPFLAGS) $< $(TEST_OBJS) $(LIBS) $(LDLIBS) $(LDFLAGS_OPT) -o $@ + +$(OUTDIR)/libffwd.a: + $(MAKE) -C $(LIBFFWD) + +$(OUTDIR)/libae.a: + $(MAKE) -C $(AE) + +clean: + $(RM) -r $(OBJDIR) $(BIN) $(BIN).dSYM \ + $(TEST_BINS) $(addsuffix .dSYM,$(TEST_BINS)) + $(MAKE) -C $(AE) clean + +-include $(DEPS) diff --git a/src/base64.c b/tools/server/base64.c index 082101f..87351d2 100644 --- a/src/base64.c +++ b/tools/server/base64.c @@ -1,5 +1,5 @@ #include "base64.h" -#include "server_util.h" +#include "util.h" #include <stdint.h> diff --git a/src/base64.h b/tools/server/base64.h index 17e2036..88d88b3 100644 --- a/src/base64.h +++ b/tools/server/base64.h @@ -1,5 +1,5 @@ -#ifndef EMBED_BASE64_H -#define EMBED_BASE64_H +#ifndef FFWD_BASE64_H +#define FFWD_BASE64_H #include <stddef.h> @@ -7,4 +7,4 @@ * string the caller frees; die-on-OOM, so it never returns NULL. */ char *base64_encode(const unsigned char *src, size_t n); -#endif /* EMBED_BASE64_H */ +#endif /* FFWD_BASE64_H */ diff --git a/src/server_encode.c b/tools/server/encode.c index ef4e3ec..faa58a0 100644 --- a/src/server_encode.c +++ b/tools/server/encode.c @@ -1,8 +1,8 @@ #include "server_internal.h" -#include "server_util.h" +#include "util.h" #include "sbuf.h" #include "base64.h" -#include "embed.h" +#include "ffwd.h" #include <math.h> #include <stdlib.h> @@ -73,7 +73,7 @@ void append_embedding_value( if (i) sbuf_putc(b, ','); float v = - api == EMBED_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i]; + api == FFWD_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i]; sbuf_printf(b, "%.9g", (double)v); } sbuf_puts(b, "]}"); @@ -101,7 +101,7 @@ int render_embedding_response(embedding_request *r, const float *embs) { for (int i = 0; i < r->n_inputs; i++) { float *dst = truncated + (size_t)i * r->dims; memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst)); - if (embed_l2_normalize(dst, r->dims) != 0) { + if (ffwd_l2_normalize(dst, r->dims) != 0) { free(truncated); return -1; } diff --git a/src/server_handlers.c b/tools/server/handlers.c index b3476d9..6720784 100644 --- a/src/server_handlers.c +++ b/tools/server/handlers.c @@ -1,6 +1,6 @@ #include "server_internal.h" -#include "server_util.h" -#include "embed.h" +#include "util.h" +#include "ffwd.h" #include <limits.h> #include <stdio.h> @@ -11,10 +11,10 @@ * Endpoint handlers: prepare, batch, execute, render * * One prepare_* per endpoint validates the request JSON and tokenizes its - * inputs into a *_request; the worker (server_schedule.c) groups compatible + * inputs into a *_request; the worker (schedule.c) groups compatible * ready requests and calls the matching execute_*_request_list, which runs the - * batched forward pass through model_embed_* and renders each response via - * server_encode.c. Rerank runs MaxSim over late-interaction token vectors. + * batched forward pass through model_ffwd_* and renders each response via + * encode.c. Rerank runs MaxSim over late-interaction token vectors. * ======================================================================== */ int embedding_request_compatible(const embedding_request *a, const embedding_request *b) { @@ -23,7 +23,7 @@ int embedding_request_compatible(const embedding_request *a, const embedding_req } typedef struct { - embed_input_t input; + ffwd_input_t input; int output_index; } embedding_batch_item; @@ -74,7 +74,7 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { if (max_batch > total_inputs) max_batch = total_inputs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + ffwd_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float)); int failed = 0; @@ -91,9 +91,9 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) { cur++; } uint64_t t0 = nstime(); - int embed_rc = model_embed_batch(m, inputs, cur, batch_embs); + int ffwd_rc = model_ffwd_batch(m, inputs, cur, batch_embs); infer_ns += nstime() - t0; - if (embed_rc != 0) { + if (ffwd_rc != 0) { failed = 1; break; } @@ -135,12 +135,11 @@ int contextual_request_fits_group(const contextual_request *r, int max_batch_tokens) { if (group_docs == 0) return 1; - return r->n_docs <= max_batch - group_docs && - r->total_tokens <= max_batch_tokens - group_tokens; + return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens; } typedef struct { - embed_context_input_t input; + ffwd_context_input_t input; int output_span_index; int order; } contextual_batch_item; @@ -201,7 +200,7 @@ void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { if (max_batch > total_docs) max_batch = total_docs; int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens; - embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); + ffwd_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs)); float *batch_embs = xmalloc(sizeof(*batch_embs)); size_t batch_emb_cap = 0; @@ -212,7 +211,7 @@ void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { int tokens = 0; int chunks = 0; while (start + cur < total_docs && cur < max_batch) { - const embed_context_input_t *input = &items[start + cur].input; + const ffwd_context_input_t *input = &items[start + cur].input; if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens, max_batch_tokens)) break; @@ -227,9 +226,9 @@ void execute_contextual_request_list(contextual_request **reqs, int n_reqs) { batch_emb_cap = needed; } uint64_t t0 = nstime(); - int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs); + int ffwd_rc = model_ffwd_spans_batch(m, inputs, cur, batch_embs); infer_ns += nstime() - t0; - if (embed_rc != 0) { + if (ffwd_rc != 0) { failed = 1; break; } @@ -283,13 +282,12 @@ static int validate_common(cJSON *root, ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint", "enum"); } else if (m->info->kind != expected_kind) { - ve_add(detail, "[\"body\",\"model\"]", - "model is not available on this endpoint", "enum"); + ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum"); m = NULL; } } *out_model = m; - const char *enc = encoding_from_root(root, detail, m ? m->info->api : EMBED_API_OPENAI); + const char *enc = encoding_from_root(root, detail, m ? m->info->api : FFWD_API_OPENAI); int min_dim = m ? m->info->min_dim : 128; int max_dim = m ? m->info->dim : 2560; int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc); @@ -330,7 +328,7 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re if (n_inputs < 1) ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item", "value_error.list.min_items"); - else if (n_inputs > EMBED_API_MAX_STANDARD_INPUTS) + else if (n_inputs > FFWD_API_MAX_STANDARD_INPUTS) ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items", "value_error.list.max_items"); cJSON *item; @@ -364,7 +362,7 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re out->inputs[0].ids = out->tokens[0].ids; out->inputs[0].n_tokens = out->tokens[0].n_tokens; out->total_tokens = out->tokens[0].n_tokens; - if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS) + if (out->tokens[0].n_tokens > FFWD_API_MAX_ITEM_TOKENS) ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit", "value_error.context_length"); } @@ -378,14 +376,13 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re idx++; continue; } - if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != - 0) { + if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != 0) { ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { out->inputs[idx].ids = out->tokens[idx].ids; out->inputs[idx].n_tokens = out->tokens[idx].n_tokens; out->total_tokens += out->tokens[idx].n_tokens; - if (out->tokens[idx].n_tokens > EMBED_API_MAX_ITEM_TOKENS) + if (out->tokens[idx].n_tokens > FFWD_API_MAX_ITEM_TOKENS) ve_add(detail, loc, "input exceeds 32768 token limit", "value_error.context_length"); } @@ -393,7 +390,7 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re } } - if (out->total_tokens > EMBED_API_MAX_TOTAL_TOKENS) + if (out->total_tokens > FFWD_API_MAX_TOTAL_TOKENS) ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", "value_error.context_length"); } @@ -435,7 +432,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ if (n_docs < 1) ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document", "value_error.list.min_items"); - else if (n_docs > EMBED_API_MAX_CONTEXT_DOCS) + else if (n_docs > FFWD_API_MAX_CONTEXT_DOCS) ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents", "value_error.list.max_items"); cJSON *doc_arr; @@ -449,8 +446,8 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ continue; } int doc_chunks = cJSON_GetArraySize(doc_arr); - if (doc_chunks > EMBED_API_MAX_CONTEXT_CHUNKS - total_chunks) - total_chunks = EMBED_API_MAX_CONTEXT_CHUNKS + 1; + if (doc_chunks > FFWD_API_MAX_CONTEXT_CHUNKS - total_chunks) + total_chunks = FFWD_API_MAX_CONTEXT_CHUNKS + 1; else total_chunks += doc_chunks; if (doc_chunks < 1) { @@ -473,7 +470,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ } di++; } - if (total_chunks > EMBED_API_MAX_CONTEXT_CHUNKS) + if (total_chunks > FFWD_API_MAX_CONTEXT_CHUNKS) ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks", "value_error.list.max_items"); } @@ -507,7 +504,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ ci++; } - if (doc_tokens > EMBED_API_MAX_ITEM_TOKENS) { + if (doc_tokens > FFWD_API_MAX_ITEM_TOKENS) { char loc[64]; snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di); ve_add(detail, loc, "document exceeds 32768 token limit", @@ -519,6 +516,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ if (doc_valid) { doc->n_tokens = (int)doc_tokens; doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids)); + int sep_id = ffwd_tok_context_separator_id(m->tok); int pos = 0; for (ci = 0; ci < n_chunks; ci++) { doc->spans[ci].start = pos; @@ -531,7 +529,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids)); pos += chunk_tokens[ci].n_tokens; if (ci + 1 < n_chunks) { - doc->ids[pos++] = m->context_separator_id; + doc->ids[pos++] = sep_id; } } } @@ -539,7 +537,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_ free(chunk_tokens); di++; } - if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) { + if (request_tokens > FFWD_API_MAX_TOTAL_TOKENS) { ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit", "value_error.context_length"); } else { @@ -617,7 +615,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request if (n_documents < 1) ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item", "value_error.list.min_items"); - else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS) + else if (n_documents > FFWD_API_MAX_RERANK_DOCUMENTS) ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items", "value_error.list.max_items"); cJSON *document; @@ -665,11 +663,10 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request out->n_documents = n_documents; out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents)); if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) { - ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", - "value_error.tokenization"); + ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization"); } else { out->query_tokens = out->query.n_tokens; - if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS) + if (out->query_tokens > FFWD_API_MAX_ITEM_TOKENS) ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit", "value_error.context_length"); } @@ -684,7 +681,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request ve_add(detail, loc, "tokenization failed", "value_error.tokenization"); } else { int n = out->documents[i].n_tokens; - if (n > EMBED_API_MAX_ITEM_TOKENS) + if (n > FFWD_API_MAX_ITEM_TOKENS) ve_add(detail, loc, "document exceeds 32768 token limit", "value_error.context_length"); if (out->document_tokens <= INT_MAX - n) @@ -694,7 +691,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request } i++; } - if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) { + if (out->query_tokens > FFWD_API_MAX_TOTAL_TOKENS - out->document_tokens) { ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit", "value_error.context_length"); } @@ -725,101 +722,17 @@ static int rerank_result_cmp(const void *a, const void *b) { return ra->index - rb->index; } -static int execute_rerank_cpu(rerank_request *r, float *scores) { - loaded_model *m = r->model; - int dim = m->info->token_dim; - int total_doc_vecs = 0; - for (int i = 0; i < r->n_documents; i++) - total_doc_vecs += r->documents[i].n_keep; - - float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); - float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); - int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - - /* Encode every candidate in one block-diagonal forward. The encoder packs - * each document's kept token vectors back-to-back and fills offsets, so the - * cost of N short documents collapses to a single packed forward. */ - const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); - const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); - int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); - int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); - for (int i = 0; i < r->n_documents; i++) { - doc_ids[i] = r->documents[i].ids; - keep[i] = r->documents[i].keep; - n_tokens[i] = r->documents[i].n_tokens; - n_keep[i] = r->documents[i].n_keep; - } - - int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids, - r->query.n_tokens, 1, query); - if (rc == 0) - rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens, - keep, n_keep, r->n_documents, 1, docs, offsets); - if (rc == 0) - rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, - scores); - free(n_keep); - free(n_tokens); - free(keep); - free(doc_ids); - free(offsets); - free(docs); - free(query); - return rc; -} - -#ifdef USE_MLX -static int execute_rerank_mlx(rerank_request *r, float *scores) { - loaded_model *m = r->model; - embed_mlx_late_vectors_t *query = - embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1); - - /* Encode every candidate in one padded transformer pass. The encoder packs - * each document's kept token vectors back-to-back and fills offsets, so the - * cost of N short documents collapses to a single batched forward. */ - const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); - const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); - int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); - int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep)); - int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); - for (int i = 0; i < r->n_documents; i++) { - doc_ids[i] = r->documents[i].ids; - keep[i] = r->documents[i].keep; - n_tokens[i] = r->documents[i].n_tokens; - n_keep[i] = r->documents[i].n_keep; +void execute_rerank_request(rerank_request *r) { + if (!r || !r->model) { + if (r && r->j) + job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); + return; } - embed_mlx_late_vectors_t *packed = - query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep, - r->n_documents, 1, offsets) - : NULL; - int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets, - r->n_documents, scores) - : -1; - embed_mlx_late_vectors_free(packed); - embed_mlx_late_vectors_free(query); - free(offsets); - free(n_keep); - free(n_tokens); - free(keep); - free(doc_ids); - return rc; -} -#endif - -#ifdef USE_CUDA -static int execute_rerank_cuda(rerank_request *r, float *scores) { - loaded_model *m = r->model; - int dim = m->info->token_dim; - int total_doc_vecs = 0; - for (int i = 0; i < r->n_documents; i++) - total_doc_vecs += r->documents[i].n_keep; - - float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query)); - float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs)); - int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets)); + float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); - /* GPU encodes the query and every candidate (one packed forward); MaxSim - * runs on the host, where the grouped-GEMM scorer beats a device graph. */ + /* Flatten the per-document fields into the arrays the backend rerank takes; + * it encodes the query and every candidate in one packed forward and scores + * MaxSim, so N short documents collapse to a single batch. */ const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids)); const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep)); int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens)); @@ -830,43 +743,23 @@ static int execute_rerank_cuda(rerank_request *r, float *scores) { n_tokens[i] = r->documents[i].n_tokens; n_keep[i] = r->documents[i].n_keep; } - - int rc = - embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query); - if (rc == 0) - rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep, - r->n_documents, 1, docs, offsets); - if (rc == 0) - rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim, - scores); + ffwd_rerank_input_t in = { + .query_ids = r->query.ids, + .query_n_tokens = r->query.n_tokens, + .query_n_keep = r->query.n_keep, + .doc_ids = doc_ids, + .doc_n_tokens = n_tokens, + .doc_keep = keep, + .doc_n_keep = n_keep, + .n_docs = r->n_documents, + }; + uint64_t t0 = nstime(); + int rc = ffwd_rerank(r->model->backend, &in, scores); + r->j->infer_ns += nstime() - t0; free(n_keep); free(n_tokens); free(keep); free(doc_ids); - free(offsets); - free(docs); - free(query); - return rc; -} -#endif - -void execute_rerank_request(rerank_request *r) { - if (!r || !r->model) { - if (r && r->j) - job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error"); - return; - } - float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores)); - uint64_t t0 = nstime(); -#if defined(USE_MLX) - int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores); -#elif defined(USE_CUDA) - int rc = - r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores); -#else - int rc = execute_rerank_cpu(r, scores); -#endif - r->j->infer_ns += nstime() - t0; if (rc != 0) { free(scores); job_set_error(r->j, 500, "late-interaction reranking failed", "server_error"); diff --git a/src/server_http.c b/tools/server/http.c index f155aec..a1d347f 100644 --- a/src/server_http.c +++ b/tools/server/http.c @@ -1,5 +1,5 @@ #include "server_internal.h" -#include "server_util.h" +#include "util.h" #include <ctype.h> #include <errno.h> @@ -240,7 +240,7 @@ static int parse_request_headers(client *c) { char *end = NULL; errno = 0; unsigned long long v = strtoull(cl, &end, 10); - if (errno || !end || *end || v > EMBED_HTTP_MAX_BODY) { + if (errno || !end || *end || v > FFWD_HTTP_MAX_BODY) { free(cl); return -1; } @@ -255,7 +255,7 @@ static int complete_request(client *c) { if (!c->header_done) { ssize_t hend = find_header_end(c->in.ptr, c->in.len); if (hend < 0) { - if (c->in.len > EMBED_HTTP_MAX_HEADER) + if (c->in.len > FFWD_HTTP_MAX_HEADER) return -1; return 0; } @@ -264,7 +264,7 @@ static int complete_request(client *c) { if (parse_request_headers(c) != 0) return -1; } - if (c->content_length > EMBED_HTTP_MAX_BODY) + if (c->content_length > FFWD_HTTP_MAX_BODY) return -1; if (c->in.len < c->header_len + c->content_length) return 0; @@ -368,7 +368,7 @@ void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { (void)loop; (void)mask; client *c = clientData; - char tmp[EMBED_HTTP_IO_BUF]; + char tmp[FFWD_HTTP_IO_BUF]; for (;;) { ssize_t n = read(fd, tmp, sizeof(tmp)); if (n > 0) { diff --git a/src/server_json.c b/tools/server/json.c index cca8fd3..4812164 100644 --- a/src/server_json.c +++ b/tools/server/json.c @@ -1,5 +1,5 @@ #include "server_internal.h" -#include "server_util.h" +#include "util.h" #include <cjson/cJSON.h> #include <ctype.h> @@ -143,7 +143,7 @@ bool cjson_is_integer(cJSON *item) { } const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) { - const char *dflt = api == EMBED_API_PERPLEXITY ? "base64_int8" : "float"; + const char *dflt = api == FFWD_API_PERPLEXITY ? "base64_int8" : "float"; cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format"); if (!encoding) return dflt; @@ -153,13 +153,12 @@ const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) return dflt; } const char *v = encoding->valuestring; - int ok = api == EMBED_API_PERPLEXITY - ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || - !strcmp(v, "float")) + int ok = api == FFWD_API_PERPLEXITY + ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || !strcmp(v, "float")) : (!strcmp(v, "float") || !strcmp(v, "base64")); if (!ok) { ve_add(detail, "[\"body\",\"encoding_format\"]", - api == EMBED_API_PERPLEXITY + api == FFWD_API_PERPLEXITY ? "value is not a valid enum member; permitted: " "'base64_int8', 'base64_binary', 'float'" : "value is not a valid enum member; permitted: 'float', 'base64'", @@ -182,8 +181,7 @@ int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) { return 0; } if (!cJSON_IsString(tt) || !tt->valuestring) { - ve_add(detail, "[\"body\",\"text_type\"]", "text_type must be a string", - "type_error.string"); + ve_add(detail, "[\"body\",\"text_type\"]", "text_type must be a string", "type_error.string"); return 0; } const char *v = tt->valuestring; @@ -196,8 +194,7 @@ int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) { return 0; } -int dimensions_from_root( - cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) { +int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) { int dims = max_dim; cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions"); if (dimensions) { @@ -208,8 +205,7 @@ int dimensions_from_root( dims = dimensions->valueint; if (dims < min_dim || dims > max_dim) { char msg[96]; - snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim, - max_dim); + snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim, max_dim); ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range"); } } diff --git a/tools/server/main.c b/tools/server/main.c new file mode 100644 index 0000000..1cbdc3b --- /dev/null +++ b/tools/server/main.c @@ -0,0 +1,342 @@ +/* main.c - ffwd-server entry point. + * + * The frontend for the ffwd-server binary: argument parsing, model-spec + * handling, and main(). The server logic lives in server.c; this file only + * builds the ffwd_server_config_t and calls ffwd_run_server. It is split + * from server.c so the test suite can include server.c (for its internal + * functions) without a conflicting main() and without a compile-out guard. */ +#include "build.h" +#include "server.h" +#include "server_internal.h" +#include "ffwd.h" +#include "internal.h" + +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static void print_usage(const char *prog) { + fprintf(stderr, + "Usage: %s --model LABEL=DIR [options]\n" + "\n" + "Options:\n" + " --model LABEL=DIR[:key=val]\n" + " Standard embedding model (repeatable)\n" + " --contextual-model LABEL=DIR[:key=val]\n" + " Contextual embedding model (repeatable)\n" + " --late-model LABEL=DIR[:key=val]\n" + " Late-interaction rerank model (repeatable)\n" + " keys: api=openai|perplexity, min_dim=N,\n" + " query=TEXT (last key; \\n escapes newline)\n" + " --host HOST Bind host (default: 127.0.0.1)\n" + " --port N Bind port (default: 8000)\n" + " --api-key K Require Authorization: Bearer K\n" + " -b, --batch-size N Max texts or documents per inference batch\n" + " (default: 32)\n" + " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n" + " --batch-wait-us N First-arrival micro-batch deadline in us\n" + " (default: %d)\n", + prog, ffwd_default_batch_wait_us()); + ffwd_server_help(stderr); + fprintf(stderr, + " -v, --verbose Verbose (-vv for debug)\n" + " -V, --version Print version and exit\n" + " --build-info Print build details and exit\n" + " -h, --help Show this help\n" + "\n" + "Examples:\n" + " %s --model docs=./model --port 8000\n" + " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n", + prog, prog); +} + +typedef struct { + ffwd_server_model_spec_t *v; + int n; + int cap; +} model_specs_t; + +static char *dup_range(const char *p, size_t n) { + char *s = (char *)malloc(n + 1); + if (!s) + return NULL; + memcpy(s, p, n); + s[n] = '\0'; + return s; +} + +static char *unescape_model_option(const char *s) { + size_t n = strlen(s); + char *out = (char *)malloc(n + 1); + if (!out) + return NULL; + size_t w = 0; + for (size_t r = 0; r < n; r++) { + if (s[r] == '\\' && r + 1 < n) { + r++; + if (s[r] == 'n') + out[w++] = '\n'; + else if (s[r] == 't') + out[w++] = '\t'; + else if (s[r] == 'r') + out[w++] = '\r'; + else + out[w++] = s[r]; + } else { + out[w++] = s[r]; + } + } + out[w] = '\0'; + return out; +} + +static int parse_model_option(ffwd_server_model_spec_t *spec, const char *opt, const char *flag) { + const char *eq = strchr(opt, '='); + if (!eq || eq == opt || !eq[1]) { + fprintf(stderr, "%s option expects key=value: %s\n", flag, opt); + return -1; + } + size_t key_len = (size_t)(eq - opt); + const char *val = eq + 1; + if (key_len == 3 && strncmp(opt, "api", key_len) == 0) { + if (!strcmp(val, "openai")) { + spec->api = FFWD_SERVER_API_OPENAI; + } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) { + spec->api = FFWD_SERVER_API_PERPLEXITY; + } else if (!strcmp(val, "default")) { + spec->api = FFWD_SERVER_API_DEFAULT; + } else { + fprintf(stderr, "%s api must be openai or perplexity\n", flag); + return -1; + } + return 0; + } + if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) { + char *end = NULL; + long v = strtol(val, &end, 10); + if (!end || *end || v <= 0 || v > INT_MAX) { + fprintf(stderr, "%s min_dim must be a positive integer\n", flag); + return -1; + } + spec->min_dim = (int)v; + return 0; + } + fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt); + return -1; +} + +static int append_model_spec(model_specs_t *specs, + const char *arg, + ffwd_server_model_kind_t kind, + const char *flag) { + const char *eq = strchr(arg, '='); + if (!eq || eq == arg || !eq[1]) { + fprintf(stderr, "%s expects LABEL=DIR\n", flag); + return -1; + } + if (specs->n == specs->cap) { + int new_cap = specs->cap ? specs->cap * 2 : 4; + void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0])); + if (!p) + return -1; + specs->v = p; + specs->cap = new_cap; + } + size_t id_len = (size_t)(eq - arg); + const char *path_start = eq + 1; + const char *opts = strchr(path_start, ':'); + size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start); + if (path_len == 0) { + fprintf(stderr, "%s expects a non-empty DIR\n", flag); + return -1; + } + ffwd_server_model_spec_t spec; + memset(&spec, 0, sizeof(spec)); + spec.id = dup_range(arg, id_len); + spec.path = dup_range(path_start, path_len); + spec.kind = kind; + if (!spec.id || !spec.path) { + free((char *)spec.id); + free((char *)spec.path); + return -1; + } + const char *p = opts ? opts + 1 : NULL; + while (p && *p) { + if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) { + const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15); + spec.query_instruct = unescape_model_option(val); + if (!spec.query_instruct) + goto fail; + break; + } + const char *next = strchr(p, ':'); + size_t opt_len = next ? (size_t)(next - p) : strlen(p); + char *opt = dup_range(p, opt_len); + if (!opt) + goto fail; + int rc = parse_model_option(&spec, opt, flag); + free(opt); + if (rc != 0) + goto fail; + p = next ? next + 1 : NULL; + } + specs->v[specs->n] = spec; + specs->n++; + return 0; + +fail: + free((char *)spec.id); + free((char *)spec.path); + free((char *)spec.query_instruct); + return -1; +} + +static void free_model_specs(model_specs_t *specs) { + if (!specs) + return; + for (int i = 0; i < specs->n; i++) { + free((char *)specs->v[i].id); + free((char *)specs->v[i].path); + free((char *)specs->v[i].query_instruct); + } + free(specs->v); +} + +int main(int argc, char *argv[]) { + int verbose = 0; + int batch_size = 0; + int max_batch_tokens = 0; + int batch_wait_us = -1; + /* Backend tuning flags parse into opts regardless of backend; the linked + * backend uses what it supports. */ + ffwd_options_t opts = {.gpu_quant_group_size = 64}; + const char *host = "127.0.0.1"; + const char *api_key = NULL; + int port = 8000; + model_specs_t model_specs = {0}; + + const char *prog = ffwd_prog_name(argv[0]); + int arg = 1; + while (arg < argc && argv[arg][0] == '-') { + const char *f = argv[arg]; + if (!strcmp(f, "-V") || !strcmp(f, "--version")) { + ffwd_print_version(prog); + free_model_specs(&model_specs); + return 0; + } else if (!strcmp(f, "--build-info")) { + ffwd_print_build_info(prog); + free_model_specs(&model_specs); + return 0; + } else if (!strcmp(f, "--model")) { + if (arg + 1 >= argc || + append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_STANDARD, f) != 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--contextual-model")) { + if (arg + 1 >= argc || append_model_spec(&model_specs, argv[++arg], + FFWD_SERVER_MODEL_CONTEXTUAL, f) != 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--late-model")) { + if (arg + 1 >= argc || + append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_LATE, f) != 0) { + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--gpu-quant-bits")) { + opts.gpu_quant_bits = atoi(argv[++arg]); + } else if (!strcmp(f, "--gpu-quant-group-size")) { + opts.gpu_quant_group_size = atoi(argv[++arg]); + } else if (!strcmp(f, "--memory-utilization")) { + opts.memory_utilization = atof(argv[++arg]); + if (opts.memory_utilization <= 0.0) { + fprintf(stderr, "--memory-utilization must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--gpu-gemm-mode")) { + opts.gpu_gemm_mode = argv[++arg]; + } else if (!strcmp(f, "--gpu-weight-dtype")) { + opts.gpu_weight_dtype = argv[++arg]; + } else if (!strcmp(f, "--host")) { + host = argv[++arg]; + } else if (!strcmp(f, "--port")) { + port = atoi(argv[++arg]); + } else if (!strcmp(f, "--api-key")) { + api_key = argv[++arg]; + } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) { + batch_size = atoi(argv[++arg]); + } else if (!strcmp(f, "--max-batch-tokens")) { + max_batch_tokens = atoi(argv[++arg]); + if (max_batch_tokens <= 0) { + fprintf(stderr, "--max-batch-tokens must be > 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "--batch-wait-us")) { + batch_wait_us = atoi(argv[++arg]); + if (batch_wait_us < 0) { + fprintf(stderr, "--batch-wait-us must be >= 0\n"); + free_model_specs(&model_specs); + return 1; + } + } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) { + opts.n_threads = atoi(argv[++arg]); + } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) { + verbose++; + } else if (!strcmp(f, "-vv")) { + verbose = 2; + } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) { + print_usage(prog); + free_model_specs(&model_specs); + return 0; + } else { + fprintf(stderr, "unknown option: %s\n", f); + print_usage(prog); + free_model_specs(&model_specs); + return 1; + } + arg++; + } + + ffwd_verbose = verbose; + + if (arg < argc) { + fprintf(stderr, "unexpected argument: %s\n", argv[arg]); + print_usage(prog); + free_model_specs(&model_specs); + return 1; + } + if (model_specs.n == 0) { + fprintf(stderr, "at least one --model LABEL=DIR is required\n"); + print_usage(prog); + free_model_specs(&model_specs); + return 1; + } + char err[256]; + if (ffwd_init(&opts, err, sizeof(err)) != 0) { + fprintf(stderr, "%s\n", err); + free_model_specs(&model_specs); + return 1; + } + + ffwd_server_config_t scfg = { + .models = model_specs.v, + .n_models = model_specs.n, + .host = host, + .port = port, + .batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE, + .max_batch_tokens = max_batch_tokens, + .batch_wait_us = batch_wait_us, + .gpu_quantize_bits = opts.gpu_quant_bits, + .gpu_quantize_group_size = opts.gpu_quant_group_size, + .memory_utilization = opts.memory_utilization, + .api_key = api_key, + }; + int rc = ffwd_run_server(&scfg); + free_model_specs(&model_specs); + return rc; +} diff --git a/tools/server/models.c b/tools/server/models.c new file mode 100644 index 0000000..8fdc590 --- /dev/null +++ b/tools/server/models.c @@ -0,0 +1,342 @@ +#include "server_internal.h" +#include "util.h" + +#include <errno.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +/* ======================================================================== + * Model load/lifecycle, and tokenize/inference dispatch + * + * Each loaded model owns resolved serving metadata for the operator-chosen + * label. Transformer behavior comes from the model files through ffwd_config; + * server-only facts absent from those files, such as output API family and + * Matryoshka floor, come from the load spec. The tokenize_* and model_ffwd_* + * helpers are the single dispatch point the request handlers call, hiding the + * compiled backend choice behind one signature. + * ======================================================================== */ + +loaded_model *loaded_model_for_label(http_server *s, const char *label) { + if (!label) + return NULL; + for (int i = 0; i < s->n_models; i++) { + if (s->models[i].info && !strcmp(label, s->models[i].info->id)) + return &s->models[i]; + } + return NULL; +} + +static model_kind model_kind_from_public(ffwd_server_model_kind_t kind) { + switch (kind) { + case FFWD_SERVER_MODEL_CONTEXTUAL: + return MODEL_KIND_CONTEXTUAL; + case FFWD_SERVER_MODEL_LATE: + return MODEL_KIND_LATE; + case FFWD_SERVER_MODEL_STANDARD: + default: + return MODEL_KIND_STANDARD; + } +} + +static embedding_api_t embedding_api_from_public(ffwd_server_embedding_api_t api) { + return api == FFWD_SERVER_API_PERPLEXITY ? FFWD_API_PERPLEXITY : FFWD_API_OPENAI; +} + +static int read_optional_text_file(const char *model_dir, const char *relative, char **out) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative); + if (n < 0 || (size_t)n >= sizeof(path)) { + server_log("ffwd-server: model path too long"); + return -1; + } + FILE *f = fopen(path, "rb"); + if (!f && errno == ENOENT) + return 0; + if (!f) { + server_log("ffwd-server: failed to open %s", path); + return -1; + } + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf) { + fclose(f); + return -1; + } + if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + free(buf); + fclose(f); + return -1; + } + fclose(f); + buf[sz] = '\0'; + *out = buf; + return 1; +} + +static int read_query_instruct_from_files(const char *model_dir, char **out) { + char *buf = NULL; + int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf); + if (rc <= 0) + return rc; + cJSON *root = cJSON_Parse(buf); + free(buf); + if (!root) { + server_log("ffwd-server: invalid config_sentence_transformers.json: %s", model_dir); + return -1; + } + cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts"); + cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL; + if (query && !cJSON_IsString(query)) { + cJSON_Delete(root); + server_log("ffwd-server: prompts.query must be a string: %s", model_dir); + return -1; + } + if (query && query->valuestring && query->valuestring[0]) { + *out = xstrdup(query->valuestring); + cJSON_Delete(root); + return *out ? 1 : -1; + } + cJSON_Delete(root); + return 0; +} + +static int init_model_info(model_info **out, const ffwd_server_model_spec_t *spec) { + model_info *info = xcalloc(1, sizeof(*info)); + info->id = xstrdup(spec->id); + info->kind = model_kind_from_public(spec->kind); + info->api = embedding_api_from_public(spec->api); + info->min_dim = spec->min_dim; + if (spec->query_instruct) { + info->query_instruct = xstrdup(spec->query_instruct); + } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) { + free(info->id); + free(info); + return -1; + } + if (!info->id || (spec->query_instruct && !info->query_instruct)) { + free(info->query_instruct); + free(info->id); + free(info); + return -1; + } + *out = info; + return 0; +} + +static void free_model_info(model_info *info) { + if (!info) + return; + free(info->id); + free(info->query_instruct); + free(info); +} + +void free_token_bufs(token_buf *t, int n) { + if (!t) + return; + for (int i = 0; i < n; i++) + free(t[i].ids); +} + +void embedding_request_free(embedding_request *r) { + if (!r) + return; + free_token_bufs(r->tokens, r->n_inputs); + free(r->tokens); + free(r->inputs); + if (r->root) + cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +void contextual_request_free(contextual_request *r) { + if (!r) + return; + for (int i = 0; i < r->n_docs; i++) { + free(r->docs[i].ids); + free(r->docs[i].spans); + } + free(r->docs); + if (r->root) + cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +void late_tokens_free(late_tokens *t) { + if (!t) + return; + free(t->ids); + free(t->keep); + memset(t, 0, sizeof(*t)); +} + +void rerank_request_free(rerank_request *r) { + if (!r) + return; + late_tokens_free(&r->query); + for (int i = 0; i < r->n_documents; i++) + late_tokens_free(&r->documents[i]); + free(r->documents); + if (r->root) + cJSON_Delete(r->root); + memset(r, 0, sizeof(*r)); +} + +/* Tokenization itself lives in libffwd (ffwd_tok). These thin wrappers only + * add the server's per-stage timing onto the job; the request handlers call + * them unchanged. */ +int tokenize_input( + loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) { + uint64_t t0 = nstime(); + out->ids = ffwd_tokenize(m->tok, text, query_instruct, &out->n_tokens); + if (j) + j->tokenize_ns += nstime() - t0; + if (!out->ids) { + memset(out, 0, sizeof(*out)); + return -1; + } + return 0; +} + +int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) { + return tokenize_input(m, j, text, NULL, out); +} + +int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) { + uint64_t t0 = nstime(); + int rc = ffwd_tokenize_late(m->tok, text, is_query, out); + if (j) + j->tokenize_ns += nstime() - t0; + return rc; +} + +int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out) { + return ffwd_encode_batch(m->backend, inputs, batch, out); +} + +int model_ffwd_spans_batch(loaded_model *m, + const ffwd_context_input_t *inputs, + int batch, + float *out) { + return ffwd_encode_spans_batch(m->backend, inputs, batch, out); +} + +static int model_uses_dense_batches(const loaded_model *m) { + return ffwd_uses_dense_batches(m->backend); +} + +/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX + * pads each dense row to the longest input in the chunk. */ +int inference_batch_accepts_input( + const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) { + if (batch == 0) + return 1; + if (model_uses_dense_batches(m)) + return next_tokens <= max_batch_tokens / (batch + 1); + return next_tokens <= max_batch_tokens - packed_tokens; +} + +int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim) { + if (!m || !m->info || !config || config->hidden_size <= 0) { + if (m && m->info) + server_log("ffwd-server: model %s has invalid config", m->info->id); + return -1; + } + if (m->info->kind == MODEL_KIND_LATE) { + if (token_dim <= 0) { + server_log("ffwd-server: late model %s is missing token projection metadata", + m->info->id); + return -1; + } + } else if (token_dim != 0) { + server_log("ffwd-server: non-late model %s unexpectedly has token projection metadata", + m->info->id); + return -1; + } + + int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size; + if (min_dim <= 0 || min_dim > config->hidden_size) { + server_log("ffwd-server: model %s has invalid min_dim=%d for dim=%d", m->info->id, min_dim, + config->hidden_size); + return -1; + } + m->info->dim = config->hidden_size; + m->info->min_dim = min_dim; + m->info->token_dim = token_dim; + m->info->attention_mode = config->attention_mode; + m->info->pooling_mode = config->pooling_mode; + m->info->normalize_embeddings = config->normalize_embeddings; + m->renormalize_truncated = + config->normalize_embeddings && config->pooling_mode == FFWD_POOL_LAST_TOKEN; + return 0; +} + +/* Apply the loaded model's config to its serving metadata and validate it. CPU + * runs this in load_one_model; a GPU build runs it on the inference thread once + * ffwd_activate has created the context. */ +int finalize_loaded_model(loaded_model *m) { + const ffwd_config_t *config = ffwd_config(m->backend); + int token_dim = ffwd_token_dim(m->backend); + ffwd_tok_set_append_terminal(m->tok, config->append_terminal_token); + return configure_loaded_model(m, config, token_dim); +} + +int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec) { + /* Grow the model list by one entry. */ + loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models)); + if (!nm) + return -1; + s->models = nm; + loaded_model *m = &s->models[s->n_models]; + memset(m, 0, sizeof(*m)); + s->n_models++; + + if (init_model_info(&m->info, spec) != 0) + return -1; + const char *path = spec->path; + m->path = xstrdup(path); + char err[256]; + /* The tokenizer (file probe, special-token resolution, [CLS]/[SEP] vs Qwen + * terminal-token layout) is owned by libffwd now. */ + m->tok = ffwd_tok_open(path, m->info->kind == MODEL_KIND_LATE, err, sizeof(err)); + if (!m->tok) { + server_log("ffwd-server: %s: %s", path, err); + return -1; + } + + m->backend = + ffwd_open(path, m->info->kind == MODEL_KIND_LATE, &s->backend_opts, err, sizeof(err)); + if (!m->backend) { + server_log("ffwd-server: %s: %s", path, err); + return -1; + } + /* CPU loads the model in open, so its config is ready to finalize now. A GPU + * build defers the load - and this finalize - to the inference thread. */ + if (ffwd_config(m->backend)) + return finalize_loaded_model(m); + return 0; +} + +void free_models(http_server *s) { + for (int i = 0; i < s->n_models; i++) { + loaded_model *m = &s->models[i]; + free_model_info(m->info); + free(m->path); + ffwd_tok_free(m->tok); + ffwd_free(m->backend); + } + free(s->models); + s->models = NULL; + s->n_models = 0; +} diff --git a/src/sbuf.c b/tools/server/sbuf.c index f286d8c..b341c31 100644 --- a/src/sbuf.c +++ b/tools/server/sbuf.c @@ -1,5 +1,5 @@ #include "sbuf.h" -#include "server_util.h" +#include "util.h" #include <stdarg.h> #include <stdint.h> diff --git a/src/sbuf.h b/tools/server/sbuf.h index 0c8929e..3116090 100644 --- a/src/sbuf.h +++ b/tools/server/sbuf.h @@ -1,5 +1,5 @@ -#ifndef EMBED_SBUF_H -#define EMBED_SBUF_H +#ifndef FFWD_SBUF_H +#define FFWD_SBUF_H #include <stddef.h> @@ -21,4 +21,4 @@ void sbuf_clear(sbuf *b); void sbuf_printf(sbuf *b, const char *fmt, ...) __attribute__((format(printf, 2, 3))); void sbuf_free(sbuf *b); -#endif /* EMBED_SBUF_H */ +#endif /* FFWD_SBUF_H */ diff --git a/src/server_schedule.c b/tools/server/schedule.c index 1daf8dd..c13e0cf 100644 --- a/src/server_schedule.c +++ b/tools/server/schedule.c @@ -1,5 +1,5 @@ #include "server_internal.h" -#include "server_util.h" +#include "util.h" #include <errno.h> #include <pthread.h> @@ -12,7 +12,7 @@ /* ======================================================================== * Job queue, micro-batching, and completion * - * dispatch_request (server_http.c) enqueues one job per framed request; the + * dispatch_request (http.c) enqueues one job per framed request; the * inference worker thread (worker_main, server.c) drains them with * collect_job_batch (a short first-arrival wait groups concurrent arrivals) and * runs process_job_group, which routes each job to its endpoint handler and diff --git a/tools/server/server.c b/tools/server/server.c new file mode 100644 index 0000000..abff2c2 --- /dev/null +++ b/tools/server/server.c @@ -0,0 +1,414 @@ +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif + +#include "server.h" +#include "server_internal.h" +#include "util.h" +#include "sbuf.h" +#include "ffwd.h" +#include "ae.h" +#include "anet.h" + +#include <arpa/inet.h> +#include <ctype.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <netinet/in.h> +#include <pthread.h> +#include <signal.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <strings.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +#ifdef __APPLE__ +# include <sys/sysctl.h> +#endif + +#include <cjson/cJSON.h> + +static volatile sig_atomic_t g_stop_requested = 0; +static volatile sig_atomic_t g_signal_wfd = -1; + +static void stop_signal_handler(int sig) { + (void)sig; + if (g_stop_requested) + _exit(130); + int saved = errno; + g_stop_requested = 1; + int fd = (int)g_signal_wfd; + if (fd >= 0) { + const unsigned char byte = 1; + if (write(fd, &byte, 1) < 0) { + /* best-effort stop wakeup; a failed self-pipe write is ignored */ + } + } + errno = saved; +} + +static bool auth_ok(http_server *s, const char *auth) { + if (!s->api_key || !s->api_key[0]) + return true; + if (!auth) + return false; + const char prefix[] = "Bearer "; + return !strncmp(auth, prefix, sizeof(prefix) - 1) && + !strcmp(auth + sizeof(prefix) - 1, s->api_key); +} + +void dispatch_request(client *c) { + aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE); + + if (!strcmp(c->req.method, "OPTIONS")) { + append_http_response(c, 204, NULL, "", 0); + if (queue_write(c) != AE_OK) + close_client(c); + return; + } + + if (!route_is_inference(c->req.method, c->req.path)) { + size_t len; + char *body = json_error_body("unknown endpoint", "invalid_request_error", &len); + append_http_response(c, 404, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) + close_client(c); + return; + } + + if (!auth_ok(c->srv, c->req.auth)) { + size_t len; + char *body = json_error_body("missing or invalid bearer token", "authentication_error", &len); + append_http_response(c, 401, "application/json", body, len); + free(body); + if (queue_write(c) != AE_OK) + close_client(c); + return; + } + + job *j = xcalloc(1, sizeof(*j)); + j->srv = c->srv; + j->c = c; + snprintf(j->method, sizeof(j->method), "%s", c->req.method); + snprintf(j->path, sizeof(j->path), "%s", c->req.path); + j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len); + j->body_len = c->req.body_len; + j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL; + j->created_ns = nstime(); + client_incref(c); + enqueue_job(j); +} + +static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)loop; + (void)mask; + http_server *s = clientData; + char cip[64]; + int cport = 0; + for (;;) { + int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; + if (errno == EINTR) + continue; + return; + } + set_nonblock(cfd); + client *c = xcalloc(1, sizeof(*c)); + c->srv = s; + c->fd = cfd; + c->refcount = 1; + c->last_active_ms = mstime(); + c->linked = 1; + c->next = s->clients; + if (s->clients) + s->clients->prev = c; + s->clients = c; + s->n_clients++; + if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK) + close_client(c); + } +} + +static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) { + (void)loop; + (void)id; + http_server *s = clientData; + uint64_t now = mstime(); + client *c = s->clients; + while (c) { + client *next = c->next; + /* refcount > 1 means a request from this client is queued or being + * processed by the worker; long batch queues (e.g. 4B long-document + * batches) must not be reaped as idle. */ + if (!c->cancelled && c->refcount <= 1 && + now - c->last_active_ms > FFWD_HTTP_CLIENT_TIMEOUT_MS) + close_client(c); + c = next; + } + return FFWD_HTTP_SWEEP_MS; +} + +static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) { + (void)fd; + (void)mask; + http_server *s = clientData; + unsigned char tmp[128]; + while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) { + } + aeStop(loop); +} + +static void *worker_main(void *arg) { + http_server *s = arg; + /* Create each model's context on this thread (a GPU build defers the load + * here so the context lives on the thread that uses it) and finalize the + * config a GPU build could not finalize at open time. CPU is loaded and + * finalized already, marked by info->dim being set. */ + int rc = 0; + char load_err[256]; + for (int i = 0; i < s->n_models; i++) { + loaded_model *m = &s->models[i]; + if (ffwd_activate(m->backend, load_err, sizeof(load_err)) != 0) { + server_log("ffwd-server: failed to load model on worker: %s: %s", m->path, load_err); + rc = 1; + break; + } + if (m->info->dim == 0 && finalize_loaded_model(m) != 0) { + rc = 1; + break; + } + } + pthread_mutex_lock(&s->mu); + s->worker_init_rc = rc; + s->worker_ready = 1; + if (rc) + s->stopping = 1; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + if (rc) { + for (int i = 0; i < s->n_models; i++) + ffwd_worker_free(s->models[i].backend); + return NULL; + } + + for (;;) { + job *jobs[FFWD_SERVER_MICROBATCH_MAX_JOBS]; + int n_jobs = collect_job_batch(s, jobs, FFWD_SERVER_MICROBATCH_MAX_JOBS); + if (n_jobs == 0) + break; + process_job_group(s, jobs, n_jobs); + } + for (int i = 0; i < s->n_models; i++) + ffwd_worker_free(s->models[i].backend); + return NULL; +} + +static int listen_on(const char *host, int port) { + char portbuf[32]; + snprintf(portbuf, sizeof(portbuf), "%d", port); + char err[ANET_ERR_LEN]; + int fd = anetTcpServer(err, port, (char *)host, FFWD_HTTP_BACKLOG); + (void)portbuf; + if (fd == ANET_ERR) { + server_log("ffwd-server: listen failed on %s:%d: %s", host, port, err); + return -1; + } + set_nonblock(fd); + return fd; +} + +static int validate_model_specs(const ffwd_server_config_t *cfg) { + for (int i = 0; i < cfg->n_models; i++) { + const ffwd_server_model_spec_t *spec = &cfg->models[i]; + if (!spec->id || !spec->id[0]) { + fprintf(stderr, "ffwd-server: model label must not be empty\n"); + return -1; + } + if (!spec->path || !spec->path[0]) { + fprintf(stderr, "ffwd-server: model %s has an empty path\n", spec->id); + return -1; + } + if (spec->kind < FFWD_SERVER_MODEL_STANDARD || spec->kind > FFWD_SERVER_MODEL_LATE) { + fprintf(stderr, "ffwd-server: model %s has an invalid kind\n", spec->id); + return -1; + } + if (spec->api < FFWD_SERVER_API_DEFAULT || spec->api > FFWD_SERVER_API_PERPLEXITY) { + fprintf(stderr, "ffwd-server: model %s has an invalid api\n", spec->id); + return -1; + } + if (spec->min_dim < 0) { + fprintf(stderr, "ffwd-server: model %s has an invalid min_dim\n", spec->id); + return -1; + } + for (int j = 0; j < i; j++) { + if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) { + fprintf(stderr, "ffwd-server: duplicate model label: %s\n", spec->id); + return -1; + } + } + } + return 0; +} + +/* Hold the model paths in one array so the backend preflight can size the set + * against host memory in a single pass. */ +static int run_memory_preflight(const ffwd_server_config_t *cfg, const ffwd_options_t *opts) { + const char **paths = (const char **)malloc((size_t)cfg->n_models * sizeof(*paths)); + if (!paths) + return -1; + for (int i = 0; i < cfg->n_models; i++) + paths[i] = cfg->models[i].path; + char err[256] = {0}; + int rc = ffwd_preflight(paths, cfg->n_models, opts, err, sizeof(err)); + free(paths); + if (rc != 0 && err[0]) + server_log("ffwd-server: %s", err); + return rc; +} + +int ffwd_run_server(const ffwd_server_config_t *cfg) { + if (!cfg || !cfg->models || cfg->n_models <= 0) { + fprintf(stderr, "ffwd-server: at least one model label and path is required\n"); + return 1; + } + ffwd_options_t backend_opts = { + .gpu_quant_bits = cfg->gpu_quantize_bits, + .gpu_quant_group_size = cfg->gpu_quantize_group_size > 0 ? cfg->gpu_quantize_group_size : 64, + .memory_utilization = cfg->memory_utilization, + }; + if (validate_model_specs(cfg) != 0 || run_memory_preflight(cfg, &backend_opts) != 0) + return 1; + + http_server s; + memset(&s, 0, sizeof(s)); + s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1"; + s.port = cfg->port > 0 ? cfg->port : 8000; + s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE; + s.max_batch_tokens = + cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS; + s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us(); + s.backend_opts = backend_opts; + if (cfg->api_key && cfg->api_key[0]) + s.api_key = xstrdup(cfg->api_key); + else { + const char *env_key = getenv("FFWD_API_KEY"); + if (env_key && env_key[0]) + s.api_key = xstrdup(env_key); + } + + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + for (int i = 0; i < cfg->n_models; i++) { + if (load_one_model(&s, &cfg->models[i]) != 0) { + free_models(&s); + return 1; + } + } + + if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) { + perror("pipe"); + free_models(&s); + return 1; + } + set_nonblock(s.completion_pipe[0]); + set_nonblock(s.completion_pipe[1]); + set_nonblock(s.signal_pipe[0]); + set_nonblock(s.signal_pipe[1]); + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = stop_signal_handler; + sigemptyset(&sa.sa_mask); + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + signal(SIGPIPE, SIG_IGN); + g_signal_wfd = s.signal_pipe[1]; + + s.loop = aeCreateEventLoop(FFWD_HTTP_SETSIZE); + if (!s.loop) { + fprintf(stderr, "ffwd-server: failed to create event loop\n"); + free_models(&s); + return 1; + } + + if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) { + fprintf(stderr, "ffwd-server: failed to start worker\n"); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + pthread_mutex_lock(&s.mu); + while (!s.worker_ready) + pthread_cond_wait(&s.cv, &s.mu); + int worker_init_rc = s.worker_init_rc; + pthread_mutex_unlock(&s.mu); + if (worker_init_rc) { + pthread_join(s.worker, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 1; + } + + s.listen_fd = listen_on(s.host, s.port); + if (s.listen_fd < 0) { + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + aeDeleteEventLoop(s.loop); + free_models(&s); + return 1; + } + + aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s); + aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s); + aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s); + aeCreateTimeEvent(s.loop, FFWD_HTTP_SWEEP_MS, timeout_cb, &s, NULL); + + server_log("ffwd-server: listening on http://%s:%d", s.host, s.port); + aeMain(s.loop); + + server_log("ffwd-server: shutdown requested"); + if (s.listen_fd >= 0) + close(s.listen_fd); + client *c = s.clients; + while (c) { + client *next = c->next; + close_client(c); + c = next; + } + + pthread_mutex_lock(&s.mu); + s.stopping = 1; + pthread_cond_signal(&s.cv); + pthread_mutex_unlock(&s.mu); + pthread_join(s.worker, NULL); + + aeDeleteEventLoop(s.loop); + close(s.completion_pipe[0]); + close(s.completion_pipe[1]); + close(s.signal_pipe[0]); + close(s.signal_pipe[1]); + free_models(&s); + free(s.api_key); + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); + return 0; +} diff --git a/tools/server/server.h b/tools/server/server.h new file mode 100644 index 0000000..b8ec7e4 --- /dev/null +++ b/tools/server/server.h @@ -0,0 +1,57 @@ +#ifndef FFWD_SERVER_H +#define FFWD_SERVER_H + +/* Public-API export annotation. The libraries are built with + * -fvisibility=hidden, so only declarations carrying FFWD_API are exported + * from the shared library; everything else stays internal. */ +#ifndef FFWD_API +# if defined(__GNUC__) +# define FFWD_API __attribute__((visibility("default"))) +# else +# define FFWD_API +# endif +#endif + +#include <stdbool.h> + +typedef enum { + FFWD_SERVER_MODEL_STANDARD = 0, + FFWD_SERVER_MODEL_CONTEXTUAL = 1, + FFWD_SERVER_MODEL_LATE = 2, +} ffwd_server_model_kind_t; + +typedef enum { + FFWD_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */ + FFWD_SERVER_API_OPENAI = 1, + FFWD_SERVER_API_PERPLEXITY = 2, +} ffwd_server_embedding_api_t; + +typedef struct { + const char *id; /* served label; requests match this string exactly */ + const char *path; + ffwd_server_model_kind_t kind; + ffwd_server_embedding_api_t api; + int min_dim; /* 0 means the loaded model's full hidden size */ + const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */ +} ffwd_server_model_spec_t; + +typedef struct { + const ffwd_server_model_spec_t *models; + int n_models; + const char *host; + int port; + int batch_size; + int max_batch_tokens; + int batch_wait_us; + /* GPU build tuning; ignored by backends that do not use it. */ + int gpu_quantize_bits; + int gpu_quantize_group_size; + double memory_utilization; /* fraction of physical memory the GPU preflight + may plan for; 0 = default 0.90, above 1.0 + allows overcommit */ + const char *api_key; +} ffwd_server_config_t; + +FFWD_API int ffwd_run_server(const ffwd_server_config_t *cfg); + +#endif diff --git a/src/server_internal.h b/tools/server/server_internal.h index 1f106ca..91b51c5 100644 --- a/src/server_internal.h +++ b/tools/server/server_internal.h @@ -1,27 +1,17 @@ -#ifndef EMBED_SERVER_INTERNAL_H -#define EMBED_SERVER_INTERNAL_H +#ifndef FFWD_SERVER_INTERNAL_H +#define FFWD_SERVER_INTERNAL_H -/* Declarations shared across the embed-server translation units (HTTP framing, +/* Declarations shared across the ffwd-server translation units (HTTP framing, * JSON validation, output encoding, scheduling, model lifecycle, endpoint * handlers). The server is split by concern but compiles as several objects * linked into one binary; this header is what they share. Not a public API - * the CLI and library do not include it. */ -#include "embed.h" +#include "ffwd.h" #include "server.h" #include "sbuf.h" -#include "tokenizer_bpe.h" -#include "tokenizer_sentencepiece.h" -#include "tokenizer_wordpiece.h" -#ifdef USE_MLX -#include "mlx.h" -#endif -#ifdef USE_CUDA -#include "cuda.h" -#endif - -#include "deps/ae/ae.h" +#include "ae.h" #include <cjson/cJSON.h> #include <pthread.h> @@ -29,45 +19,43 @@ #include <stddef.h> #include <stdint.h> -#define EMBED_HTTP_IO_BUF 8192 -#define EMBED_HTTP_MAX_HEADER (64u * 1024u) -#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u) -#define EMBED_HTTP_MAX_PATH 255u -#define EMBED_HTTP_BACKLOG 128 -#define EMBED_HTTP_SETSIZE 10240 -#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000 -#define EMBED_HTTP_SWEEP_MS 1000 +#define FFWD_HTTP_IO_BUF 8192 +#define FFWD_HTTP_MAX_HEADER (64u * 1024u) +#define FFWD_HTTP_MAX_BODY (64u * 1024u * 1024u) +#define FFWD_HTTP_MAX_PATH 255u +#define FFWD_HTTP_BACKLOG 128 +#define FFWD_HTTP_SETSIZE 10240 +#define FFWD_HTTP_CLIENT_TIMEOUT_MS 30000 +#define FFWD_HTTP_SWEEP_MS 1000 -#define EMBED_API_MAX_STANDARD_INPUTS 512 -#define EMBED_API_MAX_CONTEXT_DOCS 512 -#define EMBED_API_MAX_CONTEXT_CHUNKS 16000 -#define EMBED_API_MAX_RERANK_DOCUMENTS 1000 -#define EMBED_API_MAX_ITEM_TOKENS 32768 -#define EMBED_API_MAX_TOTAL_TOKENS 120000 -#define EMBED_LATE_QUERY_TOKENS 32 -#define EMBED_LATE_MASK_TOKEN_ID 151642 -#define EMBED_LATE_QUERY_PREFIX_ID 151669 -#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670 +#define FFWD_API_MAX_STANDARD_INPUTS 512 +#define FFWD_API_MAX_CONTEXT_DOCS 512 +#define FFWD_API_MAX_CONTEXT_CHUNKS 16000 +#define FFWD_API_MAX_RERANK_DOCUMENTS 1000 +#define FFWD_API_MAX_ITEM_TOKENS 32768 +#define FFWD_API_MAX_TOTAL_TOKENS 120000 +/* FFWD_LATE_* and FFWD_CONTEXT_SEPARATOR_TOKEN_ID now live in ffwd.h, owned + * by the tokenizer (ffwd_tok). */ /* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent * short-request throughput over 8 with no long-document penalty, while 128 * was no faster and inflated long-document tail latency. */ -#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32 -#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 +#define FFWD_SERVER_DEFAULT_BATCH_SIZE 32 +#define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384 /* Microseconds the worker waits for more requests before dispatching a batch. * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU * busy; MLX and CPU dispatch immediately, where waiting only adds latency. * Override with --batch-wait-us. */ -#define EMBED_SERVER_BATCH_WAIT_US 0 -#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000 -#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128 -#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90 -#define EMBED_MLX_RESIDENT_MULTIPLIER 2 +#define FFWD_SERVER_BATCH_WAIT_US 0 +#define FFWD_SERVER_CUDA_BATCH_WAIT_US 1000 +#define FFWD_SERVER_MICROBATCH_MAX_JOBS 128 +#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90 +#define FFWD_MLX_RESIDENT_MULTIPLIER 2 typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind; typedef enum { - EMBED_API_OPENAI = 0, - EMBED_API_PERPLEXITY = 1, + FFWD_API_OPENAI = 0, + FFWD_API_PERPLEXITY = 1, } embedding_api_t; typedef struct { @@ -76,8 +64,8 @@ typedef struct { int dim; int min_dim; int token_dim; - embed_attention_mode_t attention_mode; - embed_pooling_mode_t pooling_mode; + ffwd_attention_mode_t attention_mode; + ffwd_pooling_mode_t pooling_mode; int normalize_embeddings; embedding_api_t api; char *query_instruct; @@ -86,38 +74,9 @@ typedef struct { typedef struct { model_info *info; char *path; - embed_tokenizer_t *tok; - embed_tokenizer_workspace_t *tok_ws; - /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model. - * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the - * Qwen byte-level BPE fields above stay NULL. */ - wordpiece_tokenizer_t *wp_tok; - wordpiece_workspace_t *wp_tok_ws; - sentencepiece_tokenizer_t *sp_tok; - sentencepiece_workspace_t *sp_tok_ws; - int cls_id; - int sep_id; - int context_separator_id; - int late_mask_id; - int late_query_prefix_id; - int late_document_prefix_id; - int late_skip_ids[64]; - int n_late_skip_ids; - int append_terminal_token; - int terminal_token_id; - int renormalize_truncated; - embed_model_t *cpu_model; - embed_workspace_t *cpu_ws; - embed_late_model_t *cpu_late_model; - embed_late_workspace_t *cpu_late_ws; -#ifdef USE_MLX - embed_mlx_ctx_t *mlx_ctx; - embed_mlx_late_ctx_t *mlx_late_ctx; -#endif -#ifdef USE_CUDA - embed_cuda_ctx_t *cuda_ctx; - embed_cuda_late_ctx_t *cuda_late_ctx; -#endif + ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */ + ffwd_t *backend; /* compute handle */ + int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */ } loaded_model; typedef struct client client; @@ -136,10 +95,7 @@ typedef struct { char *api_key; loaded_model *models; int n_models; - int use_mlx; - int use_cuda; - int mlx_quantize_bits; - int mlx_quantize_group_size; + ffwd_options_t backend_opts; pthread_t worker; pthread_mutex_t mu; @@ -158,7 +114,7 @@ typedef struct { typedef struct { char method[8]; - char path[EMBED_HTTP_MAX_PATH + 1]; + char path[FFWD_HTTP_MAX_PATH + 1]; char version[16]; char *body; size_t body_len; @@ -188,7 +144,7 @@ struct job { http_server *srv; client *c; char method[8]; - char path[EMBED_HTTP_MAX_PATH + 1]; + char path[FFWD_HTTP_MAX_PATH + 1]; char *body; size_t body_len; char *auth; @@ -222,7 +178,7 @@ typedef struct { const char *encoding; int n_inputs; token_buf *tokens; - embed_input_t *inputs; + ffwd_input_t *inputs; int total_tokens; int ready; } embedding_request; @@ -230,7 +186,7 @@ typedef struct { typedef struct { int *ids; int n_tokens; - embed_span_t *spans; + ffwd_span_t *spans; int n_spans; } contextual_doc; @@ -247,12 +203,8 @@ typedef struct { int ready; } contextual_request; -typedef struct { - int *ids; - int n_tokens; - int *keep; - int n_keep; -} late_tokens; +/* The server's late_tokens is exactly what ffwd_tokenize_late fills. */ +typedef ffwd_late_tokens_t late_tokens; typedef struct { job *j; @@ -278,8 +230,7 @@ cJSON *parse_json_body(job *j, cJSON *detail); bool cjson_is_integer(cJSON *item); const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api); int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct); -int dimensions_from_root( - cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding); +int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding); /* ---- server_http.c: connection lifecycle, header parse, response framing ---- */ int set_nonblock(int fd); @@ -317,19 +268,21 @@ void embedding_request_free(embedding_request *r); void contextual_request_free(contextual_request *r); void late_tokens_free(late_tokens *t); void rerank_request_free(rerank_request *r); +/* Thin timing wrappers over libffwd's ffwd_tokenize / ffwd_tokenize_late. */ int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out); int tokenize_input( loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out); int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out); -int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out); -int model_embed_spans_batch(loaded_model *m, - const embed_context_input_t *inputs, - int batch, - float *out); +int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out); +int model_ffwd_spans_batch(loaded_model *m, + const ffwd_context_input_t *inputs, + int batch, + float *out); int inference_batch_accepts_input( const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens); -int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim); -int load_one_model(http_server *s, const embed_server_model_spec_t *spec); +int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim); +int finalize_loaded_model(loaded_model *m); +int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec); void free_models(http_server *s); /* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */ @@ -358,4 +311,4 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs); void process_job_group(http_server *s, job **jobs, int n_jobs); void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask); -#endif /* EMBED_SERVER_INTERNAL_H */ +#endif /* FFWD_SERVER_INTERNAL_H */ diff --git a/tools/server/test_base64.c b/tools/server/test_base64.c new file mode 100644 index 0000000..3519c0c --- /dev/null +++ b/tools/server/test_base64.c @@ -0,0 +1,51 @@ +/* test_base64.c - base64_encode against RFC 4648 vectors and binary inputs. */ + +#include "base64.h" +#include "test_util.h" + +#include <stdlib.h> +#include <string.h> + +static void test_base64_rfc4648(void) { + /* RFC 4648 test vectors. */ + static const struct { + const char *in, *out; + } cases[] = { + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char *got = base64_encode((const unsigned char *)cases[i].in, strlen(cases[i].in)); + TEST_ASSERT(strcmp(got, cases[i].out) == 0); + free(got); + } +} + +static void test_base64_binary_vectors(void) { + static const struct { + unsigned char bytes[5]; + size_t n; + const char *out; + } cases[] = { + {{0xff}, 1, "/w=="}, + {{0xff, 0xee}, 2, "/+4="}, + {{0xff, 0xee, 0xdd}, 3, "/+7d"}, + {{0x00, 0x01, 0x02, 0x03, 0x04}, 5, "AAECAwQ="}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + char *got = base64_encode(cases[i].bytes, cases[i].n); + TEST_ASSERT(strcmp(got, cases[i].out) == 0); + free(got); + } +} + +int main(void) { + test_base64_rfc4648(); + test_base64_binary_vectors(); + return TEST_REPORT("base64"); +} diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c new file mode 100644 index 0000000..a7db9c4 --- /dev/null +++ b/tools/server/test_encode.c @@ -0,0 +1,90 @@ +/* test_encode.c - embedding quantization and the output encodings + * (base64_int8, base64_binary, base64 float32) plus float rendering. */ + +#include "base64.h" +#include "server_internal.h" +#include "test_util.h" + +#include <stdlib.h> +#include <string.h> + +static void test_quantize_int8_tanh(void) { + TEST_ASSERT(quantize_int8_tanh(0.0f) == 0); + /* tanh saturates to +-1, scaled by 127. */ + TEST_ASSERT(quantize_int8_tanh(100.0f) == 127); + TEST_ASSERT(quantize_int8_tanh(-100.0f) == -127); + /* Symmetry. */ + TEST_ASSERT(quantize_int8_tanh(0.5f) == -quantize_int8_tanh(-0.5f)); + /* round(tanh(0.5) * 127) = round(58.69) = 59. */ + TEST_ASSERT(quantize_int8_tanh(0.5f) == 59); +} + +static void test_encode_embedding_int8(void) { + float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f}; + signed char expect[4] = {0, 59, -59, 127}; + char *got = encode_embedding(emb, 4, "base64_int8"); + char *want = base64_encode((const unsigned char *)expect, 4); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); +} + +static void test_encode_embedding_binary(void) { + /* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */ + float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f}; + unsigned char expect = 0x85; /* bits 0, 2, 7 set */ + char *got = encode_embedding(emb, 8, "base64_binary"); + char *want = base64_encode(&expect, 1); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); + + /* Zero counts as non-negative (sign bit set). */ + float zeros[8] = {0}; + unsigned char all = 0xff; + got = encode_embedding(zeros, 8, "base64_binary"); + want = base64_encode(&all, 1); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); +} + +static void test_encode_embedding_base64_float32(void) { + /* OpenAI/DashScope "base64" is base64 of the raw little-endian float32 + * vector - lossless, unlike the int8 formats. */ + float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f}; + /* Reference bytes via memcpy: a float->byte reinterpret cast confuses the + * static analyzer's uninitialized-read model. */ + unsigned char bytes[sizeof emb]; + memcpy(bytes, emb, sizeof emb); + char *want = base64_encode(bytes, sizeof emb); + char *got = encode_embedding(emb, 4, "base64"); + TEST_ASSERT(strcmp(got, want) == 0); + free(got); + free(want); +} + +static void test_append_embedding_value_float(void) { + float emb[1] = {0.5f}; + + /* OpenAI-compatible models render the true float32 value. */ + sbuf b = {0}; + append_embedding_value(&b, 0, emb, 1, "float", FFWD_API_OPENAI); + TEST_ASSERT(strstr(b.ptr, "\"embedding\":[0.5]") != NULL); + sbuf_free(&b); + + /* Perplexity-compatible models render the int8-decoded view. */ + sbuf b2 = {0}; + append_embedding_value(&b2, 0, emb, 1, "float", FFWD_API_PERPLEXITY); + TEST_ASSERT(strstr(b2.ptr, "0.4609375") != NULL); + sbuf_free(&b2); +} + +int main(void) { + test_quantize_int8_tanh(); + test_encode_embedding_int8(); + test_encode_embedding_binary(); + test_encode_embedding_base64_float32(); + test_append_embedding_value_float(); + return TEST_REPORT("encode"); +} diff --git a/tools/server/test_json.c b/tools/server/test_json.c new file mode 100644 index 0000000..76a0f45 --- /dev/null +++ b/tools/server/test_json.c @@ -0,0 +1,143 @@ +/* test_json.c - request validation, JSON error bodies, and 422 detail + * handling (encoding_format, text_type, error escaping, job_set_422). */ + +#include "server_internal.h" +#include "test_util.h" +#include "util.h" /* xstrdup */ + +#include <stdlib.h> +#include <string.h> + +static const char *TEST_QWEN3_QUERY_INSTRUCT = + "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:"; + +static void test_json_error_body_escaping(void) { + size_t len = 0; + char *body = json_error_body("line\nquote\" tab\t ctrl\001 slash\\", "bad\ntype", &len); + TEST_ASSERT(body != NULL); + if (!body) + return; + TEST_ASSERT(len == strlen(body)); + TEST_ASSERT(strchr(body, '\n') == NULL); + TEST_ASSERT(strchr(body, '\t') == NULL); + cJSON *root = cJSON_Parse(body); + TEST_ASSERT(root != NULL); + if (!root) { + free(body); + return; + } + cJSON *err = cJSON_GetObjectItemCaseSensitive(root, "error"); + cJSON *msg = err ? cJSON_GetObjectItemCaseSensitive(err, "message") : NULL; + cJSON *type = err ? cJSON_GetObjectItemCaseSensitive(err, "type") : NULL; + if (!msg || !type || !cJSON_IsString(msg) || !cJSON_IsString(type)) { + TEST_ASSERT(0 && "error body fields must be strings"); + cJSON_Delete(root); + free(body); + return; + } + TEST_ASSERT(strcmp(msg->valuestring, "line\nquote\" tab\t ctrl\001 slash\\") == 0); + TEST_ASSERT(strcmp(type->valuestring, "bad\ntype") == 0); + cJSON_Delete(root); + free(body); +} + +static void test_job_set_422_replaces_response(void) { + job j; + memset(&j, 0, sizeof(j)); + j.response = xstrdup("old response"); + j.response_len = strlen(j.response); + j.status = 500; + + cJSON *detail = cJSON_CreateArray(); + ve_add(detail, "[\"body\",\"field\"]", "bad value", "value_error"); + job_set_422(&j, detail); + TEST_ASSERT(j.status == 422); + TEST_ASSERT(j.response != NULL); + if (j.response) { + TEST_ASSERT(j.response_len == strlen(j.response)); + TEST_ASSERT(strstr(j.response, "bad value") != NULL); + } + + cJSON_Delete(detail); + free(j.response); +} + +static void test_encoding_from_root_family(void) { + cJSON *detail = cJSON_CreateArray(); + + /* Default encoding follows the explicit serving API. */ + cJSON *empty = cJSON_CreateObject(); + TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_OPENAI), "float") == 0); + TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_PERPLEXITY), "base64_int8") == + 0); + cJSON_Delete(empty); + TEST_ASSERT(cJSON_GetArraySize(detail) == 0); + + /* OpenAI rejects int8 formats; Perplexity rejects the OpenAI base64. */ + cJSON *o = cJSON_CreateObject(); + cJSON_AddStringToObject(o, "encoding_format", "base64_int8"); + encoding_from_root(o, detail, FFWD_API_OPENAI); + TEST_ASSERT(cJSON_GetArraySize(detail) == 1); + cJSON_Delete(o); + + cJSON *p = cJSON_CreateObject(); + cJSON_AddStringToObject(p, "encoding_format", "base64"); + encoding_from_root(p, detail, FFWD_API_PERPLEXITY); + TEST_ASSERT(cJSON_GetArraySize(detail) == 2); + cJSON_Delete(p); + + /* An unknown format is rejected. */ + cJSON *bad = cJSON_CreateObject(); + cJSON_AddStringToObject(bad, "encoding_format", "base64_fp16"); + encoding_from_root(bad, detail, FFWD_API_OPENAI); + TEST_ASSERT(cJSON_GetArraySize(detail) == 3); + cJSON_Delete(bad); + + cJSON_Delete(detail); +} + +static void test_text_type(void) { + cJSON *detail = cJSON_CreateArray(); + + /* Absent: no-op for both model types, no error. */ + cJSON *empty = cJSON_CreateObject(); + TEST_ASSERT(text_type_is_query(empty, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0); + TEST_ASSERT(text_type_is_query(empty, detail, NULL) == 0); + cJSON_Delete(empty); + TEST_ASSERT(cJSON_GetArraySize(detail) == 0); + + /* Qwen3: query -> 1, document -> 0, both accepted. */ + cJSON *q = cJSON_CreateObject(); + cJSON_AddStringToObject(q, "text_type", "query"); + TEST_ASSERT(text_type_is_query(q, detail, TEST_QWEN3_QUERY_INSTRUCT) == 1); + cJSON_Delete(q); + cJSON *d = cJSON_CreateObject(); + cJSON_AddStringToObject(d, "text_type", "document"); + TEST_ASSERT(text_type_is_query(d, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0); + cJSON_Delete(d); + TEST_ASSERT(cJSON_GetArraySize(detail) == 0); + + /* Invalid enum value on Qwen3 is reported. */ + cJSON *bad = cJSON_CreateObject(); + cJSON_AddStringToObject(bad, "text_type", "passage"); + TEST_ASSERT(text_type_is_query(bad, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0); + TEST_ASSERT(cJSON_GetArraySize(detail) == 1); + cJSON_Delete(bad); + + /* text_type on a model with no query instruction is rejected. */ + cJSON *p = cJSON_CreateObject(); + cJSON_AddStringToObject(p, "text_type", "query"); + TEST_ASSERT(text_type_is_query(p, detail, NULL) == 0); + TEST_ASSERT(cJSON_GetArraySize(detail) == 2); + cJSON_Delete(p); + + cJSON_Delete(detail); +} + +int main(void) { + test_json_error_body_escaping(); + test_job_set_422_replaces_response(); + test_encoding_from_root_family(); + test_text_type(); + return TEST_REPORT("json"); +} diff --git a/tools/server/test_models.c b/tools/server/test_models.c new file mode 100644 index 0000000..3dce62b --- /dev/null +++ b/tools/server/test_models.c @@ -0,0 +1,26 @@ +/* test_models.c - resolving a served model by its request label. */ + +#include "server_internal.h" +#include "test_util.h" + +#include <string.h> + +static void test_loaded_model_lookup(void) { + model_info info; + memset(&info, 0, sizeof(info)); + info.id = "served-label"; + loaded_model models[1]; + memset(models, 0, sizeof(models)); + models[0].info = &info; + http_server s; + memset(&s, 0, sizeof(s)); + s.models = models; + s.n_models = 1; + TEST_ASSERT(loaded_model_for_label(&s, "served-label") == &models[0]); + TEST_ASSERT(loaded_model_for_label(&s, "compiled-name") == NULL); +} + +int main(void) { + test_loaded_model_lookup(); + return TEST_REPORT("models"); +} diff --git a/tools/server/test_sbuf.c b/tools/server/test_sbuf.c new file mode 100644 index 0000000..65d0eba --- /dev/null +++ b/tools/server/test_sbuf.c @@ -0,0 +1,35 @@ +/* test_sbuf.c - growable string buffer growth, formatting, and clearing. */ + +#include "sbuf.h" +#include "test_util.h" + +#include <string.h> + +static void test_sbuf_growth_and_formatting(void) { + sbuf b = {0}; + sbuf_puts(&b, "ab"); + sbuf_putc(&b, 'c'); + TEST_ASSERT(b.len == 3); + TEST_ASSERT(strcmp(b.ptr, "abc") == 0); + + size_t old_cap = b.cap; + sbuf_printf(&b, "-%04d-%s", 7, "tail"); + TEST_ASSERT(strcmp(b.ptr, "abc-0007-tail") == 0); + TEST_ASSERT(b.cap == old_cap); + + char big[600]; + memset(big, 'x', sizeof(big) - 1); + big[sizeof(big) - 1] = '\0'; + sbuf_puts(&b, big); + TEST_ASSERT(b.len == strlen("abc-0007-tail") + sizeof(big) - 1); + TEST_ASSERT(b.ptr[b.len] == '\0'); + sbuf_clear(&b); + TEST_ASSERT(b.len == 0); + TEST_ASSERT(b.ptr && b.ptr[0] == '\0'); + sbuf_free(&b); +} + +int main(void) { + test_sbuf_growth_and_formatting(); + return TEST_REPORT("sbuf"); +} diff --git a/tools/server/test_schedule.c b/tools/server/test_schedule.c new file mode 100644 index 0000000..1f799f1 --- /dev/null +++ b/tools/server/test_schedule.c @@ -0,0 +1,73 @@ +/* test_schedule.c - job queue micro-batching: the batch-wait deadline and the + * zero-wait fast path through enqueue_job / collect_job_batch. */ + +#include "server_internal.h" +#include "test_util.h" +#include "util.h" /* nstime */ + +#include <pthread.h> +#include <stdio.h> +#include <string.h> + +static void test_collect_job_batch_deadline(void) { + http_server s; + memset(&s, 0, sizeof(s)); + s.batch_size = 32; + s.batch_wait_us = 1000; + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + uint64_t first_ns = nstime() - 5000000u; + job first = {.srv = &s, .created_ns = first_ns}; + job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u}; + job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u}; + snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings"); + snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", "/v1/embeddings"); + snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", "/v1/embeddings"); + enqueue_job(&first); + enqueue_job(&before_deadline); + enqueue_job(&after_deadline); + + job *batch[3] = {0}; + TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2); + TEST_ASSERT(batch[0] == &first); + TEST_ASSERT(batch[1] == &before_deadline); + TEST_ASSERT(s.job_head == &after_deadline); + TEST_ASSERT(s.job_tail == &after_deadline); + + s.job_head = NULL; + s.job_tail = NULL; + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); +} + +static void test_collect_job_batch_zero_wait(void) { + http_server s; + memset(&s, 0, sizeof(s)); + s.batch_size = 32; + pthread_mutex_init(&s.mu, NULL); + pthread_cond_init(&s.cv, NULL); + + job first = {.srv = &s, .created_ns = nstime()}; + job second = {.srv = &s, .created_ns = nstime()}; + snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings"); + snprintf(second.path, sizeof(second.path), "%s", "/v1/embeddings"); + enqueue_job(&first); + enqueue_job(&second); + + job *batch[2] = {0}; + TEST_ASSERT(collect_job_batch(&s, batch, 2) == 2); + TEST_ASSERT(batch[0] == &first); + TEST_ASSERT(batch[1] == &second); + TEST_ASSERT(s.job_head == NULL); + TEST_ASSERT(s.job_tail == NULL); + + pthread_cond_destroy(&s.cv); + pthread_mutex_destroy(&s.mu); +} + +int main(void) { + test_collect_job_batch_deadline(); + test_collect_job_batch_zero_wait(); + return TEST_REPORT("schedule"); +} diff --git a/tools/server/test_util.h b/tools/server/test_util.h new file mode 100644 index 0000000..c6b6a13 --- /dev/null +++ b/tools/server/test_util.h @@ -0,0 +1,26 @@ +#ifndef FFWD_SERVER_TEST_UTIL_H +#define FFWD_SERVER_TEST_UTIL_H + +/* Shared harness for the ffwd-server unit tests. TEST_ASSERT records failures + * (rather than aborting) so a single run reports every failure, not just the + * first. Each test file includes this, runs its checks, and ends main with + * `return TEST_REPORT("label");`, which prints a summary and yields the exit + * code (0 when clean, 1 otherwise). */ + +#include <stdio.h> + +static int test_failures = 0; + +#define TEST_ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + test_failures++; \ + } \ + } while (0) + +#define TEST_REPORT(label) \ + (test_failures ? (fprintf(stderr, "%s: %d failure(s)\n", (label), test_failures), 1) \ + : (printf("ok: %s\n", (label)), 0)) + +#endif /* FFWD_SERVER_TEST_UTIL_H */ diff --git a/src/server_util.c b/tools/server/util.c index bc20804..1ec4a2a 100644 --- a/src/server_util.c +++ b/tools/server/util.c @@ -1,4 +1,4 @@ -#include "server_util.h" +#include "util.h" #include <stdarg.h> #include <stdio.h> @@ -8,7 +8,7 @@ #include <time.h> void die_oom(void) { - fprintf(stderr, "embed-server: out of memory\n"); + fprintf(stderr, "ffwd-server: out of memory\n"); exit(1); } diff --git a/src/server_util.h b/tools/server/util.h index 955903c..3175cd0 100644 --- a/src/server_util.h +++ b/tools/server/util.h @@ -1,21 +1,21 @@ -#ifndef EMBED_SERVER_UTIL_H -#define EMBED_SERVER_UTIL_H +#ifndef FFWD_SERVER_UTIL_H +#define FFWD_SERVER_UTIL_H #include <stdint.h> #include <stddef.h> /* Die-on-OOM allocation, monotonic clocks, and timestamped logging shared by - * the embed-server translation units. The x* allocators abort the process on + * the ffwd-server translation units. The x* allocators abort the process on * failure rather than returning NULL, so callers never null-check them; the * attributes below carry that contract across translation-unit boundaries so * the static analyzer does not flag the (correctly) unchecked uses. */ #if defined(__GNUC__) || defined(__clang__) -#define SERVER_NORETURN __attribute__((noreturn)) -#define SERVER_RETURNS_NONNULL __attribute__((returns_nonnull)) +# define SERVER_NORETURN __attribute__((noreturn)) +# define SERVER_RETURNS_NONNULL __attribute__((returns_nonnull)) #else -#define SERVER_NORETURN -#define SERVER_RETURNS_NONNULL +# define SERVER_NORETURN +# define SERVER_RETURNS_NONNULL #endif SERVER_NORETURN void die_oom(void); @@ -33,4 +33,4 @@ double ns_to_ms(uint64_t ns); /* printf-style line logger: timestamp prefix, newline appended, to stderr. */ void server_log(const char *fmt, ...) __attribute__((format(printf, 1, 2))); -#endif /* EMBED_SERVER_UTIL_H */ +#endif /* FFWD_SERVER_UTIL_H */ |
