aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
-rw-r--r--Makefile499
-rw-r--r--build.mk230
-rw-r--r--libffwd/Makefile61
-rw-r--r--libffwd/act.c (renamed from src/kernels_act.c)32
-rw-r--r--libffwd/alloc.c (renamed from src/alloc.c)12
-rw-r--r--libffwd/alloc.h (renamed from src/alloc.h)6
-rw-r--r--libffwd/attn.c464
-rw-r--r--libffwd/avx.c (renamed from src/kernels_avx.c)186
-rw-r--r--libffwd/backend_blas.c168
-rw-r--r--libffwd/backend_cuda.c172
-rw-r--r--libffwd/backend_mlx.c228
-rw-r--r--libffwd/bpe.c (renamed from src/tokenizer_bpe.c)124
-rw-r--r--libffwd/bpe.h74
-rw-r--r--libffwd/build.h49
-rw-r--r--libffwd/config.c (renamed from src/config.c)87
-rw-r--r--libffwd/config.h8
-rw-r--r--libffwd/cuda.cu (renamed from src/cuda.cu)407
-rw-r--r--libffwd/cuda.h (renamed from src/cuda.h)64
-rw-r--r--libffwd/dtype.h (renamed from src/dtype.h)12
-rw-r--r--libffwd/ffwd.h280
-rw-r--r--libffwd/forward.c (renamed from src/forward.c)309
-rw-r--r--libffwd/gemm.c (renamed from src/kernels_gemm.c)101
-rw-r--r--libffwd/generic.c (renamed from src/kernels_generic.c)14
-rw-r--r--libffwd/impl.h279
-rw-r--r--libffwd/internal.h134
-rw-r--r--libffwd/kernels.h (renamed from src/kernels.h)76
-rw-r--r--libffwd/late.c218
-rw-r--r--libffwd/late_blas.c173
-rw-r--r--libffwd/mlx.c (renamed from src/mlx.c)360
-rw-r--r--libffwd/mlx.h (renamed from src/mlx.h)117
-rw-r--r--libffwd/model.c (renamed from src/model.c)352
-rw-r--r--libffwd/model_blas.c201
-rw-r--r--libffwd/model_internal.h (renamed from src/model_internal.h)40
-rw-r--r--libffwd/model_types.h (renamed from src/internal.h)63
-rw-r--r--libffwd/neon.c (renamed from src/kernels_neon.c)71
-rw-r--r--libffwd/norm.c142
-rw-r--r--libffwd/platform.h27
-rw-r--r--libffwd/rope.c (renamed from src/kernels_rope.c)26
-rw-r--r--libffwd/safetensors.c (renamed from src/safetensors.c)50
-rw-r--r--libffwd/safetensors.h (renamed from src/safetensors.h)20
-rw-r--r--libffwd/sentencepiece.c (renamed from src/tokenizer_sentencepiece.c)144
-rw-r--r--libffwd/sentencepiece.h88
-rw-r--r--libffwd/threadpool.c (renamed from src/threadpool.c)59
-rw-r--r--libffwd/threadpool.h (renamed from src/threadpool.h)22
-rw-r--r--libffwd/tokenize.c350
-rw-r--r--libffwd/unicode.h (renamed from src/tokenizer_unicode.h)95
-rw-r--r--libffwd/vec.c (renamed from src/vec.c)10
-rw-r--r--libffwd/wordpiece.c (renamed from src/tokenizer_wordpiece.c)80
-rw-r--r--libffwd/wordpiece.h70
-rw-r--r--src/build.h64
-rw-r--r--src/cli.c911
-rw-r--r--src/config.h8
-rw-r--r--src/embed.h357
-rw-r--r--src/kernels_attn.c469
-rw-r--r--src/kernels_impl.h53
-rw-r--r--src/kernels_norm.c245
-rw-r--r--src/late.c383
-rw-r--r--src/server.c1042
-rw-r--r--src/server.h58
-rw-r--r--src/server_models.c627
-rw-r--r--src/tokenizer_bpe.h86
-rw-r--r--src/tokenizer_sentencepiece.h99
-rw-r--r--src/tokenizer_wordpiece.h79
-rw-r--r--tools/cli/Makefile22
-rw-r--r--tools/cli/main.c556
-rw-r--r--tools/server/Makefile49
-rw-r--r--tools/server/base64.c (renamed from src/base64.c)2
-rw-r--r--tools/server/base64.h (renamed from src/base64.h)6
-rw-r--r--tools/server/encode.c (renamed from src/server_encode.c)8
-rw-r--r--tools/server/handlers.c (renamed from src/server_handlers.c)221
-rw-r--r--tools/server/http.c (renamed from src/server_http.c)10
-rw-r--r--tools/server/json.c (renamed from src/server_json.c)20
-rw-r--r--tools/server/main.c342
-rw-r--r--tools/server/models.c342
-rw-r--r--tools/server/sbuf.c (renamed from src/sbuf.c)2
-rw-r--r--tools/server/sbuf.h (renamed from src/sbuf.h)6
-rw-r--r--tools/server/schedule.c (renamed from src/server_schedule.c)4
-rw-r--r--tools/server/server.c414
-rw-r--r--tools/server/server.h57
-rw-r--r--tools/server/server_internal.h (renamed from src/server_internal.h)153
-rw-r--r--tools/server/test_base64.c51
-rw-r--r--tools/server/test_encode.c90
-rw-r--r--tools/server/test_json.c143
-rw-r--r--tools/server/test_models.c26
-rw-r--r--tools/server/test_sbuf.c35
-rw-r--r--tools/server/test_schedule.c73
-rw-r--r--tools/server/test_util.h26
-rw-r--r--tools/server/util.c (renamed from src/server_util.c)4
-rw-r--r--tools/server/util.h (renamed from src/server_util.h)16
89 files changed, 7044 insertions, 6969 deletions
diff --git a/Makefile b/Makefile
index a2fe295..50fe210 100644
--- a/Makefile
+++ b/Makefile
@@ -1,480 +1,65 @@
-CC ?= gcc
-AR ?= ar
-ARCH_FLAGS ?= -march=native
+include build.mk
-BUILD_DATE ?= $(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
-BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
-BUILD_OS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
-BUILD_ARCH ?= $(shell uname -m)
+# Top-level build:
+# Root owns BACKEND/MODE and exports the resolved flags through build.mk.
-# Version and build identity compiled into the embed and embed-server binaries.
-# Applied on the cli.o and server.o recipes. VERSION is set for release builds;
-# otherwise the C fallbacks apply.
-VERSION_CFLAGS = \
- -DEMBED_BUILD_DATE=\"$(BUILD_DATE)\" \
- -DEMBED_BUILD_COMMIT=\"$(BUILD_COMMIT)\" \
- -DEMBED_BUILD_OS=\"$(BUILD_OS)\" \
- -DEMBED_BUILD_ARCH=\"$(BUILD_ARCH)\"
-ifneq ($(strip $(VERSION)),)
-VERSION_CFLAGS += -DEMBED_VERSION=\"$(VERSION)\"
-endif
-
-# Allow FP reassociation and contraction so reduction loops vectorize, but do
-# not assume finite math or approximate libm calls: full -ffast-math produced
-# NaN embeddings on GCC/OpenBLAS Linux builds (-ffinite-math-only and
-# -fapprox-func are the unsafe parts and stay off).
-# -fvisibility=hidden pairs with EMBED_API in the public headers: a new
-# public function without the annotation will be missing from the shared lib.
-CFLAGS_BASE = -Wall -Wextra -O3 $(ARCH_FLAGS) -fPIC -fvisibility=hidden \
- -fno-math-errno -ffp-contract=fast -fno-trapping-math \
- -fno-signed-zeros -fassociative-math -freciprocal-math
-CJSON_CFLAGS ?= $(shell sh -c 'pkg-config --cflags libcjson 2>/dev/null')
-CJSON_LDFLAGS ?= $(shell sh -c 'pkg-config --libs libcjson 2>/dev/null')
-ifeq ($(strip $(CJSON_LDFLAGS)),)
- CJSON_LDFLAGS = -lcjson
-endif
-
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Darwin)
-ifeq ($(strip $(CJSON_CFLAGS)),)
- CJSON_CFLAGS = -I/opt/homebrew/include -I/usr/local/include
-endif
-ifeq ($(strip $(CJSON_LDFLAGS)),-lcjson)
- CJSON_LDFLAGS = -L/opt/homebrew/lib -L/usr/local/lib -lcjson
-endif
-endif
+LIBS := libffwd
+TOOLS := tools/cli tools/server
-# OpenBLAS lives in different places across distros (/usr/include/openblas on
-# Debian/Ubuntu, plain /usr/include on Arch/Alpine, openblas-openmp variants).
-# Prefer pkg-config and fall back to the Debian layout so existing setups keep
-# working. Used by the Linux/CUDA backends and the test/bench BLAS flags below.
-OPENBLAS_CFLAGS ?= $(shell sh -c 'pkg-config --cflags openblas 2>/dev/null')
-OPENBLAS_LDFLAGS ?= $(shell sh -c 'pkg-config --libs openblas 2>/dev/null')
-ifeq ($(strip $(OPENBLAS_CFLAGS)),)
- OPENBLAS_CFLAGS = -I/usr/include/openblas
-endif
-ifeq ($(strip $(OPENBLAS_LDFLAGS)),)
- OPENBLAS_LDFLAGS = -lopenblas
-endif
-
-BACKEND ?= cpu
-MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null)
-MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null)
-CUDA_HOME ?= /usr/local/cuda
-NVCC ?= $(CUDA_HOME)/bin/nvcc
-
-ifeq ($(BACKEND),mlx)
-BACKEND_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK -DUSE_MLX -I$(MLXC_PREFIX)/include
-BACKEND_LDFLAGS = -framework Accelerate -framework Metal -framework Foundation \
- -L$(MLXC_PREFIX)/lib -lmlxc \
- -L$(MLX_PREFIX)/lib -lmlx \
- -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib
-EXTRA_OBJS = src/mlx.o
-else ifeq ($(BACKEND),cuda)
-BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA $(OPENBLAS_CFLAGS) -I$(CUDA_HOME)/include
-BACKEND_LDFLAGS = -L$(CUDA_HOME)/lib64 $(OPENBLAS_LDFLAGS) -lcudart -lcublas
-EXTRA_OBJS = src/cuda.o
-else ifeq ($(UNAME_S),Darwin)
-BACKEND_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK
-BACKEND_LDFLAGS = -framework Accelerate
-else
-BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS $(OPENBLAS_CFLAGS)
-BACKEND_LDFLAGS = $(OPENBLAS_LDFLAGS)
-endif
+.PHONY: all libs tools test clean debug profile profiling release blas cpu mlx cuda gpu \
+ bench bench-model bench-tokenizer bench-server-utils $(LIBS) $(TOOLS)
-# -Isrc so frontends, tests, and benches resolve the public/internal headers
-# now that every source lives under src/. EXTRA_CFLAGS/EXTRA_LDFLAGS are the
-# append points the debug and prof targets use to layer on sanitizers and
-# debug info without dropping the architecture, visibility, or backend flags.
-# -MMD -MP make the compiler emit a .d sidecar per object recording the headers
-# it included (-MP adds phony targets so a deleted header is not a hard error);
-# these are -include'd at the foot of the file so a header edit rebuilds only
-# the objects that use it. They live here, not in CFLAGS_BASE, so the one-shot
-# test/bench compile+link commands (which reuse CFLAGS_BASE) do not emit stray
-# .d files next to their output binaries.
-CFLAGS = $(CFLAGS_BASE) $(BACKEND_CFLAGS) -Isrc -MMD -MP $(EXTRA_CFLAGS)
-LDFLAGS = -lm -lpthread $(BACKEND_LDFLAGS) $(EXTRA_LDFLAGS)
+all: libs tools
+ @ln -sf $(BUILDREL)/ffwd-cli ./ffwd-cli
+ @ln -sf $(BUILDREL)/ffwd-server ./ffwd-server
-# Source groups. CORE_SRCS is the engine the tests and benches reuse; keeping it
-# in one variable stops the per-test source lists from drifting apart.
-KERNEL_SRCS = src/kernels_gemm.c src/kernels_norm.c src/kernels_act.c src/kernels_attn.c \
- src/kernels_rope.c src/threadpool.c \
- src/kernels_generic.c src/kernels_neon.c src/kernels_avx.c
-TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c src/tokenizer_sentencepiece.c
-CORE_SRCS = src/model.c src/forward.c src/late.c src/vec.c src/alloc.c \
- src/config.c src/safetensors.c $(KERNEL_SRCS)
+libs: $(LIBS)
+tools: $(TOOLS)
-# Server-only objects: the embed-server binary and the server test link these
-# alongside server.o. Not in libembed.a - the CLI and library do not use them.
-SERVER_SRCS = src/server_util.c src/sbuf.c src/base64.c src/server_json.c src/server_encode.c src/server_http.c src/server_models.c src/server_handlers.c src/server_schedule.c
-SERVER_OBJS = $(SERVER_SRCS:.c=.o)
+$(LIBS):
+ $(MAKE) -C $@
-SRCS = $(CORE_SRCS) $(TOKENIZER_SRCS) \
- deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c
+tools/cli: | libffwd
+ $(MAKE) -C $@
-MLX_SRCS = src/mlx.c
-CUDA_SRCS = src/cuda.cu
+tools/server: | libffwd
+ $(MAKE) -C $@
-OBJS = $(SRCS:.c=.o)
-MLX_OBJS = $(MLX_SRCS:.c=.o)
-HEADERS = $(wildcard src/*.h)
-TARGET = embed
-SERVER_TARGET = embed-server
-LIB = libembed.a
-
-ifeq ($(shell uname -s),Darwin)
-SHARED_LIB = libembed.dylib
-SHARED_FLAGS = -dynamiclib
-else
-SHARED_LIB = libembed.so
-SHARED_FLAGS = -shared
-endif
+# The current unit suite is CPU-internal; GPU checks are explicit driver targets.
+test:
+ $(MAKE) -C tests BACKEND=blas test
+ $(MAKE) -C tools/server BACKEND=blas test
-.PHONY: all cpu mlx cuda gpu prof test test-bf16 test-safetensors coverage bench \
- bench-model bench-tokenizer bench-server-utils debug clean help force
+clean:
+ $(RM) -r build
+ @for d in $(TOOLS) tests bench; do $(MAKE) -C $$d clean; done
+ $(RM) -f ./ffwd-cli ./ffwd-server ffwd.exports ffwd.map
-all: help
+blas:
+ $(MAKE) BACKEND=blas MODE=$(MODE) all
-help:
- @echo "Targets:"
- @echo " make cpu CPU-only build"
- @echo " make gpu Build with GPU backend support (mlx on macOS, cuda on Linux)"
- @echo " make mlx Apple Silicon GPU via MLX"
- @echo " make cuda NVIDIA GPU via CUDA"
- @echo " make test Build and run the test suite (no model files needed)"
- @echo " make coverage Test-suite line coverage report (clang/llvm-cov)"
- @echo " make bench Kernel microbenchmarks; records bench/results/*.json"
- @echo " make bench-model MODEL_DIR=/path/to/model"
- @echo " End-to-end embedding benchmark"
- @echo " make bench-server-utils"
- @echo " Base64 and server string-buffer microbenchmarks"
- @echo " make debug Debug build"
- @echo " make prof Profiling build"
- @echo " make clean Remove build artifacts"
+cpu: blas
-# =============================================================================
-# Backend builds: build both binaries and the shared library for one backend.
-# =============================================================================
-# Switching BACKEND changes CFLAGS but not the source mtimes, so Make alone
-# would relink stale objects on a backend switch. The .backend stamp below
-# tracks the active backend so these targets keep incremental builds within a
-# backend and still rebuild correctly across one.
-cpu mlx cuda:
- $(MAKE) BACKEND=$@ $(TARGET) $(SERVER_TARGET) $(SHARED_LIB)
+mlx:
+ $(MAKE) BACKEND=mlx MODE=$(MODE) all
-# Stamp recording the backend the current objects were built for. `force` makes
-# the recipe run every build, but it rewrites the file (bumping its mtime) only
-# when BACKEND actually changed - so the engine objects, which list it as a
-# prerequisite, rebuild on a switch and stay untouched otherwise.
-.backend: force
- @cur=`cat $@ 2>/dev/null || true`; \
- [ "$$cur" = "$(BACKEND)" ] || { echo "$(BACKEND)" > $@; }
-force:
+cuda:
+ $(MAKE) BACKEND=cuda MODE=$(MODE) all
-ifeq ($(UNAME_S),Darwin)
+ifeq ($(OS),Darwin)
gpu: mlx
else
gpu: cuda
endif
-# =============================================================================
-# Compile rules
-# =============================================================================
-# Generic rule for the plain core/kernel/tokenizer/safetensors/mlx objects.
-# Sibling headers in src/ resolve via the quote-include search; -Isrc (in
-# CFLAGS) makes it explicit. Header prerequisites come from the -MMD-generated
-# .d files (-include'd at the foot of the file), which track them per object.
-# The .backend prerequisite is attached in bulk further down.
-src/%.o: src/%.c
- $(CC) $(CFLAGS) -c -o $@ $<
-
-# -fPIC so the same object links into both binaries and the shared library.
-src/cuda.o: src/cuda.cu src/cuda.h src/internal.h src/embed.h
- $(NVCC) -O3 -std=c++17 -arch=native -Xcompiler -fPIC,-fvisibility=hidden -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -Isrc $(OPENBLAS_CFLAGS) -I$(CUDA_HOME)/include -x cu -c -o $@ $<
-
-# server.c reaches deps/ae via "deps/ae/ae.h", so it needs -I. (repo root).
-src/server.o: src/server.c
- $(CC) $(CFLAGS) $(VERSION_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae -c -o $@ $<
-
-# Server concern objects: server_internal.h pulls deps/ae/ae.h and cJSON, so
-# they need the same -I. and cJSON flags as server.o (the leaf util/sbuf/base64
-# objects do not include it but are unharmed by the extra search paths).
-$(SERVER_OBJS): src/%.o: src/%.c
- $(CC) $(CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae -c -o $@ $<
-
-src/cli.o: src/cli.c
- $(CC) $(CFLAGS) $(VERSION_CFLAGS) -c -o $@ $<
-
-deps/ae/%.o: deps/ae/%.c
- $(CC) $(CFLAGS) -Ideps/ae -c -o $@ $<
-
-# Every engine object is built for one backend; attach the .backend stamp to
-# all of them at once (the recipes come from the pattern rules above) so a
-# backend switch invalidates them. cli.o/server.o are listed too because they
-# inline backend-specific paths through the shared headers.
-$(OBJS) $(EXTRA_OBJS) $(SERVER_OBJS) src/cli.o src/server.o: .backend
-
-# =============================================================================
-# Shared library (built by every backend target from the same objects)
-# =============================================================================
-$(SHARED_LIB): $(OBJS) $(EXTRA_OBJS)
- $(CC) $(CFLAGS) $(SHARED_FLAGS) -o $@ $^ $(LDFLAGS)
-
-# =============================================================================
-# Static library and binary link
-# =============================================================================
-$(LIB): $(OBJS) $(EXTRA_OBJS)
- $(AR) rcs $@ $^
-
-$(TARGET): $(LIB) src/cli.o
- $(CC) $(CFLAGS) -o $@ src/cli.o $(LIB) $(LDFLAGS)
-
-$(SERVER_TARGET): $(LIB) src/server.o $(SERVER_OBJS)
- $(CC) $(CFLAGS) -o $@ src/server.o $(SERVER_OBJS) $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS)
-
-# =============================================================================
-# Tests (no model files required)
-# =============================================================================
-ifeq ($(UNAME_S),Darwin)
-TEST_BLAS_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK
-TEST_BLAS_LDFLAGS = -framework Accelerate
-else
-TEST_BLAS_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS $(OPENBLAS_CFLAGS)
-TEST_BLAS_LDFLAGS = $(OPENBLAS_LDFLAGS)
-endif
-
-# Source lists shared by test and coverage targets so the two cannot drift.
-TEST_CC_FLAGS = -Wall -Wextra -O2 -Isrc
-TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS)
-TEST_TOKENIZER_SRCS = tests/test_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS)
-TEST_WORDPIECE_SRCS = tests/test_wordpiece.c src/tokenizer_wordpiece.c
-TEST_SENTENCEPIECE_SRCS = tests/test_sentencepiece.c src/tokenizer_sentencepiece.c
-TEST_SAFETENSORS_SRCS = tests/test_safetensors.c src/safetensors.c
-TEST_BF16_SRCS = tests/test_bf16_model.c $(CORE_SRCS)
-TEST_QWEN3_SRCS = tests/test_qwen3.c $(CORE_SRCS)
-TEST_QWEN2_SRCS = tests/test_qwen2.c $(CORE_SRCS)
-TEST_CLS_SRCS = tests/test_cls.c $(CORE_SRCS)
-TEST_XLM_ROBERTA_SRCS = tests/test_xlm_roberta.c $(CORE_SRCS)
-TEST_WORKSPACE_SRCS = tests/test_workspace.c $(CORE_SRCS) src/tokenizer_bpe.c
-TEST_LATE_SRCS = tests/test_late.c $(CORE_SRCS) src/tokenizer_bpe.c
-# test_server.c #includes ../src/server.c, so server is not a separate object
-# here; it needs -I. for server.c's "deps/ae/ae.h" plus the ae sources.
-# test_server.c #includes ../src/server.c, so server.c itself is not listed;
-# the carved-out server objects (util/sbuf/base64) are separate TUs and must be.
-TEST_SERVER_SRCS = tests/test_server.c $(CORE_SRCS) $(TOKENIZER_SRCS) \
- $(SERVER_SRCS) \
- deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c
-# The CLI check builds the real CLI (CPU backend) plus a driver that runs it.
-TEST_CLI_BIN_SRCS = src/cli.c $(CORE_SRCS) $(TOKENIZER_SRCS)
+release:
+ $(MAKE) MODE=release BACKEND=$(BACKEND) all
-test:
- $(CC) $(TEST_CC_FLAGS) -o tests/test_kernels_generic \
- $(TEST_KERNELS_SRCS) -lm -lpthread
- ./tests/test_kernels_generic
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_kernels_blas \
- $(TEST_KERNELS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- ./tests/test_kernels_blas
- $(CC) $(TEST_CC_FLAGS) -o tests/test_tokenizer \
- $(TEST_TOKENIZER_SRCS) -lm -lpthread
- ./tests/test_tokenizer
- $(CC) $(TEST_CC_FLAGS) -o tests/test_wordpiece $(TEST_WORDPIECE_SRCS)
- ./tests/test_wordpiece
- $(CC) $(TEST_CC_FLAGS) -o tests/test_sentencepiece $(TEST_SENTENCEPIECE_SRCS)
- ./tests/test_sentencepiece
- $(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS)
- ./tests/test_safetensors
- $(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \
- $(TEST_BF16_SRCS) -lm -lpthread
- ./tests/test_bf16_model
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \
- $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- ./tests/test_qwen3
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \
- $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- ./tests/test_qwen2
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \
- $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- ./tests/test_cls
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \
- $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- ./tests/test_xlm_roberta
- $(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \
- $(TEST_WORKSPACE_SRCS) -lm -lpthread
- ./tests/test_workspace
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_late \
- $(TEST_LATE_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- ./tests/test_late
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/cli_under_test \
- $(TEST_CLI_BIN_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- $(CC) $(TEST_CC_FLAGS) -o tests/test_cli tests/test_cli.c -lm
- ./tests/test_cli ./tests/cli_under_test
- $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae \
- -o tests/test_server $(TEST_SERVER_SRCS) \
- -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CJSON_LDFLAGS)
- ./tests/test_server
-
-test-bf16:
- $(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \
- $(TEST_BF16_SRCS) -lm -lpthread
- ./tests/test_bf16_model
-
-test-safetensors:
- $(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors \
- $(TEST_SAFETENSORS_SRCS) -lm -lpthread
- ./tests/test_safetensors
+debug:
+ $(MAKE) MODE=debug BACKEND=$(BACKEND) all
-# Test-suite line coverage (requires clang + llvm-cov/llvm-profdata. Writes a
-# per-file text summary to stdout and an HTML report to coverage/html/index.html.)
-COV_DIR = coverage
-# Source-based coverage; -O0 so no line is folded away by the optimizer.
-COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \
- -fcoverage-compilation-dir=.
-COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \
- tests/test_tokenizer tests/test_sentencepiece tests/test_safetensors \
- tests/test_bf16_model tests/test_qwen3 tests/test_qwen2 tests/test_workspace \
- tests/test_cls tests/test_xlm_roberta tests/test_late \
- tests/cli_under_test tests/test_server
-# Report on project sources only - not the harnesses, vendored deps, or
-# system headers (cJSON lands under /opt or /usr otherwise).
-COV_IGNORE = -ignore-filename-regex='deps/|tests/|/opt/|/usr/'
-
-ifeq ($(UNAME_S),Darwin)
-LLVM_PROFDATA = xcrun llvm-profdata
-LLVM_COV = xcrun llvm-cov
-else
-LLVM_PROFDATA = llvm-profdata
-LLVM_COV = llvm-cov
-endif
-
-coverage:
- rm -rf $(COV_DIR) && mkdir -p $(COV_DIR)
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_kernels_generic \
- $(TEST_KERNELS_SRCS) -lm -lpthread
- LLVM_PROFILE_FILE=$(COV_DIR)/kernels_generic.profraw ./tests/test_kernels_generic
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_kernels_blas \
- $(TEST_KERNELS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- LLVM_PROFILE_FILE=$(COV_DIR)/kernels_blas.profraw ./tests/test_kernels_blas
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_tokenizer \
- $(TEST_TOKENIZER_SRCS) -lm -lpthread
- LLVM_PROFILE_FILE=$(COV_DIR)/tokenizer.profraw ./tests/test_tokenizer
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_sentencepiece \
- $(TEST_SENTENCEPIECE_SRCS)
- LLVM_PROFILE_FILE=$(COV_DIR)/sentencepiece.profraw ./tests/test_sentencepiece
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS)
- LLVM_PROFILE_FILE=$(COV_DIR)/safetensors.profraw ./tests/test_safetensors
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_bf16_model \
- $(TEST_BF16_SRCS) -lm -lpthread
- LLVM_PROFILE_FILE=$(COV_DIR)/bf16_model.profraw ./tests/test_bf16_model
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen3 \
- $(TEST_QWEN3_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- LLVM_PROFILE_FILE=$(COV_DIR)/qwen3.profraw ./tests/test_qwen3
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \
- $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- LLVM_PROFILE_FILE=$(COV_DIR)/qwen2.profraw ./tests/test_qwen2
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \
- $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- LLVM_PROFILE_FILE=$(COV_DIR)/cls.profraw ./tests/test_cls
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \
- $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- LLVM_PROFILE_FILE=$(COV_DIR)/xlm_roberta.profraw ./tests/test_xlm_roberta
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \
- $(TEST_WORKSPACE_SRCS) -lm -lpthread
- LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_late \
- $(TEST_LATE_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- LLVM_PROFILE_FILE=$(COV_DIR)/late.profraw ./tests/test_late
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/cli_under_test \
- $(TEST_CLI_BIN_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- $(CC) $(TEST_CC_FLAGS) -o tests/test_cli tests/test_cli.c -lm
- LLVM_PROFILE_FILE=$(COV_DIR)/cli_%p.profraw ./tests/test_cli ./tests/cli_under_test
- $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae \
- -o tests/test_server $(TEST_SERVER_SRCS) \
- -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CJSON_LDFLAGS)
- LLVM_PROFILE_FILE=$(COV_DIR)/server.profraw ./tests/test_server
- $(LLVM_PROFDATA) merge -sparse $(COV_DIR)/*.profraw -o $(COV_DIR)/tests.profdata
- $(LLVM_COV) report -instr-profile=$(COV_DIR)/tests.profdata $(COV_IGNORE) \
- $(addprefix -object ,$(COV_BINS))
- $(LLVM_COV) show \
- -format=html \
- -output-dir=$(COV_DIR)/html \
- -instr-profile=$(COV_DIR)/tests.profdata \
- -show-line-counts-or-regions \
- -show-branches=count \
- -show-expansions \
- -show-instantiations \
- $(COV_IGNORE) \
- $(addprefix -object ,$(COV_BINS))
- @echo "HTML report: $(COV_DIR)/html/index.html"
-
-bench-tokenizer:
- $(CC) -Wall -Wextra -O3 -march=native -Isrc -o bench/bench_tokenizer \
- bench/bench_tokenizer.c $(TOKENIZER_SRCS) $(KERNEL_SRCS) -lm -lpthread
- @echo "run: bench/bench_tokenizer <model_dir-or-vocab.json> [runs] TEXT..."
-
-# Regression microbenchmarks (see bench/bench.h). Each run records a JSON
-# sample set under bench/results/, keyed by commit; compare two records with
-# tools/benchstat.py to see if a change helped or hurt.
-BENCH_STAMP = $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet -- ':!.gitignore' 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S)
-BENCH_CC_FLAGS = -Wall -Wextra $(CFLAGS_BASE) $(TEST_BLAS_CFLAGS) -Isrc
-# In-tree alongside the other bench binaries (cleaned by `make clean`) so the
-# default works on any OS; the old /private/tmp default existed only on macOS.
-BENCH_SERVER_UTILS_BIN ?= bench/bench_server_utils
-BENCH_ARGS ?=
-
-bench:
- $(CC) $(BENCH_CC_FLAGS) -o bench/bench_kernels \
- bench/bench_kernels.c $(KERNEL_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
- @mkdir -p bench/results
- ./bench/bench_kernels --json bench/results/kernels-$(BENCH_STAMP).json
- @echo "compare: ./tools/benchstat.py bench/results/OLD.json bench/results/NEW.json"
-
-bench-model:
- @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; }
- $(CC) $(BENCH_CC_FLAGS) -o bench/bench_model \
- bench/bench_model.c $(CORE_SRCS) \
- -lm -lpthread $(TEST_BLAS_LDFLAGS)
- @mkdir -p bench/results
- ./bench/bench_model "$(MODEL_DIR)" --json bench/results/model-$(BENCH_STAMP).json
-
-bench-server-utils:
- $(CC) $(BENCH_CC_FLAGS) -Ibench -o $(BENCH_SERVER_UTILS_BIN) \
- bench/bench_server_utils.c src/base64.c src/sbuf.c src/server_util.c \
- -lm -lpthread $(TEST_BLAS_LDFLAGS)
- $(BENCH_SERVER_UTILS_BIN) $(BENCH_ARGS)
-
-# =============================================================================
-# Debug and profiling builds
-# =============================================================================
-# Both layer extra flags onto the normal build through EXTRA_CFLAGS/EXTRA_LDFLAGS.
-# Backend defaults to cpu; override with BACKEND=mlx or BACKEND=cuda.
-SANITIZE ?= address,undefined
-DEBUG_CFLAGS = -pedantic -fno-omit-frame-pointer -g -O0 -DDEBUG -fsanitize=$(SANITIZE)
-DEBUG_LDFLAGS = -fsanitize=$(SANITIZE)
-
-debug: clean
- $(MAKE) BACKEND=$(BACKEND) \
- EXTRA_CFLAGS="$(DEBUG_CFLAGS)" EXTRA_LDFLAGS="$(DEBUG_LDFLAGS)" \
- $(TARGET) $(SERVER_TARGET) $(SHARED_LIB)
-
-prof: clean
- $(MAKE) BACKEND=$(BACKEND) \
- EXTRA_CFLAGS="-g -fno-omit-frame-pointer" \
- $(TARGET) $(SERVER_TARGET) $(SHARED_LIB)
-
-
-# =============================================================================
-clean:
- rm -f src/*.o src/*.d deps/ae/*.o deps/ae/*.d .backend \
- $(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \
- tests/test_kernels_generic tests/test_kernels_blas \
- tests/test_safetensors tests/test_bf16_model tests/test_server \
- tests/test_qwen3 tests/test_qwen2 tests/test_cls tests/test_xlm_roberta \
- tests/test_tokenizer tests/test_wordpiece tests/test_sentencepiece \
- tests/test_workspace tests/test_cli tests/test_late \
- tests/cli_under_test bench/bench_tokenizer bench/bench_kernels \
- bench/bench_model bench/bench_server_utils
- rm -rf $(COV_DIR)
+profile profiling:
+ $(MAKE) MODE=profile BACKEND=$(BACKEND) all
-# Per-object header dependencies emitted by -MMD -MP. Missing on a clean tree;
-# -include ignores them until the first build creates them.
--include $(OBJS:.o=.d) $(EXTRA_OBJS:.o=.d) $(SERVER_OBJS:.o=.d) src/cli.d src/server.d
+bench bench-model bench-tokenizer bench-server-utils:
+ $(MAKE) -C bench $@
diff --git a/build.mk b/build.mk
new file mode 100644
index 0000000..62e7f45
--- /dev/null
+++ b/build.mk
@@ -0,0 +1,230 @@
+OS := $(shell uname -s)
+ARCH := $(shell uname -m)
+
+CC ?= cc
+AR ?= ar
+RM ?= rm -f
+
+ARCH_FLAGS ?= -march=native
+BACKEND ?= blas
+MODE ?= release
+prefix ?= /usr/local
+
+# Per-configuration build tree under the repo root. Each MODE/BACKEND pair gets
+# its own objects and artifacts, so switching configs never reuses another's
+# output and needs no clean in between. A consumer sets ROOT (path to the repo
+# root) and COMPONENT (a unique name) before including this file; objects land in
+# $(OBJDIR), final artifacts in $(OUTDIR).
+ROOT ?= .
+COMPONENT ?= misc
+BUILDREL := build/$(MODE)/$(BACKEND)
+OUTDIR := $(ROOT)/$(BUILDREL)
+OBJDIR := $(OUTDIR)/obj/$(COMPONENT)
+
+CUDA_HOME ?= /usr/local/cuda
+NVCC ?= $(CUDA_HOME)/bin/nvcc
+CUDAFLAGS ?= -O3 -std=c++17 -arch=native -Xcompiler -fPIC,-fvisibility=hidden
+
+OPENBLAS_CFLAGS ?= $(shell sh -c 'pkg-config --cflags openblas 2>/dev/null')
+OPENBLAS_LDFLAGS ?= $(shell sh -c 'pkg-config --libs openblas 2>/dev/null')
+ifeq ($(strip $(OPENBLAS_CFLAGS)),)
+ OPENBLAS_CFLAGS = -I/usr/include/openblas
+endif
+ifeq ($(strip $(OPENBLAS_LDFLAGS)),)
+ OPENBLAS_LDFLAGS = -lopenblas
+endif
+
+CJSON_CFLAGS ?= $(shell sh -c 'pkg-config --cflags libcjson 2>/dev/null')
+CJSON_LDFLAGS ?= $(shell sh -c 'pkg-config --libs libcjson 2>/dev/null')
+ifeq ($(OS),Darwin)
+ifeq ($(strip $(CJSON_CFLAGS)),)
+ CJSON_CFLAGS = -I/opt/homebrew/include -I/usr/local/include
+endif
+ifeq ($(strip $(CJSON_LDFLAGS)),)
+ CJSON_LDFLAGS = -L/opt/homebrew/lib -L/usr/local/lib -lcjson
+endif
+endif
+ifeq ($(strip $(CJSON_LDFLAGS)),)
+ CJSON_LDFLAGS = -lcjson
+endif
+
+MLX_PREFIX := $(shell brew --prefix mlx 2>/dev/null)
+MLXC_PREFIX := $(shell brew --prefix mlx-c 2>/dev/null)
+
+# Allow FP reassociation and contraction so reduction loops vectorize, but do
+# not assume finite math or approximate libm calls: full -ffast-math produced
+# NaN embeddings on GCC/OpenBLAS Linux builds (-ffinite-math-only and
+# -fapprox-func are the unsafe parts and stay off).
+# -fvisibility=hidden pairs with FFWD_API in the public headers: a new
+# public function without the annotation will be missing from the shared lib.
+CFLAGS_BASE = -Wall -Wextra -O3 $(ARCH_FLAGS) -fPIC -fvisibility=hidden \
+ -fno-math-errno -ffp-contract=fast -fno-trapping-math \
+ -fno-signed-zeros -fassociative-math -freciprocal-math
+
+SANITIZE ?= address,undefined
+ifeq ($(MODE),debug)
+ OPT ?= $(CFLAGS_BASE) -pedantic -fno-omit-frame-pointer -g -O0 -DDEBUG -fsanitize=$(SANITIZE)
+ LDFLAGS_OPT ?= -fsanitize=$(SANITIZE)
+else ifeq ($(MODE),profile)
+ OPT ?= $(CFLAGS_BASE) -g -fno-omit-frame-pointer
+ LDFLAGS_OPT ?=
+else ifeq ($(MODE),release)
+ OPT ?= $(CFLAGS_BASE)
+ LDFLAGS_OPT ?=
+else
+ $(error MODE must be release, debug, or profile)
+endif
+
+# The shared library exports exactly the public ABI listed in the export map
+# (ffwd.exports on Mach-O, ffwd.map on ELF) and hides everything else, so
+# the exported set is identical on every backend and no internal symbol leaks.
+# This pairs with -fvisibility=hidden. Only the libffwd shared library applies
+# EXPORT_FLAGS; the static lib and the binaries link all objects directly.
+ifeq ($(OS),Darwin)
+ SHARED_EXT = dylib
+ SHARED_FLAGS = -dynamiclib
+ EXPORT_FILE = $(ROOT)/ffwd.exports
+ EXPORT_FLAGS = -Wl,-exported_symbols_list,$(ROOT)/ffwd.exports
+ HOST_BLAS_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK
+ HOST_BLAS_LDFLAGS = -framework Accelerate
+else
+ SHARED_EXT = so
+ SHARED_FLAGS = -shared
+ EXPORT_FILE = $(ROOT)/ffwd.map
+ EXPORT_FLAGS = -Wl,--version-script,$(ROOT)/ffwd.map
+ HOST_BLAS_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS $(OPENBLAS_CFLAGS)
+ HOST_BLAS_LDFLAGS = $(OPENBLAS_LDFLAGS)
+endif
+
+# Each backend defines only the flags its own compiled sources need.
+# blas: the CPU build; the kernel files call cblas, so USE_BLAS is on.
+# mlx: the Apple GPU build runs all math on the GPU and compiles no kernel
+# files, so it defines neither USE_BLAS nor links Accelerate. USE_GPU selects
+# MLX via platform.h.
+# cuda: the NVIDIA GPU build still compiles the CPU kernel files for host-side
+# work, so it keeps USE_BLAS (OpenBLAS) alongside the CUDA libraries.
+BLAS_BACKEND_CFLAGS = $(HOST_BLAS_CFLAGS)
+BLAS_BACKEND_LDFLAGS = $(HOST_BLAS_LDFLAGS)
+MLX_BACKEND_CFLAGS = -DUSE_GPU -I$(MLXC_PREFIX)/include
+MLX_BACKEND_LDFLAGS = -framework Metal -framework Foundation \
+ -L$(MLXC_PREFIX)/lib -lmlxc \
+ -L$(MLX_PREFIX)/lib -lmlx \
+ -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib
+CUDA_BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -DUSE_GPU $(OPENBLAS_CFLAGS) \
+ -I$(CUDA_HOME)/include
+CUDA_BACKEND_LDFLAGS = -L$(CUDA_HOME)/lib64 $(OPENBLAS_LDFLAGS) -lcudart -lcublas
+
+ifeq ($(BACKEND),blas)
+ BACKEND_CFLAGS = $(BLAS_BACKEND_CFLAGS)
+ BACKEND_LDFLAGS = $(BLAS_BACKEND_LDFLAGS)
+else ifeq ($(BACKEND),mlx)
+ BACKEND_CFLAGS = $(MLX_BACKEND_CFLAGS)
+ BACKEND_LDFLAGS = $(MLX_BACKEND_LDFLAGS)
+else ifeq ($(BACKEND),cuda)
+ BACKEND_CFLAGS = $(CUDA_BACKEND_CFLAGS)
+ BACKEND_LDFLAGS = $(CUDA_BACKEND_LDFLAGS)
+else
+ $(error BACKEND must be blas, mlx, or cuda)
+endif
+
+BUILD_DATE ?= $(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
+BUILD_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)
+BUILD_OS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
+BUILD_ARCH ?= $(shell uname -m)
+VERSION_CFLAGS = \
+ -DFFWD_BUILD_DATE=\"$(BUILD_DATE)\" \
+ -DFFWD_BUILD_COMMIT=\"$(BUILD_COMMIT)\" \
+ -DFFWD_BUILD_OS=\"$(BUILD_OS)\" \
+ -DFFWD_BUILD_ARCH=\"$(BUILD_ARCH)\"
+ifneq ($(strip $(VERSION)),)
+VERSION_CFLAGS += -DFFWD_VERSION=\"$(VERSION)\"
+endif
+
+ifdef CFLAGS
+else
+CFLAGS = $(OPT)
+endif
+CPPFLAGS ?=
+LDLIBS ?=
+SHARED_LDLIBS ?=
+
+objs = $(patsubst %.c,$(OBJDIR)/%.o,$(filter %.c,$1)) $(patsubst %.cu,$(OBJDIR)/%.o,$(filter %.cu,$1))
+deps = $(patsubst %.o,%.d,$1)
+
+$(OBJDIR)/%.o: %.c
+ @mkdir -p $(@D)
+ $(CC) $(CFLAGS) $(CPPFLAGS) -MMD -MP -c -o $@ $<
+
+$(OBJDIR)/%.o: %.cu
+ @mkdir -p $(@D)
+ $(NVCC) $(CUDAFLAGS) $(CUDA_BACKEND_CFLAGS) $(CPPFLAGS) -x cu -c -o $@ $<
+
+define static_archive
+$(1): $(2) $(3)
+ @mkdir -p $$(@D)
+ $$(RM) $$@
+ $$(AR) rcs $$@ $(2)
+endef
+
+define shared_library
+$(1): $(2) $(3)
+ @mkdir -p $$(@D)
+ $$(CC) $$(SHARED_FLAGS) $$(SHARED_LDFLAGS) -o $$@ $(2) $(3) $(4) $$(LDFLAGS_OPT)
+endef
+
+define link_program
+$(1): $(2) $(3)
+ @mkdir -p $$(@D)
+ $$(CC) $$(CFLAGS) -o $$@ $(2) $(3) $(4) $$(LDFLAGS_OPT)
+endef
+
+define program
+OBJS := $$(call objs,$$(SRCS))
+DEPS := $$(call deps,$$(OBJS))
+.PHONY: all clean
+all: $$(BIN)
+$$(eval $$(call link_program,$$(BIN),$$(OBJS),$$(LIBS),$$(LDLIBS)))
+clean:
+ $$(RM) -r $$(OBJS) $$(DEPS) $$(BIN) $$(BIN).dSYM
+-include $$(DEPS)
+endef
+
+define simple_library
+STATIC := $$(OUTDIR)/lib$$(LIB).a
+SHARED := $$(OUTDIR)/lib$$(LIB).$$(SHARED_EXT)
+OBJS := $$(call objs,$$(SRCS))
+DEPS := $$(call deps,$$(OBJS))
+.PHONY: all clean install
+all: $$(STATIC) $$(SHARED)
+$$(eval $$(call static_archive,$$(STATIC),$$(OBJS),))
+$$(eval $$(call shared_library,$$(SHARED),$$(OBJS),,$$(SHARED_LDLIBS)))
+install: all
+ install -d $$(prefix)/lib $$(prefix)/include
+ install -m 644 $$(STATIC) $$(prefix)/lib/
+ install -m 755 $$(SHARED) $$(prefix)/lib/
+ install -m 644 $$(HEADERS) $$(prefix)/include/
+clean:
+ $$(RM) $$(OBJS) $$(DEPS) $$(STATIC) $$(SHARED)
+-include $$(DEPS)
+endef
+
+define c_program
+$(1): $(2) $(3)
+ @mkdir -p $$(@D)
+ $$(CC) $$(CFLAGS) $$(CPPFLAGS) $(4) -o $$@ $(2) $(3) $(5) $$(LDFLAGS_OPT)
+endef
+
+define c_program_deps
+$(1): $(2) $(3)
+ @mkdir -p $$(@D)
+ $$(CC) $$(CFLAGS) $$(CPPFLAGS) $(5) -o $$@ $(2) $(4) $(6) $$(LDFLAGS_OPT)
+endef
+
+export BACKEND MODE ARCH_FLAGS CFLAGS_BASE OPT LDFLAGS_OPT CUDA_HOME NVCC CUDAFLAGS
+export OPENBLAS_CFLAGS OPENBLAS_LDFLAGS CJSON_CFLAGS CJSON_LDFLAGS
+export MLX_PREFIX MLXC_PREFIX
+export HOST_BLAS_CFLAGS HOST_BLAS_LDFLAGS
+export BLAS_BACKEND_CFLAGS BLAS_BACKEND_LDFLAGS
+export MLX_BACKEND_CFLAGS MLX_BACKEND_LDFLAGS
+export CUDA_BACKEND_CFLAGS CUDA_BACKEND_LDFLAGS
+export BACKEND_CFLAGS BACKEND_LDFLAGS VERSION_CFLAGS
diff --git a/libffwd/Makefile b/libffwd/Makefile
new file mode 100644
index 0000000..0c831a6
--- /dev/null
+++ b/libffwd/Makefile
@@ -0,0 +1,61 @@
+ROOT := ..
+COMPONENT := libffwd
+include ../build.mk
+
+LIB := ffwd
+STATIC := $(OUTDIR)/lib$(LIB).a
+SHARED := $(OUTDIR)/lib$(LIB).$(SHARED_EXT)
+
+CFLAGS += -std=c11 -pthread $(BACKEND_CFLAGS)
+CPPFLAGS += -I.
+SHARED_FLAGS += -pthread
+# Restrict the shared library's exported symbols to the public ABI.
+SHARED_LDFLAGS += $(EXPORT_FLAGS)
+
+# Tokenizer, config, safetensors reader, and L2/cosine helpers: every backend.
+COMMON_SRCS := config.c vec.c tokenize.c bpe.c sentencepiece.c wordpiece.c safetensors.c
+
+# CPU math kernels and SIMD dispatch. generic.c is always built; avx.c and
+# neon.c each guard their entire body by arch (__AVX2__ / __ARM_NEON), so the
+# off-target file compiles to an empty translation unit.
+KERNEL_SRCS := act.c gemm.c norm.c rope.c attn.c threadpool.c generic.c avx.c neon.c
+
+BLAS_SRCS := backend_blas.c model.c model_blas.c forward.c late.c late_blas.c alloc.c \
+ $(COMMON_SRCS) $(KERNEL_SRCS)
+MLX_SRCS := backend_mlx.c mlx.c $(COMMON_SRCS)
+CUDA_SRCS := backend_cuda.c model.c late.c alloc.c $(COMMON_SRCS) $(KERNEL_SRCS)
+
+ifeq ($(BACKEND),mlx)
+ SRCS := $(MLX_SRCS)
+ CU_SRCS :=
+else ifeq ($(BACKEND),cuda)
+ SRCS := $(CUDA_SRCS)
+ CU_SRCS := cuda.cu
+else
+ SRCS := $(BLAS_SRCS)
+ CU_SRCS :=
+endif
+
+OBJS := $(call objs,$(SRCS) $(CU_SRCS))
+DEPS := $(call deps,$(OBJS))
+
+.PHONY: all clean
+
+all: $(STATIC) $(SHARED)
+
+$(eval $(call static_archive,$(STATIC),$(OBJS)))
+$(eval $(call shared_library,$(SHARED),$(OBJS),,$(BACKEND_LDFLAGS) -lm))
+# The export map is generated from the public ABI (the FFWD_API declarations in
+# ffwd.h) instead of being committed, so it cannot drift from the header. It is
+# regenerated when the header or the generator changes, and the shared library
+# relinks when it does.
+$(EXPORT_FILE): ffwd.h $(ROOT)/devtools/gen_exports.py
+ python3 $(ROOT)/devtools/gen_exports.py --outdir $(ROOT)
+$(SHARED): $(EXPORT_FILE)
+
+# Removes only this MODE/BACKEND's objects and artifacts; the root `make clean`
+# removes the whole build/ tree.
+clean:
+ $(RM) -r $(OBJDIR) $(STATIC) $(SHARED) $(SHARED).dSYM
+
+-include $(DEPS)
diff --git a/src/kernels_act.c b/libffwd/act.c
index abbade4..688dd71 100644
--- a/src/kernels_act.c
+++ b/libffwd/act.c
@@ -1,28 +1,32 @@
-#include "kernels.h"
-#include "kernels_impl.h"
+/*
+ * act.c - activations
+ */
#include <math.h>
#include <stdlib.h>
#include <string.h>
+#include "impl.h"
+#include "kernels.h"
+
#if (defined(__AVX512F__) || defined(__AVX2__)) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
+# include <immintrin.h>
#endif
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
+# include <arm_neon.h>
#endif
#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
+# ifdef __APPLE__
+# include <Accelerate/Accelerate.h>
+# else
+# include <cblas.h>
+# endif
#endif
#ifndef M_PI
-#define M_PI 3.14159265358979323846
+# define M_PI 3.14159265358979323846
#endif
/* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU
@@ -32,7 +36,7 @@
* Activation Functions
* ======================================================================== */
-void embed_silu_mul_inplace(float *gate, const float *up, int n) {
+void silu_mul_inplace(float *gate, const float *up, int n) {
#if defined(__APPLE__) && defined(USE_BLAS)
enum { CHUNK = 4096 };
float exp_neg[CHUNK];
@@ -59,13 +63,13 @@ void embed_silu_mul_inplace(float *gate, const float *up, int n) {
#endif
}
-void embed_gelu_inplace(float *x, int n) {
+void gelu_inplace(float *x, int n) {
/* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */
for (int i = 0; i < n; i++)
x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f));
}
-void embed_gelu_tanh_inplace(float *x, int n) {
+void gelu_tanh_inplace(float *x, int n) {
/* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */
for (int i = 0; i < n; i++) {
float v = x[i];
@@ -74,7 +78,7 @@ void embed_gelu_tanh_inplace(float *x, int n) {
}
}
-void embed_softmax(float *x, int rows, int cols) {
+void softmax(float *x, int rows, int cols) {
for (int r = 0; r < rows; r++) {
float *row = x + r * cols;
float max_val = row[0];
diff --git a/src/alloc.c b/libffwd/alloc.c
index 41a59a4..6af854a 100644
--- a/src/alloc.c
+++ b/libffwd/alloc.c
@@ -8,8 +8,8 @@
* growth, never allocate fewer than the minimum, and round up to the
* granularity so small length changes do not trigger a realloc on every call.
* Existing larger capacity is preserved. */
-#define EMBED_MIN_WORKSPACE_SEQ_CAP 16
-#define EMBED_WORKSPACE_SEQ_GRANULARITY 16
+#define FFWD_MIN_WORKSPACE_SEQ_CAP 16
+#define FFWD_WORKSPACE_SEQ_GRANULARITY 16
int mul_size(size_t a, size_t b, size_t *out) {
if (a != 0 && b > SIZE_MAX / a)
@@ -30,12 +30,12 @@ int grow_cap(int current, int needed, int *out) {
return -1;
int cap = needed;
- if (cap < EMBED_MIN_WORKSPACE_SEQ_CAP)
- cap = EMBED_MIN_WORKSPACE_SEQ_CAP;
+ if (cap < FFWD_MIN_WORKSPACE_SEQ_CAP)
+ cap = FFWD_MIN_WORKSPACE_SEQ_CAP;
- int rem = cap % EMBED_WORKSPACE_SEQ_GRANULARITY;
+ int rem = cap % FFWD_WORKSPACE_SEQ_GRANULARITY;
if (rem != 0) {
- int pad = EMBED_WORKSPACE_SEQ_GRANULARITY - rem;
+ int pad = FFWD_WORKSPACE_SEQ_GRANULARITY - rem;
if (cap > INT_MAX - pad)
return -1;
cap += pad;
diff --git a/src/alloc.h b/libffwd/alloc.h
index bc40e8b..0712e21 100644
--- a/src/alloc.h
+++ b/libffwd/alloc.h
@@ -1,5 +1,5 @@
-#ifndef EMBED_ALLOC_H
-#define EMBED_ALLOC_H
+#ifndef FFWD_ALLOC_H
+#define FFWD_ALLOC_H
#include <stddef.h>
@@ -31,4 +31,4 @@ int realloc_floats_2d(float **ptr, int rows, int cols);
/* realloc *ptr to hold `count` ints; *ptr unchanged on failure. */
int realloc_ints(int **ptr, size_t count);
-#endif /* EMBED_ALLOC_H */
+#endif /* FFWD_ALLOC_H */
diff --git a/libffwd/attn.c b/libffwd/attn.c
new file mode 100644
index 0000000..6134064
--- /dev/null
+++ b/libffwd/attn.c
@@ -0,0 +1,464 @@
+/*
+ * attn.c - attention
+ */
+
+#include "kernels.h"
+#include "impl.h"
+#include "threadpool.h"
+
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef USE_BLAS
+# ifdef __APPLE__
+# include <Accelerate/Accelerate.h>
+# else
+# include <cblas.h>
+# endif
+#endif
+
+#ifndef M_PI
+# define M_PI 3.14159265358979323846
+#endif
+
+/* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE)
+ * GQA, each with an online-softmax and a BLAS-GEMM path. */
+
+/* ========================================================================
+ * Attention Operations
+ * ======================================================================== */
+
+float dot_f32(const float *a, const float *b, int n) { return dot_f32_impl(a, b, n); }
+
+static inline float dot_f32_fast(const float *a, const float *b, int n) {
+ return dot_f32_impl(a, b, n);
+}
+
+/* dst = dst * scale */
+static inline void vec_scale_inplace(float *dst, float scale, int n) {
+ vec_scale_inplace_impl(dst, scale, n);
+}
+
+/* dst += alpha * src */
+static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
+ vec_axpy_inplace_impl(dst, src, alpha, n);
+}
+
+/* dst = dst * correction + src */
+static inline void vec_scale_add(float *dst, const float *src, float correction, int n) {
+ vec_scale_add_impl(dst, src, correction, n);
+}
+
+#define PACKED_ATTN_QUERY_TILE 32
+#define PACKED_ATTN_BLAS_MIN_SEQ 128
+
+static void bidirectional_gqa_attention_packed_online_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int end,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+
+ for (int i = query_start; i < query_end; i++) {
+ const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
+ float *o_row = out + (size_t)i * q_hidden + h * head_dim;
+
+ float max_score = -1e30f;
+ float sum_exp = 0.0f;
+ for (int d = 0; d < head_dim; d++)
+ o_row[d] = 0.0f;
+
+ for (int j = start; j < end; j++) {
+ const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
+ const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
+
+ float score = dot_f32_fast(q_row, k_row, head_dim) * scale;
+
+ if (score > max_score) {
+ float correction = expf(max_score - score);
+ sum_exp = sum_exp * correction + 1.0f;
+ vec_scale_add(o_row, v_row, correction, head_dim);
+ max_score = score;
+ } else {
+ float wt = expf(score - max_score);
+ sum_exp += wt;
+ vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ }
+ }
+
+ if (sum_exp > 0.0f) {
+ float inv_sum = 1.0f / sum_exp;
+ vec_scale_inplace(o_row, inv_sum, head_dim);
+ }
+ }
+}
+
+#ifdef USE_BLAS
+static void bidirectional_gqa_attention_packed_blas_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int end,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end,
+ float *scores) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+ int rows = query_end - query_start;
+ int keys = end - start;
+
+ const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
+ const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
+ const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
+ float *o = out + (size_t)query_start * q_hidden + h * head_dim;
+
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k,
+ kv_hidden, 0.0f, scores, keys);
+ softmax(scores, rows, keys);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
+ v, kv_hidden, 0.0f, o, q_hidden);
+}
+#endif
+
+typedef struct {
+ float *out;
+ const float *Q;
+ const float *K;
+ const float *V;
+ const int *offsets;
+ int batch;
+ int n_heads, n_kv_heads;
+ int head_dim;
+ float scale;
+ float *scores;
+ int max_seq;
+} packed_gqa_attn_task_t;
+
+static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
+ packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
+ int task_id = 0;
+#ifdef USE_BLAS
+ float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
+#endif
+
+ for (int h = 0; h < t->n_heads; h++) {
+ for (int b = 0; b < t->batch; b++) {
+ int start = t->offsets[b];
+ int end = t->offsets[b + 1];
+ for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) {
+ int q1 = q0 + PACKED_ATTN_QUERY_TILE;
+ if (q1 > end)
+ q1 = end;
+ if (task_id++ % n_threads != tid)
+ continue;
+#ifdef USE_BLAS
+ if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) {
+ bidirectional_gqa_attention_packed_blas_rows(
+ t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
+ t->scale, h, q0, q1, scores);
+ continue;
+ }
+#endif
+ bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end,
+ t->n_heads, t->n_kv_heads, t->head_dim,
+ t->scale, h, q0, q1);
+ }
+ }
+ }
+}
+
+size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
+#ifdef USE_BLAS
+ if (!offsets || batch <= 0)
+ return 0;
+
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ if (len > max_seq)
+ max_seq = len;
+ }
+ if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ)
+ return 0;
+ if ((size_t)max_seq >
+ SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads()))
+ return 0;
+ return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
+#else
+ (void)offsets;
+ (void)batch;
+ return 0;
+#endif
+}
+
+void bidirectional_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes) {
+ long long qk_work = 0;
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ qk_work += (long long)len * len * n_heads;
+ if (len > max_seq)
+ max_seq = len;
+ }
+
+ float *scores = NULL;
+#ifdef USE_BLAS
+ size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required)
+ scores = scratch;
+#else
+ (void)scratch;
+ (void)scratch_bytes;
+#endif
+ packed_gqa_attn_task_t task = {.out = out,
+ .Q = Q,
+ .K = K,
+ .V = V,
+ .offsets = offsets,
+ .batch = batch,
+ .n_heads = n_heads,
+ .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim,
+ .scale = scale,
+ .scores = scores,
+ .max_seq = max_seq};
+ if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) {
+ parallel_for(packed_gqa_attn_worker, &task);
+ } else {
+ packed_gqa_attn_worker(0, 1, &task);
+ }
+}
+
+void bidirectional_gqa_attention_packed(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale) {
+ size_t scratch_bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
+ bidirectional_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
+ head_dim, scale, scores, scratch_bytes);
+ free(scores);
+}
+
+static void causal_gqa_attention_packed_online_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+
+ for (int i = query_start; i < query_end; i++) {
+ const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
+ float *o_row = out + (size_t)i * q_hidden + h * head_dim;
+
+ float max_score = -1e30f;
+ float sum_exp = 0.0f;
+ for (int d = 0; d < head_dim; d++)
+ o_row[d] = 0.0f;
+
+ for (int j = start; j <= i; j++) {
+ const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
+ const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
+ float score = dot_f32_fast(q_row, k_row, head_dim) * scale;
+
+ if (score > max_score) {
+ float correction = expf(max_score - score);
+ sum_exp = sum_exp * correction + 1.0f;
+ vec_scale_add(o_row, v_row, correction, head_dim);
+ max_score = score;
+ } else {
+ float wt = expf(score - max_score);
+ sum_exp += wt;
+ vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ }
+ }
+
+ if (sum_exp > 0.0f)
+ vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim);
+ }
+}
+
+#ifdef USE_BLAS
+static void causal_gqa_attention_packed_blas_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int start,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end,
+ float *scores) {
+ int heads_per_kv = n_heads / n_kv_heads;
+ int q_hidden = n_heads * head_dim;
+ int kv_hidden = n_kv_heads * head_dim;
+ int kv_h = h / heads_per_kv;
+ int rows = query_end - query_start;
+ int keys = query_end - start;
+
+ const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
+ const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
+ const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
+ float *o = out + (size_t)query_start * q_hidden + h * head_dim;
+
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k,
+ kv_hidden, 0.0f, scores, keys);
+ for (int r = 0; r < rows; r++) {
+ int allowed = query_start + r - start + 1;
+ float *row = scores + (size_t)r * keys;
+ for (int j = allowed; j < keys; j++)
+ row[j] = -INFINITY;
+ }
+ softmax(scores, rows, keys);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
+ v, kv_hidden, 0.0f, o, q_hidden);
+}
+#endif
+
+static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) {
+ packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
+ int task_id = 0;
+#ifdef USE_BLAS
+ float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
+#endif
+
+ for (int h = 0; h < t->n_heads; h++) {
+ for (int b = 0; b < t->batch; b++) {
+ int start = t->offsets[b];
+ int end = t->offsets[b + 1];
+ for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) {
+ int q1 = q0 + PACKED_ATTN_QUERY_TILE;
+ if (q1 > end)
+ q1 = end;
+ if (task_id++ % n_threads != tid)
+ continue;
+#ifdef USE_BLAS
+ if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) {
+ causal_gqa_attention_packed_blas_rows(t->out, t->Q, t->K, t->V, start, t->n_heads,
+ t->n_kv_heads, t->head_dim, t->scale, h, q0,
+ q1, scores);
+ continue;
+ }
+#endif
+ causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, t->n_heads,
+ t->n_kv_heads, t->head_dim, t->scale, h, q0,
+ q1);
+ }
+ }
+ }
+}
+
+void causal_gqa_attention_packed_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes) {
+ long long qk_work = 0;
+ int max_seq = 0;
+ for (int b = 0; b < batch; b++) {
+ int len = offsets[b + 1] - offsets[b];
+ qk_work += (long long)len * (len + 1) / 2 * n_heads;
+ if (len > max_seq)
+ max_seq = len;
+ }
+
+ float *scores = NULL;
+#ifdef USE_BLAS
+ size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (required != 0 && scratch && scratch_bytes >= required)
+ scores = scratch;
+#else
+ (void)scratch;
+ (void)scratch_bytes;
+#endif
+ packed_gqa_attn_task_t task = {.out = out,
+ .Q = Q,
+ .K = K,
+ .V = V,
+ .offsets = offsets,
+ .batch = batch,
+ .n_heads = n_heads,
+ .n_kv_heads = n_kv_heads,
+ .head_dim = head_dim,
+ .scale = scale,
+ .scores = scores,
+ .max_seq = max_seq};
+ if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096)
+ parallel_for(packed_causal_gqa_attn_worker, &task);
+ else
+ packed_causal_gqa_attn_worker(0, 1, &task);
+}
+
+void causal_gqa_attention_packed(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ const int *offsets,
+ int batch,
+ int n_heads,
+ int n_kv_heads,
+ int head_dim,
+ float scale) {
+ size_t scratch_bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
+ causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
+ head_dim, scale, scores, scratch_bytes);
+ free(scores);
+}
diff --git a/src/kernels_avx.c b/libffwd/avx.c
index 920fecf..41259a1 100644
--- a/src/kernels_avx.c
+++ b/libffwd/avx.c
@@ -1,5 +1,5 @@
/*
- * embed_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available)
+ * avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available)
*
* bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16.
* Uses AVX-512F+BW for 16-wide bf16 matvec (dominant for seq_len==1), and
@@ -9,20 +9,20 @@
* overhead and improve out-of-order execution on memory-bound workloads.
*/
-#include "kernels_impl.h"
+#include "impl.h"
#if defined(__AVX2__) && defined(__FMA__)
-#include <immintrin.h>
-#include <string.h>
+# include <immintrin.h>
+# include <string.h>
/* =====================================================================
* BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise
* ===================================================================== */
-#if defined(__AVX512F__) && defined(__AVX512BW__)
+# if defined(__AVX512F__) && defined(__AVX512BW__)
-void embed_bf16_matvec_fused_avx(
+void bf16_matvec_fused_avx(
float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
int o = 0;
@@ -188,7 +188,7 @@ void embed_bf16_matvec_fused_avx(
}
}
-#else /* AVX2 only (or AVX-512F without BW) */
+# else /* AVX2 only (or AVX-512F without BW) */
/* Helper: bf16 to f32 for 8 elements */
static inline __m256 bf16x8_to_f32(__m128i raw) {
@@ -201,7 +201,7 @@ static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) {
*hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1));
}
-void embed_bf16_matvec_fused_avx(
+void bf16_matvec_fused_avx(
float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
int o = 0;
@@ -266,174 +266,6 @@ void embed_bf16_matvec_fused_avx(
}
}
-#endif /* AVX-512F+BW vs AVX2 for bf16 */
-
-/* =====================================================================
- * f32 attention helpers - AVX2+FMA, with AVX-512F when available
- * (operates on L1-resident head vectors)
- * ===================================================================== */
-
-float embed_dot_f32_avx(const float *a, const float *b, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 acc0 = _mm512_setzero_ps();
- __m512 acc1 = _mm512_setzero_ps();
- __m512 acc2 = _mm512_setzero_ps();
- __m512 acc3 = _mm512_setzero_ps();
- for (; i + 64 <= n; i += 64) {
- acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
- acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
- acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
- acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
- }
- __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3));
- for (; i + 16 <= n; i += 16) {
- acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
- }
- float sum = _mm512_reduce_add_ps(acc);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-#else
- int i = 0;
- __m256 acc0 = _mm256_setzero_ps();
- __m256 acc1 = _mm256_setzero_ps();
- __m256 acc2 = _mm256_setzero_ps();
- __m256 acc3 = _mm256_setzero_ps();
- for (; i + 32 <= n; i += 32) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1);
- acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2);
- acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3);
- }
- acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
- for (; i + 8 <= n; i += 8) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- }
- __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
- r = _mm_hadd_ps(r, r);
- r = _mm_hadd_ps(r, r);
- float sum = _mm_cvtss_f32(r);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-#endif
-}
-
-void embed_vec_scale_inplace_avx(float *dst, float scale, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 s = _mm512_set1_ps(scale);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
- _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
- _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-#else
- int i = 0;
- __m256 s = _mm256_set1_ps(scale);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s));
- _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s));
- _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-#endif
-}
-
-void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 a = _mm512_set1_ps(alpha);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a,
- _mm512_loadu_ps(dst + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a,
- _mm512_loadu_ps(dst + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a,
- _mm512_loadu_ps(dst + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-#else
- int i = 0;
- __m256 a = _mm256_set1_ps(alpha);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a,
- _mm256_loadu_ps(dst + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a,
- _mm256_loadu_ps(dst + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a,
- _mm256_loadu_ps(dst + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-#endif
-}
-
-void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 c = _mm512_set1_ps(correction);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c,
- _mm512_loadu_ps(src + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c,
- _mm512_loadu_ps(src + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c,
- _mm512_loadu_ps(src + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-#else
- int i = 0;
- __m256 c = _mm256_set1_ps(correction);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c,
- _mm256_loadu_ps(src + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c,
- _mm256_loadu_ps(src + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c,
- _mm256_loadu_ps(src + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-#endif
-}
+# endif /* AVX-512F+BW vs AVX2 for bf16 */
#endif /* __AVX2__ && __FMA__ */
diff --git a/libffwd/backend_blas.c b/libffwd/backend_blas.c
new file mode 100644
index 0000000..edf54bb
--- /dev/null
+++ b/libffwd/backend_blas.c
@@ -0,0 +1,168 @@
+/* backend_blas.c - CPU backend (BLAS), implements the engine API in ffwd.h.
+ * Linked only into CPU builds. The CPU model loads on the caller's thread, so
+ * ffwd_open does the full load and activate/worker_free are no-ops. */
+#include "internal.h"
+#include "threadpool.h"
+#include "platform.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+struct embed {
+ int is_late;
+ ffwd_model_t *model;
+ ffwd_workspace_t *ws;
+ ffwd_late_model_t *late;
+ ffwd_late_workspace_t *late_ws;
+};
+
+const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; }
+
+void ffwd_cli_help(FILE *f) {
+ fputs(" -t, --threads N\n"
+ " CPU threads (default: available cores, cgroup-aware)\n",
+ f);
+}
+
+void ffwd_server_help(FILE *f) {
+ fputs(" -t, --threads N CPU threads (default: available cores, cgroup-aware)\n", f);
+}
+
+int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
+ (void)err;
+ (void)errlen;
+ int n = opts->n_threads > 0 ? opts->n_threads : tp_get_num_cpus();
+ tp_set_threads(n);
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "Using %d CPU thread(s)\n", n);
+ return 0;
+}
+
+ffwd_t *ffwd_open(
+ const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
+ (void)opts;
+ ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b));
+ if (!b) {
+ snprintf(err, errlen, "out of memory");
+ return NULL;
+ }
+ b->is_late = is_late;
+ if (is_late) {
+ b->late = ffwd_late_model_load(model_dir);
+ if (!b->late) {
+ snprintf(err, errlen, "failed to load late model");
+ free(b);
+ return NULL;
+ }
+ b->late_ws = ffwd_late_workspace_new(b->late);
+ if (!b->late_ws) {
+ snprintf(err, errlen, "failed to allocate late workspace");
+ ffwd_late_model_free(b->late);
+ free(b);
+ return NULL;
+ }
+ } else {
+ b->model = ffwd_model_load(model_dir);
+ if (!b->model) {
+ snprintf(err, errlen, "failed to load model");
+ free(b);
+ return NULL;
+ }
+ b->ws = ffwd_workspace_new(b->model);
+ if (!b->ws) {
+ snprintf(err, errlen, "failed to allocate workspace");
+ ffwd_model_free(b->model);
+ free(b);
+ return NULL;
+ }
+ }
+ return b;
+}
+
+int ffwd_activate(ffwd_t *b, char *err, size_t errlen) {
+ (void)b;
+ (void)err;
+ (void)errlen;
+ return 0;
+}
+
+void ffwd_worker_free(ffwd_t *b) { (void)b; }
+
+void ffwd_free(ffwd_t *b) {
+ if (!b)
+ return;
+ ffwd_workspace_free(b->ws);
+ ffwd_model_free(b->model);
+ ffwd_late_workspace_free(b->late_ws);
+ ffwd_late_model_free(b->late);
+ free(b);
+}
+
+const ffwd_config_t *ffwd_config(const ffwd_t *b) {
+ return b->is_late ? ffwd_late_model_config(b->late) : ffwd_model_config(b->model);
+}
+
+int ffwd_token_dim(const ffwd_t *b) {
+ return b->is_late ? ffwd_late_model_token_dim(b->late) : 0;
+}
+
+int ffwd_uses_dense_batches(const ffwd_t *b) {
+ (void)b;
+ return 0;
+}
+
+int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) {
+ return ffwd_model_encode_batch(b->model, b->ws, inputs, batch, out);
+}
+
+int ffwd_encode_spans_batch(ffwd_t *b,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ return ffwd_model_encode_spans_batch(b->model, b->ws, inputs, batch, out);
+}
+
+int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) {
+ int dim = ffwd_late_model_token_dim(b->late);
+ int total_doc_vecs = 0;
+ for (int i = 0; i < in->n_docs; i++)
+ total_doc_vecs += in->doc_n_keep[i];
+
+ float *query = (float *)malloc((size_t)in->query_n_tokens * dim * sizeof(*query));
+ float *docs = (float *)malloc((size_t)total_doc_vecs * dim * sizeof(*docs));
+ int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets));
+ int rc = (query && docs && offsets) ? 0 : -1;
+
+ /* Encode every candidate in one block-diagonal forward: the encoder packs
+ * each document's kept token vectors back-to-back and fills offsets. */
+ if (rc == 0)
+ rc = ffwd_late_model_encode_tokens(b->late, b->late_ws, in->query_ids, in->query_n_tokens,
+ 1, query);
+ if (rc == 0)
+ rc = ffwd_late_model_encode_docs(b->late, b->late_ws, in->doc_ids, in->doc_n_tokens,
+ in->doc_keep, in->doc_n_keep, in->n_docs, 1, docs,
+ offsets);
+ if (rc == 0)
+ rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim,
+ scores);
+ free(offsets);
+ free(docs);
+ free(query);
+ return rc;
+}
+
+size_t ffwd_scratch_nbytes(const ffwd_t *b) {
+ return (b && b->ws) ? ffwd_workspace_nbytes(b->ws) : 0;
+}
+
+int ffwd_default_batch_wait_us(void) { return 0; }
+
+int ffwd_preflight(
+ const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) {
+ (void)paths;
+ (void)n_paths;
+ (void)opts;
+ (void)err;
+ (void)errlen;
+ return 0;
+}
diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c
new file mode 100644
index 0000000..db39345
--- /dev/null
+++ b/libffwd/backend_cuda.c
@@ -0,0 +1,172 @@
+/* backend_cuda.c - NVIDIA GPU backend (CUDA), implements the engine API in ffwd.h.
+ * Linked only into CUDA builds. The context is owned by the inference thread:
+ * open records what to load, activate creates the context, worker_free destroys
+ * it. CUDA encodes on the device and scores MaxSim on the host. */
+#include "internal.h"
+#include "cuda.h"
+#include "platform.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct embed {
+ int is_late;
+ char *dir;
+ ffwd_cuda_ctx_t *ctx;
+ ffwd_cuda_late_ctx_t *late_ctx;
+};
+
+const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; }
+
+void ffwd_cli_help(FILE *f) {
+ fputs(" --gpu-gemm-mode MODE\n"
+ " GPU GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
+ " --gpu-weight-dtype DTYPE\n"
+ " GPU weight storage: f32 or bf16 (default: f32). bf16 halves\n"
+ " weight memory and uses BF16 tensor cores\n",
+ f);
+}
+
+void ffwd_server_help(FILE *f) {
+ fputs(" --gpu-gemm-mode MODE GPU GEMM compute: f32, tf32, bf16, or 16f\n"
+ " (default: f32, exact)\n"
+ " --gpu-weight-dtype DTYPE GPU weight storage: f32 or bf16 (default:\n"
+ " f32); bf16 halves weight memory\n",
+ f);
+}
+
+int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
+ if (opts->gpu_gemm_mode && ffwd_cuda_set_fast_gemm(opts->gpu_gemm_mode) != 0) {
+ snprintf(err, errlen, "--gpu-gemm-mode must be f32, tf32, bf16, or 16f");
+ return -1;
+ }
+ if (opts->gpu_weight_dtype) {
+ if (!strcmp(opts->gpu_weight_dtype, "bf16"))
+ ffwd_cuda_set_weights_bf16(1);
+ else if (!strcmp(opts->gpu_weight_dtype, "f32"))
+ ffwd_cuda_set_weights_bf16(0);
+ else {
+ snprintf(err, errlen, "--gpu-weight-dtype must be f32 or bf16");
+ return -1;
+ }
+ }
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "Using %s\n", FFWD_BACKEND_LABEL);
+ return 0;
+}
+
+ffwd_t *ffwd_open(
+ const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
+ (void)opts;
+ ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b));
+ if (!b || !(b->dir = strdup(model_dir))) {
+ snprintf(err, errlen, "out of memory");
+ free(b);
+ return NULL;
+ }
+ b->is_late = is_late;
+ return b;
+}
+
+int ffwd_activate(ffwd_t *b, char *err, size_t errlen) {
+ if (b->is_late)
+ b->late_ctx = ffwd_cuda_late_load(b->dir);
+ else
+ b->ctx = ffwd_cuda_load(b->dir);
+ if (!b->ctx && !b->late_ctx) {
+ snprintf(err, errlen, "failed to load model");
+ return -1;
+ }
+ return 0;
+}
+
+void ffwd_worker_free(ffwd_t *b) {
+ if (!b)
+ return;
+ if (b->ctx) {
+ ffwd_cuda_free(b->ctx);
+ b->ctx = NULL;
+ }
+ if (b->late_ctx) {
+ ffwd_cuda_late_free(b->late_ctx);
+ b->late_ctx = NULL;
+ }
+}
+
+void ffwd_free(ffwd_t *b) {
+ if (!b)
+ return;
+ ffwd_worker_free(b);
+ free(b->dir);
+ free(b);
+}
+
+const ffwd_config_t *ffwd_config(const ffwd_t *b) {
+ return b->is_late ? ffwd_cuda_late_config(b->late_ctx) : ffwd_cuda_config(b->ctx);
+}
+
+int ffwd_token_dim(const ffwd_t *b) {
+ return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0;
+}
+
+int ffwd_uses_dense_batches(const ffwd_t *b) {
+ (void)b;
+ return 0;
+}
+
+int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) {
+ return ffwd_cuda_encode_batch(b->ctx, inputs, batch, out);
+}
+
+int ffwd_encode_spans_batch(ffwd_t *b,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ return ffwd_cuda_encode_spans_batch(b->ctx, inputs, batch, out);
+}
+
+int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) {
+ int dim = ffwd_cuda_late_token_dim(b->late_ctx);
+ int total_doc_vecs = 0;
+ for (int i = 0; i < in->n_docs; i++)
+ total_doc_vecs += in->doc_n_keep[i];
+
+ float *query = (float *)malloc((size_t)in->query_n_tokens * dim * sizeof(*query));
+ float *docs = (float *)malloc((size_t)total_doc_vecs * dim * sizeof(*docs));
+ int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets));
+ int rc = (query && docs && offsets) ? 0 : -1;
+
+ /* GPU encodes the query and every candidate in one packed forward; MaxSim
+ * runs on the host, where the grouped-GEMM scorer beats a device graph. */
+ if (rc == 0)
+ rc =
+ ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query);
+ if (rc == 0)
+ rc = ffwd_cuda_late_encode_docs(b->late_ctx, in->doc_ids, in->doc_n_tokens, in->doc_keep,
+ in->doc_n_keep, in->n_docs, 1, docs, offsets);
+ if (rc == 0)
+ rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim,
+ scores);
+ free(offsets);
+ free(docs);
+ free(query);
+ return rc;
+}
+
+size_t ffwd_scratch_nbytes(const ffwd_t *b) {
+ (void)b;
+ return 0;
+}
+
+int ffwd_default_batch_wait_us(void) { return 1000; }
+
+int ffwd_preflight(
+ const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) {
+ (void)paths;
+ (void)n_paths;
+ (void)opts;
+ (void)err;
+ (void)errlen;
+ return 0;
+}
diff --git a/libffwd/backend_mlx.c b/libffwd/backend_mlx.c
new file mode 100644
index 0000000..80b893c
--- /dev/null
+++ b/libffwd/backend_mlx.c
@@ -0,0 +1,228 @@
+/* backend_mlx.c - Apple GPU backend (MLX), implements the engine API in ffwd.h.
+ * Linked only into Apple Silicon GPU builds. MLX streams are thread-local, so
+ * the context is created and destroyed on the inference thread: open only
+ * records what to load, activate creates the context, worker_free destroys it. */
+#include "internal.h"
+#include "mlx.h"
+#include "platform.h"
+#include "safetensors.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#ifdef __APPLE__
+# include <sys/sysctl.h>
+#endif
+
+/* Peak resident memory is estimated as the on-disk tensor payload times this
+ * factor; the preflight refuses a model set whose estimate exceeds the budget. */
+#define FFWD_MLX_RESIDENT_MULTIPLIER 2
+#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90
+
+struct embed {
+ int is_late;
+ char *dir;
+ ffwd_options_t opts;
+ ffwd_mlx_ctx_t *ctx;
+ ffwd_mlx_late_ctx_t *late_ctx;
+};
+
+const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; }
+
+void ffwd_cli_help(FILE *f) {
+ fputs(" --gpu-quant-bits N\n"
+ " Quantize GPU linear weights to 8 bits at load time\n"
+ " --gpu-quant-group-size N\n"
+ " GPU quantization group size (default: 64)\n",
+ f);
+}
+
+void ffwd_server_help(FILE *f) {
+ fputs(" --memory-utilization F Fraction of physical memory the GPU model-set\n"
+ " preflight may plan for (default: 0.90; values\n"
+ " above 1.0 overcommit)\n"
+ " --gpu-quant-bits N Quantize GPU linear weights to 8 bits at load\n"
+ " --gpu-quant-group-size N GPU quantization group size (default: 64)\n",
+ f);
+}
+
+int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
+ if (opts->gpu_quant_bits != 0 && opts->gpu_quant_bits != 8) {
+ snprintf(err, errlen, "--gpu-quant-bits must be 0 or 8");
+ return -1;
+ }
+ if (opts->gpu_quant_group_size <= 0) {
+ snprintf(err, errlen, "--gpu-quant-group-size must be > 0");
+ return -1;
+ }
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "Using %s\n", FFWD_BACKEND_LABEL);
+ return 0;
+}
+
+ffwd_t *ffwd_open(
+ const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
+ ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b));
+ if (!b || !(b->dir = strdup(model_dir))) {
+ snprintf(err, errlen, "out of memory");
+ free(b);
+ return NULL;
+ }
+ b->is_late = is_late;
+ b->opts = *opts;
+ return b;
+}
+
+int ffwd_activate(ffwd_t *b, char *err, size_t errlen) {
+ ffwd_mlx_options_t mlx_opts = {
+ .quantize_bits = b->opts.gpu_quant_bits,
+ .quantize_group_size = b->opts.gpu_quant_group_size,
+ };
+ if (b->is_late)
+ b->late_ctx = ffwd_mlx_late_load_with_options(b->dir, &mlx_opts);
+ else
+ b->ctx = ffwd_mlx_load_with_options(b->dir, &mlx_opts);
+ if (!b->ctx && !b->late_ctx) {
+ snprintf(err, errlen, "failed to load model");
+ return -1;
+ }
+ return 0;
+}
+
+void ffwd_worker_free(ffwd_t *b) {
+ if (!b)
+ return;
+ if (b->ctx) {
+ ffwd_mlx_free(b->ctx);
+ b->ctx = NULL;
+ }
+ if (b->late_ctx) {
+ ffwd_mlx_late_free(b->late_ctx);
+ b->late_ctx = NULL;
+ }
+}
+
+void ffwd_free(ffwd_t *b) {
+ if (!b)
+ return;
+ /* The inference thread releases the context via ffwd_worker_free;
+ * by the time the main thread frees the handle both are already NULL. The
+ * single-threaded CLI loads and frees on one thread, so this also releases
+ * a context that worker_free never ran on. */
+ ffwd_worker_free(b);
+ free(b->dir);
+ free(b);
+}
+
+const ffwd_config_t *ffwd_config(const ffwd_t *b) {
+ return b->is_late ? ffwd_mlx_late_config(b->late_ctx) : ffwd_mlx_config(b->ctx);
+}
+
+int ffwd_token_dim(const ffwd_t *b) {
+ return b->is_late ? ffwd_mlx_late_token_dim(b->late_ctx) : 0;
+}
+
+int ffwd_uses_dense_batches(const ffwd_t *b) { return b->ctx != NULL; }
+
+int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) {
+ return ffwd_mlx_encode_batch(b->ctx, inputs, batch, out);
+}
+
+int ffwd_encode_spans_batch(ffwd_t *b,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ return ffwd_mlx_encode_spans_batch(b->ctx, inputs, batch, out);
+}
+
+int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) {
+ int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets));
+ ffwd_mlx_late_vectors_t *query =
+ ffwd_mlx_late_encode_tokens_device(b->late_ctx, in->query_ids, in->query_n_tokens, 1);
+ ffwd_mlx_late_vectors_t *packed =
+ (query && offsets)
+ ? ffwd_mlx_late_encode_docs_device(b->late_ctx, in->doc_ids, in->doc_n_tokens,
+ in->doc_keep, in->doc_n_keep, in->n_docs, 1,
+ offsets)
+ : NULL;
+ int rc = packed ? ffwd_mlx_late_maxsim_batch_device(b->late_ctx, query, packed, offsets,
+ in->n_docs, scores)
+ : -1;
+ ffwd_mlx_late_vectors_free(packed);
+ ffwd_mlx_late_vectors_free(query);
+ free(offsets);
+ return rc;
+}
+
+size_t ffwd_scratch_nbytes(const ffwd_t *b) {
+ (void)b;
+ return 0;
+}
+
+int ffwd_default_batch_wait_us(void) { return 0; }
+
+static uint64_t physical_memory_nbytes(void) {
+#ifdef __APPLE__
+ uint64_t bytes = 0;
+ size_t len = sizeof(bytes);
+ if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0)
+ return bytes;
+#endif
+ long pages = sysconf(_SC_PHYS_PAGES);
+ long page_size = sysconf(_SC_PAGESIZE);
+ if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
+ return 0;
+ return (uint64_t)pages * (uint64_t)page_size;
+}
+
+int ffwd_preflight(
+ const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) {
+ uint64_t payload = 0;
+ for (int i = 0; i < n_paths; i++) {
+ multi_safetensors_t *ms = multi_safetensors_open(paths[i]);
+ size_t model_payload = 0;
+ if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) {
+ multi_safetensors_close(ms);
+ snprintf(err, errlen, "failed to inspect GPU model: %s", paths[i]);
+ return -1;
+ }
+ multi_safetensors_close(ms);
+ if ((uint64_t)model_payload > UINT64_MAX - payload) {
+ snprintf(err, errlen, "GPU model payload size overflow");
+ return -1;
+ }
+ payload += (uint64_t)model_payload;
+ }
+ if (payload > UINT64_MAX / FFWD_MLX_RESIDENT_MULTIPLIER) {
+ snprintf(err, errlen, "GPU resident-memory estimate overflow");
+ return -1;
+ }
+ uint64_t estimated = payload * FFWD_MLX_RESIDENT_MULTIPLIER;
+ uint64_t physical = physical_memory_nbytes();
+ if (physical == 0) {
+ fprintf(stderr, "ffwd-server: warning: could not determine physical memory; "
+ "skipping GPU memory preflight\n");
+ return 0;
+ }
+ double utilization = opts->memory_utilization > 0.0 ? opts->memory_utilization
+ : FFWD_MLX_MEMORY_BUDGET_PERCENT / 100.0;
+ uint64_t budget = (uint64_t)((double)physical * utilization);
+ const double gib = 1024.0 * 1024.0 * 1024.0;
+ fprintf(stderr,
+ "ffwd-server: GPU memory preflight: %.1f GiB tensors, %.1f GiB estimated "
+ "resident, %.1f GiB budget (%.2f of physical memory)\n",
+ (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization);
+ if (opts->gpu_quant_bits)
+ fprintf(stderr,
+ "ffwd-server: GPU %d-bit quantization enabled; preflight uses source "
+ "payload as a conservative peak estimate\n",
+ opts->gpu_quant_bits);
+ if (estimated <= budget)
+ return 0;
+ snprintf(err, errlen,
+ "ffwd-server: GPU model set exceeds the host-memory budget; use BF16 artifacts, "
+ "load fewer models, or raise --memory-utilization");
+ return -1;
+}
diff --git a/src/tokenizer_bpe.c b/libffwd/bpe.c
index 511b29e..c74845f 100644
--- a/src/tokenizer_bpe.c
+++ b/libffwd/bpe.c
@@ -1,5 +1,4 @@
-#include "tokenizer_bpe.h"
-#include "kernels.h"
+#include "bpe.h"
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
@@ -419,8 +418,7 @@ static uint64_t pair_hash(uint64_t k) {
return k;
}
-static void
-pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) {
+static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) {
int mask = cap - 1;
int idx = (int)(pair_hash(key) & (uint64_t)mask);
for (int probe = 0; probe < cap; probe++, idx = (idx + 1) & mask) {
@@ -437,7 +435,7 @@ pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int me
/* Returns the rank and sets *merged_id, or INT_MAX when the pair never
* merges. */
-static int pair_merge_lookup(const embed_tokenizer_t *tok, int left, int right, int *merged_id) {
+static int pair_merge_lookup(const tok_bpe_t *tok, int left, int right, int *merged_id) {
const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges;
uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right;
int mask = tok->int_merges_cap - 1;
@@ -453,7 +451,7 @@ static int pair_merge_lookup(const embed_tokenizer_t *tok, int left, int right,
return INT_MAX;
}
-static int merge_rank(const embed_tokenizer_t *tok, const char *a, const char *b) {
+static int merge_rank(const tok_bpe_t *tok, const char *a, const char *b) {
if (!tok->merge_map || tok->merge_map_cap <= 0)
return INT_MAX;
@@ -495,8 +493,7 @@ static char *text_to_bpe_unicode(const char *text) {
return out;
}
-static int
-encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n);
+static int encode_bpe_word(const tok_bpe_t *tok, const char *mapped, int **out_ids, int *out_n);
static int
utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) {
@@ -828,12 +825,8 @@ static int match_contraction(const unsigned char *p, const unsigned char *end) {
return 0;
}
-static int append_encoded_piece(const embed_tokenizer_t *tok,
- const char *piece,
- size_t piece_len,
- int **ids,
- int *n_ids,
- int *cap) {
+static int append_encoded_piece(
+ const tok_bpe_t *tok, const char *piece, size_t piece_len, int **ids, int *n_ids, int *cap) {
if (piece_len == 0)
return 0;
@@ -867,8 +860,7 @@ static int append_encoded_piece(const embed_tokenizer_t *tok,
return 0;
}
-static int
-encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) {
+static int encode_pretokenized(const tok_bpe_t *tok, const char *text, int **out_ids, int *out_n) {
const unsigned char *start = (const unsigned char *)text;
const unsigned char *p = start;
const unsigned char *end = start + strlen(text);
@@ -987,8 +979,8 @@ encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_id
}
}
- if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids,
- &cap) != 0) {
+ if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids, &cap) !=
+ 0) {
free(ids);
return -1;
}
@@ -1000,8 +992,7 @@ encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_id
}
/* Encode one mapped BPE unicode string to token IDs. */
-static int
-encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) {
+static int encode_bpe_word(const tok_bpe_t *tok, const char *mapped, int **out_ids, int *out_n) {
*out_ids = NULL;
*out_n = 0;
if (!mapped || !*mapped)
@@ -1078,7 +1069,7 @@ typedef struct {
size_t off;
} bpe_sym_ref_t;
-struct embed_tokenizer_workspace {
+struct tok_bpe_workspace {
char *normalized;
size_t normalized_cap;
char *mapped;
@@ -1101,7 +1092,7 @@ struct embed_tokenizer_workspace {
int bpe_cap;
};
-static int reserve_bpe(embed_tokenizer_workspace_t *ws, int need) {
+static int reserve_bpe(tok_bpe_workspace_t *ws, int need) {
if (ws->bpe_cap >= need)
return 0;
int new_cap = ws->bpe_cap ? ws->bpe_cap : 64;
@@ -1122,7 +1113,7 @@ static int reserve_bpe(embed_tokenizer_workspace_t *ws, int need) {
R(bpe_rnk)
R(bpe_mrg)
#undef R
- ws->bpe_cap = new_cap;
+ ws->bpe_cap = new_cap;
return 0;
}
@@ -1160,7 +1151,7 @@ static int reserve_ints(int **buf, int *cap, int need) {
return 0;
}
-static int reserve_syms(embed_tokenizer_workspace_t *ws, int need) {
+static int reserve_syms(tok_bpe_workspace_t *ws, int need) {
if (ws->syms_cap >= need)
return 0;
int new_cap = ws->syms_cap ? ws->syms_cap : 64;
@@ -1177,7 +1168,7 @@ static int reserve_syms(embed_tokenizer_workspace_t *ws, int need) {
return 0;
}
-static int arena_append(embed_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) {
+static int arena_append(tok_bpe_workspace_t *ws, const char *s, size_t len, size_t *off) {
if (ws->arena_len > SIZE_MAX - len - 1)
return -1;
size_t need = ws->arena_len + len + 1;
@@ -1190,12 +1181,11 @@ static int arena_append(embed_tokenizer_workspace_t *ws, const char *s, size_t l
return 0;
}
-static const char *arena_str(const embed_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) {
+static const char *arena_str(const tok_bpe_workspace_t *ws, bpe_sym_ref_t sym) {
return ws->arena + sym.off;
}
-static int
-normalize_nfc_latin_into(embed_tokenizer_workspace_t *ws, const char *text, const char **out) {
+static int normalize_nfc_latin_into(tok_bpe_workspace_t *ws, const char *text, const char **out) {
const unsigned char *p = (const unsigned char *)text;
const unsigned char *end = p + strlen(text);
size_t in_len = (size_t)(end - p);
@@ -1235,10 +1225,8 @@ normalize_nfc_latin_into(embed_tokenizer_workspace_t *ws, const char *text, cons
return 0;
}
-static int text_to_bpe_unicode_into(embed_tokenizer_workspace_t *ws,
- const char *text,
- size_t len,
- const char **out) {
+static int
+text_to_bpe_unicode_into(tok_bpe_workspace_t *ws, const char *text, size_t len, const char **out) {
init_gpt2_mapping();
if (len > (SIZE_MAX - 1) / 3)
return -1;
@@ -1267,8 +1255,8 @@ static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow
return 0;
}
-static int encode_bpe_word_into_slow(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
+static int encode_bpe_word_into_slow(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
const char *mapped,
int *out_ids,
int out_cap,
@@ -1349,8 +1337,8 @@ static int encode_bpe_word_into_slow(const embed_tokenizer_t *tok,
* the string-based loop when the int tables are unavailable. Identical
* output: initial ids are the byte-level single-char token ids and every
* merge result is the vocab id of the concatenated pair. */
-static int encode_bpe_word_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
+static int encode_bpe_word_into(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
const char *mapped,
int *out_ids,
int out_cap,
@@ -1418,8 +1406,8 @@ static int encode_bpe_word_into(const embed_tokenizer_t *tok,
if (prv[best] != -1)
rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], &mrg[prv[best]]);
- rnk[best] = nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best])
- : INT_MAX;
+ rnk[best] =
+ nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) : INT_MAX;
}
for (int j = 0; j != -1; j = nxt[j])
@@ -1427,8 +1415,8 @@ static int encode_bpe_word_into(const embed_tokenizer_t *tok,
return 0;
}
-static int append_encoded_piece_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
+static int append_encoded_piece_into(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
const char *piece,
size_t piece_len,
int *out_ids,
@@ -1444,8 +1432,8 @@ static int append_encoded_piece_into(const embed_tokenizer_t *tok,
return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow);
}
-static int encode_pretokenized_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
+static int encode_pretokenized_into(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
const char *text,
int *out_ids,
int out_cap,
@@ -1622,7 +1610,7 @@ static int parse_merge_pair(char *line, char **a, char **b) {
return 0;
}
-static int load_merges_map(embed_tokenizer_t *tok, const char *merges_path) {
+static int load_merges_map(tok_bpe_t *tok, const char *merges_path) {
FILE *f = fopen(merges_path, "rb");
if (!f)
return -1;
@@ -1745,10 +1733,10 @@ static int load_merges_map(embed_tokenizer_t *tok, const char *merges_path) {
* Public API
* ======================================================================== */
-embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) {
+tok_bpe_t *tok_bpe_load(const char *vocab_json_path) {
FILE *f = fopen(vocab_json_path, "rb");
if (!f) {
- fprintf(stderr, "embed_tokenizer_load: cannot open %s\n", vocab_json_path);
+ fprintf(stderr, "tok_bpe_load: cannot open %s\n", vocab_json_path);
return NULL;
}
@@ -1803,7 +1791,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) {
}
int vocab_size = max_id + 1;
- embed_tokenizer_t *tok = (embed_tokenizer_t *)calloc(1, sizeof(embed_tokenizer_t));
+ tok_bpe_t *tok = (tok_bpe_t *)calloc(1, sizeof(tok_bpe_t));
if (!tok) {
free(json);
return NULL;
@@ -1812,7 +1800,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) {
tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *));
tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *));
if (!tok->id_to_text || !tok->id_to_bpe) {
- embed_tokenizer_free(tok);
+ tok_bpe_free(tok);
free(json);
return NULL;
}
@@ -1852,7 +1840,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) {
tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1);
tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t));
if (!tok->vocab_map) {
- embed_tokenizer_free(tok);
+ tok_bpe_free(tok);
return NULL;
}
for (int i = 0; i < vocab_size; i++) {
@@ -1863,7 +1851,7 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) {
char merges_path[1024];
if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) {
- if (load_merges_map(tok, merges_path) != 0 && embed_verbose >= 2) {
+ if (load_merges_map(tok, merges_path) != 0) {
fprintf(stderr,
"Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n",
merges_path);
@@ -1873,19 +1861,19 @@ embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) {
return tok;
}
-const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id) {
+const char *tok_bpe_decode(const tok_bpe_t *tok, int token_id) {
if (!tok || token_id < 0 || token_id >= tok->vocab_size)
return "";
return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : "";
}
-int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token) {
+int tok_bpe_token_id(const tok_bpe_t *tok, const char *token) {
if (!tok || !token || !tok->vocab_map)
return -1;
return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token);
}
-int *embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens) {
+int *tok_bpe_encode(const tok_bpe_t *tok, const char *text, int *out_n_tokens) {
if (out_n_tokens)
*out_n_tokens = 0;
if (!tok || !text || text[0] == '\0')
@@ -1909,11 +1897,11 @@ int *embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int
return ids;
}
-embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void) {
- return (embed_tokenizer_workspace_t *)calloc(1, sizeof(embed_tokenizer_workspace_t));
+tok_bpe_workspace_t *tok_bpe_workspace_new(void) {
+ return (tok_bpe_workspace_t *)calloc(1, sizeof(tok_bpe_workspace_t));
}
-void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws) {
+void tok_bpe_workspace_free(tok_bpe_workspace_t *ws) {
if (!ws)
return;
free(ws->normalized);
@@ -1929,12 +1917,12 @@ void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws) {
free(ws);
}
-int embed_tokenizer_encode_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
+int tok_bpe_encode_into(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens) {
if (out_n_tokens)
*out_n_tokens = 0;
if (!tok || !ws || !text || out_cap < 0)
@@ -1953,10 +1941,10 @@ int embed_tokenizer_encode_into(const embed_tokenizer_t *tok,
return rc;
}
-int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_n_tokens) {
+int *tok_bpe_encode_with_workspace(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens) {
if (out_n_tokens)
*out_n_tokens = 0;
if (!tok || !ws || !text || text[0] == '\0')
@@ -1972,11 +1960,11 @@ int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok,
return NULL;
int n_ids = 0;
- int rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
+ int rc = tok_bpe_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
if (rc == -2) {
if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0)
return NULL;
- rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
+ rc = tok_bpe_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
}
if (rc != 0 || n_ids <= 0)
return NULL;
@@ -1990,7 +1978,7 @@ int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok,
return ids;
}
-void embed_tokenizer_free(embed_tokenizer_t *tok) {
+void tok_bpe_free(tok_bpe_t *tok) {
if (!tok)
return;
diff --git a/libffwd/bpe.h b/libffwd/bpe.h
new file mode 100644
index 0000000..5b16c52
--- /dev/null
+++ b/libffwd/bpe.h
@@ -0,0 +1,74 @@
+/*
+ * bpe.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level)
+ *
+ * Supports:
+ * - decode token ID -> UTF-8 text
+ * - encode UTF-8 text -> token IDs (vocab.json + merges.txt)
+ */
+
+#ifndef TOK_BPE_H
+#define TOK_BPE_H
+
+typedef struct {
+ char **id_to_text; /* [vocab_size] decoded text strings */
+ char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
+ int vocab_size;
+
+ /* Internal hash maps (opaque to callers) */
+ void *vocab_map;
+ int vocab_map_cap;
+ void *merge_map;
+ int merge_map_cap;
+ /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id).
+ * NULL when the vocab could not resolve every merge; encoding then falls
+ * back to the string-based merge loop. */
+ void *int_merges;
+ int int_merges_cap;
+ /* Byte-level unicode codepoint -> single-character token id (512 slots),
+ * or NULL when any byte token is missing from the vocab. */
+ int *cp_to_id;
+} tok_bpe_t;
+
+typedef struct tok_bpe_workspace tok_bpe_workspace_t;
+
+/* Load tokenizer from vocab.json in model directory */
+tok_bpe_t *tok_bpe_load(const char *vocab_json_path);
+
+/* Decode a single token ID to text. Returns pointer to internal string. */
+const char *tok_bpe_decode(const tok_bpe_t *tok, int token_id);
+
+/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>").
+ * Returns -1 when the token is not in the vocab. */
+int tok_bpe_token_id(const tok_bpe_t *tok, const char *token);
+
+/* Encode UTF-8 text into token IDs using BPE.
+ * Returns malloc'd array of token IDs and sets *out_n_tokens.
+ * Returns NULL on error (and sets *out_n_tokens to 0). */
+int *tok_bpe_encode(const tok_bpe_t *tok, const char *text, int *out_n_tokens);
+
+/* Reusable scratch for allocation-light repeated encoding. */
+tok_bpe_workspace_t *tok_bpe_workspace_new(void);
+void tok_bpe_workspace_free(tok_bpe_workspace_t *ws);
+
+/* Encode into caller-provided out_ids[0..out_cap-1].
+ * Returns 0 on success, -1 on error, -2 if out_cap is too small.
+ * In all cases *out_n_tokens is set to the number of tokens needed/emitted
+ * when out_n_tokens is non-NULL. */
+int tok_bpe_encode_into(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
+
+/* Same tokenization as tok_bpe_encode(), but reuses ws internally and
+ * performs only the final returned int[] allocation. */
+int *tok_bpe_encode_with_workspace(const tok_bpe_t *tok,
+ tok_bpe_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
+/* Free tokenizer */
+void tok_bpe_free(tok_bpe_t *tok);
+
+#endif /* TOK_BPE_H */
diff --git a/libffwd/build.h b/libffwd/build.h
new file mode 100644
index 0000000..01073d5
--- /dev/null
+++ b/libffwd/build.h
@@ -0,0 +1,49 @@
+/* ffwd_build.h - build identity and capability information for the ffwd tools
+ *
+ * The real values come from the Makefile's VERSION_CFLAGS at compile time; the
+ * fallbacks below apply when those flags are absent. Front-end only: the
+ * inference library does not use these. */
+#ifndef FFWD_BUILD_H
+#define FFWD_BUILD_H
+
+#include "ffwd.h" /* ffwd_capability(), declared alongside the rest of the API */
+
+#include <stdio.h>
+#include <string.h>
+
+#ifndef FFWD_VERSION
+# define FFWD_VERSION "development"
+#endif
+#ifndef FFWD_BUILD_COMMIT
+# define FFWD_BUILD_COMMIT "unknown"
+#endif
+#ifndef FFWD_BUILD_DATE
+# define FFWD_BUILD_DATE "unknown"
+#endif
+#ifndef FFWD_BUILD_OS
+# define FFWD_BUILD_OS "unknown"
+#endif
+#ifndef FFWD_BUILD_ARCH
+# define FFWD_BUILD_ARCH "unknown"
+#endif
+
+/* Program name with any leading directory removed. */
+static inline const char *ffwd_prog_name(const char *argv0) {
+ const char *slash = strrchr(argv0, '/');
+ return slash ? slash + 1 : argv0;
+}
+
+static inline void ffwd_print_version(const char *prog) {
+ printf("%s %s (%s/%s)\n", prog, FFWD_VERSION, FFWD_BUILD_OS, FFWD_BUILD_ARCH);
+}
+
+static inline void ffwd_print_build_info(const char *prog) {
+ ffwd_print_version(prog);
+ printf("backend: %s\n"
+ "compiler: %s\n"
+ "built: %s\n"
+ "commit: %s\n",
+ ffwd_capability(), __VERSION__, FFWD_BUILD_DATE, FFWD_BUILD_COMMIT);
+}
+
+#endif /* FFWD_BUILD_H */
diff --git a/src/config.c b/libffwd/config.c
index a33a135..efc4bcb 100644
--- a/src/config.c
+++ b/libffwd/config.c
@@ -8,6 +8,10 @@
#include <stdlib.h>
#include <string.h>
+/* Verbosity flag, read across the codebase for optional stderr diagnostics.
+ * Defined in config.c because every backend links it. */
+int ffwd_verbose = 0;
+
typedef enum {
JSON_BOOL_MISSING = 0,
JSON_BOOL_FALSE = 1,
@@ -74,7 +78,7 @@ static int read_optional_file(const char *model_dir, const char *relative, char
char path[1024];
int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_config: model path too long\n");
+ fprintf(stderr, "ffwd_config: model path too long\n");
return -1;
}
@@ -82,7 +86,7 @@ static int read_optional_file(const char *model_dir, const char *relative, char
if (!f && errno == ENOENT)
return 0;
if (!f) {
- fprintf(stderr, "embed_config: cannot open %s\n", path);
+ fprintf(stderr, "ffwd_config: cannot open %s\n", path);
return -1;
}
if (fseek(f, 0, SEEK_END) != 0) {
@@ -106,7 +110,7 @@ static int read_optional_file(const char *model_dir, const char *relative, char
return 1;
}
-static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char *model_dir) {
+static int parse_sentence_transformers_metadata(ffwd_config_t *cfg, const char *model_dir) {
char *buf = NULL;
int rc = read_optional_file(model_dir, "1_Pooling/config.json", &buf);
if (rc < 0)
@@ -119,22 +123,21 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char
json_bool_t sqrt_len = json_get_bool(buf, "pooling_mode_mean_sqrt_len_tokens");
json_bool_t weighted = json_get_bool(buf, "pooling_mode_weightedmean_tokens");
if (mean < 0 || last < 0 || cls < 0 || max < 0 || sqrt_len < 0 || weighted < 0) {
- fprintf(stderr, "embed_config: invalid pooling metadata\n");
+ fprintf(stderr, "ffwd_config: invalid pooling metadata\n");
free(buf);
return -1;
}
- int supported =
- (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE) + (cls == JSON_BOOL_TRUE);
+ int supported = (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE) + (cls == JSON_BOOL_TRUE);
int unsupported =
max == JSON_BOOL_TRUE || sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE;
if (supported != 1 || unsupported) {
- fprintf(stderr, "embed_config: unsupported Sentence Transformers pooling mode\n");
+ fprintf(stderr, "ffwd_config: unsupported Sentence Transformers pooling mode\n");
free(buf);
return -1;
}
- cfg->pooling_mode = cls == JSON_BOOL_TRUE ? EMBED_POOL_CLS
- : last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN
- : EMBED_POOL_MEAN;
+ cfg->pooling_mode = cls == JSON_BOOL_TRUE ? FFWD_POOL_CLS
+ : last == JSON_BOOL_TRUE ? FFWD_POOL_LAST_TOKEN
+ : FFWD_POOL_MEAN;
free(buf);
buf = NULL;
}
@@ -143,8 +146,7 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char
if (rc < 0)
return -1;
if (rc > 0) {
- cfg->normalize_embeddings =
- strstr(buf, "\"sentence_transformers.models.Normalize\"") != NULL;
+ cfg->normalize_embeddings = strstr(buf, "\"sentence_transformers.models.Normalize\"") != NULL;
free(buf);
buf = NULL;
}
@@ -155,7 +157,7 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char
if (rc > 0) {
json_bool_t add_eos = json_get_bool(buf, "add_eos_token");
if (add_eos < 0) {
- fprintf(stderr, "embed_config: invalid add_eos_token metadata\n");
+ fprintf(stderr, "ffwd_config: invalid add_eos_token metadata\n");
free(buf);
return -1;
}
@@ -166,17 +168,17 @@ static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char
return 0;
}
-int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
+int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_config: model path too long\n");
+ fprintf(stderr, "ffwd_config: model path too long\n");
return -1;
}
FILE *f = fopen(path, "rb");
if (!f) {
- fprintf(stderr, "embed_config: cannot open %s\n", path);
+ fprintf(stderr, "ffwd_config: cannot open %s\n", path);
return -1;
}
@@ -202,46 +204,46 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0);
cfg->n_heads = json_get_int(buf, "num_attention_heads", 0);
cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0);
- cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM);
+ cfg->head_dim = json_get_int(buf, "head_dim", FFWD_HEAD_DIM);
cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0);
- cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
+ cfg->vocab_size = json_get_int(buf, "vocab_size", FFWD_VOCAB_SIZE);
cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12);
cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0);
cfg->position_id_offset = 0;
cfg->type_vocab_size = json_get_int(buf, "type_vocab_size", 2);
- cfg->family = EMBED_FAMILY_QWEN3;
+ cfg->family = FFWD_FAMILY_QWEN3;
cfg->qk_norm = 1;
cfg->qkv_bias = 0;
- cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
- cfg->pooling_mode = EMBED_POOL_MEAN;
+ cfg->attention_mode = FFWD_ATTENTION_BIDIRECTIONAL;
+ cfg->pooling_mode = FFWD_POOL_MEAN;
cfg->normalize_embeddings = 0;
cfg->append_terminal_token = 0;
if (json_string_equals(buf, "model_type", "qwen3")) {
- cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
- cfg->pooling_mode = EMBED_POOL_LAST_TOKEN;
+ cfg->attention_mode = FFWD_ATTENTION_CAUSAL;
+ cfg->pooling_mode = FFWD_POOL_LAST_TOKEN;
cfg->normalize_embeddings = 1;
cfg->append_terminal_token = 1;
} else if (json_string_equals(buf, "model_type", "qwen2")) {
cfg->qk_norm = 0;
cfg->qkv_bias = 1;
- cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
+ cfg->attention_mode = FFWD_ATTENTION_CAUSAL;
} else if (json_string_equals(buf, "model_type", "bert") ||
json_string_equals(buf, "model_type", "roberta") ||
json_string_equals(buf, "model_type", "xlm-roberta")) {
- cfg->family = EMBED_FAMILY_BERT;
+ cfg->family = FFWD_FAMILY_BERT;
cfg->qk_norm = 0;
cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */
- cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
+ cfg->attention_mode = FFWD_ATTENTION_BIDIRECTIONAL;
cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */
cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
if (json_string_equals(buf, "model_type", "roberta") ||
json_string_equals(buf, "model_type", "xlm-roberta")) {
int pad_id = json_get_int(buf, "pad_token_id", 1);
if (pad_id < 0 || pad_id == INT_MAX) {
- fprintf(stderr, "embed_config: invalid RoBERTa pad_token_id\n");
+ fprintf(stderr, "ffwd_config: invalid RoBERTa pad_token_id\n");
free(buf);
return -1;
}
@@ -252,13 +254,12 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
* (gelu, absent) is the exact erf GeLU the released encoders use. */
if (json_string_equals(buf, "hidden_act", "gelu_new") ||
json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) {
- cfg->ffn_act = EMBED_ACT_GELU_TANH;
+ cfg->ffn_act = FFWD_ACT_GELU_TANH;
}
if (cfg->max_position_embeddings <= 0 ||
- cfg->position_id_offset >= cfg->max_position_embeddings ||
- cfg->type_vocab_size <= 0 || !isfinite(cfg->layer_norm_eps) ||
- cfg->layer_norm_eps <= 0.0f) {
- fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n");
+ cfg->position_id_offset >= cfg->max_position_embeddings || cfg->type_vocab_size <= 0 ||
+ !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) {
+ fprintf(stderr, "ffwd_config: invalid BERT layer_norm_eps/max_position_embeddings\n");
free(buf);
return -1;
}
@@ -266,13 +267,13 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
json_bool_t is_causal = json_get_bool(buf, "is_causal");
if (is_causal < 0) {
- fprintf(stderr, "embed_config: invalid is_causal in %s\n", path);
+ fprintf(stderr, "ffwd_config: invalid is_causal in %s\n", path);
free(buf);
return -1;
}
if (is_causal != JSON_BOOL_MISSING) {
cfg->attention_mode =
- is_causal == JSON_BOOL_TRUE ? EMBED_ATTENTION_CAUSAL : EMBED_ATTENTION_BIDIRECTIONAL;
+ is_causal == JSON_BOOL_TRUE ? FFWD_ATTENTION_CAUSAL : FFWD_ATTENTION_BIDIRECTIONAL;
}
cfg->q_dim = 0;
@@ -293,28 +294,28 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) ||
cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
fprintf(stderr,
- "embed_config: invalid config in %s "
+ "ffwd_config: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim,
cfg->intermediate_size);
return -1;
}
- if (cfg->n_layers > EMBED_MAX_LAYERS) {
- fprintf(stderr, "embed_config: too many layers (%d > %d)\n", cfg->n_layers,
- EMBED_MAX_LAYERS);
+ if (cfg->n_layers > FFWD_MAX_LAYERS) {
+ fprintf(stderr, "ffwd_config: too many layers (%d > %d)\n", cfg->n_layers,
+ FFWD_MAX_LAYERS);
return -1;
}
- if (embed_verbose >= 1)
+ if (ffwd_verbose >= 1)
fprintf(stderr,
"config: hidden=%d, layers=%d, heads=%d/%d, "
"inter=%d, head_dim=%d, attention=%s, pooling=%s, qk_norm=%s, qkv_bias=%s\n",
cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads,
cfg->intermediate_size, cfg->head_dim,
- cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional",
- cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token"
- : cfg->pooling_mode == EMBED_POOL_CLS ? "cls"
- : "mean",
+ cfg->attention_mode == FFWD_ATTENTION_CAUSAL ? "causal" : "bidirectional",
+ cfg->pooling_mode == FFWD_POOL_LAST_TOKEN ? "last-token"
+ : cfg->pooling_mode == FFWD_POOL_CLS ? "cls"
+ : "mean",
cfg->qk_norm ? "yes" : "no", cfg->qkv_bias ? "yes" : "no");
return 0;
diff --git a/libffwd/config.h b/libffwd/config.h
new file mode 100644
index 0000000..b3e2f67
--- /dev/null
+++ b/libffwd/config.h
@@ -0,0 +1,8 @@
+#ifndef FFWD_CONFIG_H
+#define FFWD_CONFIG_H
+
+#include "ffwd.h"
+
+int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir);
+
+#endif /* FFWD_CONFIG_H */
diff --git a/src/cuda.cu b/libffwd/cuda.cu
index 872834b..2b329cd 100644
--- a/src/cuda.cu
+++ b/libffwd/cuda.cu
@@ -1,6 +1,6 @@
extern "C" {
#include "cuda.h"
-#include "internal.h"
+#include "model_types.h"
}
#include <cuda_runtime.h>
@@ -39,13 +39,13 @@ typedef struct {
float *ffn_ln_bias;
} cuda_layer_t;
-struct embed_cuda_ctx {
- embed_model_t *cpu;
- int own_cpu; // free ctx->cpu in embed_cuda_free only when this context owns it
- embed_config_t config;
+struct ffwd_cuda_ctx {
+ ffwd_model_t *cpu;
+ int own_cpu; // free ctx->cpu in ffwd_cuda_free only when this context owns it
+ ffwd_config_t config;
cudaStream_t stream;
cublasHandle_t blas;
- cuda_matrix_t embed_tokens;
+ cuda_matrix_t ffwd_tokens;
cuda_layer_t *layers;
float *norm;
/* BERT family (NULL for Qwen3): learned absolute position embeddings, the
@@ -53,8 +53,8 @@ struct embed_cuda_ctx {
* LayerNorm weight/bias. */
float *position_embeddings;
float *token_type_embedding;
- float *embed_ln_w;
- float *embed_ln_b;
+ float *ffwd_ln_w;
+ float *ffwd_ln_b;
int seq_cap;
int batch_cap;
@@ -143,7 +143,7 @@ static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) {
static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F;
static int g_gemm_compute_set = 0;
-int embed_cuda_set_fast_gemm(const char *mode) {
+int ffwd_cuda_set_fast_gemm(const char *mode) {
cublasComputeType_t ct;
if (parse_gemm_compute(mode, &ct) != 0)
return -1;
@@ -184,13 +184,13 @@ static uint16_t f32_to_bf16(float f) {
static int g_weights_bf16 = 0;
static int g_weights_bf16_set = 0;
-int embed_cuda_set_weights_bf16(int on) {
+int ffwd_cuda_set_weights_bf16(int on) {
g_weights_bf16 = on ? 1 : 0;
g_weights_bf16_set = 1;
return 0;
}
-static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, size_t count) {
+static int copy_weight_host_f32(float *dst, const ffwd_weight_ref_t *w, size_t count) {
if (!dst || !w || !w->data)
return -1;
if (w->dtype == DTYPE_F32) {
@@ -244,7 +244,7 @@ static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, int r
return 0;
}
-static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, int cols) {
+static int load_matrix(cuda_matrix_t *m, const ffwd_weight_ref_t *w, int rows, int cols) {
if (!m || !w || rows <= 0 || cols <= 0)
return -1;
size_t count = (size_t)rows * (size_t)cols;
@@ -260,7 +260,7 @@ static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows,
return r;
}
-static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
+static int load_qkv_matrix(cuda_matrix_t *m, const ffwd_layer_t *src, const ffwd_config_t *c) {
if (!m || !src || !c)
return -1;
int rows = c->q_dim + 2 * c->kv_dim;
@@ -283,7 +283,7 @@ static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const emb
}
static int
-load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
+load_gate_up_matrix(cuda_matrix_t *m, const ffwd_layer_t *src, const ffwd_config_t *c) {
if (!m || !src || !c)
return -1;
int rows = 2 * c->intermediate_size;
@@ -321,7 +321,7 @@ static int load_vector(float **out, const float *src, int n) {
return 0;
}
-static int load_qkv_bias(float **out, const embed_layer_t *src, const embed_config_t *c) {
+static int load_qkv_bias(float **out, const ffwd_layer_t *src, const ffwd_config_t *c) {
int n = c->q_dim + 2 * c->kv_dim;
float *host = (float *)malloc((size_t)n * sizeof(float));
if (!host)
@@ -341,7 +341,7 @@ __global__ static void add_row_bias_kernel(float *x, const float *bias, int rows
x[idx] += bias[idx % dim];
}
-__global__ static void embed_lookup_kernel(
+__global__ static void ffwd_lookup_kernel(
float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int count = total * hidden;
@@ -492,8 +492,8 @@ __global__ static void rms_norm_rope_qk_kernel(float *x,
// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16
// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM.
template <typename OUT_T>
-__global__ static void silu_mul_packed_kernel(
- OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) {
+__global__ static void
+silu_mul_packed_kernel(OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
int n = rows * intermediate;
if (i < n) {
@@ -541,8 +541,7 @@ __global__ static void attn_expand_kv_kernel(float *kexp,
// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM.
// P_T is float with probs == scores (in place), or __nv_bfloat16 when the
// probabilities feed a BF16 @V GEMM.
-template <typename P_T>
-__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) {
+template <typename P_T> __global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) {
float *s = scores + (size_t)blockIdx.x * L;
P_T *p = probs + (size_t)blockIdx.x * L;
int tid = threadIdx.x;
@@ -669,14 +668,9 @@ __global__ static void span_pool_kernel(
// BERT LayerNorm: mean-subtract, biased variance (/dim), scale by gamma, shift
// by beta. One block per row, two block reductions before any write, so it is
-// safe in place (out == x). Matches the CPU embed_layer_norm.
-__global__ static void layer_norm_kernel(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int rows,
- int dim,
- float eps) {
+// safe in place (out == x). Matches the CPU layer_norm.
+__global__ static void layer_norm_kernel(
+ float *out, const float *x, const float *gamma, const float *beta, int rows, int dim, float eps) {
int row = blockIdx.x;
int tid = threadIdx.x;
const float *xr = x + (size_t)row * dim;
@@ -744,13 +738,13 @@ static int launch_check(void) {
}
// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm).
-// EMBED_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core
+// FFWD_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core
// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands
// internally.
static cublasComputeType_t gemm_compute(void) {
static int env_init = 0;
if (!g_gemm_compute_set && !env_init) {
- const char *e = getenv("EMBED_CUDA_GEMM_MODE");
+ const char *e = getenv("FFWD_CUDA_GEMM_MODE");
if (e && parse_gemm_compute(e, &g_gemm_compute) == 0)
g_gemm_compute_set = 1;
env_init = 1;
@@ -760,7 +754,7 @@ static cublasComputeType_t gemm_compute(void) {
// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only
// BF16 path materializes one F32 weight at a time into this buffer.
-static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) {
+static int ensure_weight_f32(ffwd_cuda_ctx_t *ctx, size_t elems) {
if (ctx->weight_f32_elems >= elems)
return 0;
cudaFree(ctx->weight_f32);
@@ -777,7 +771,7 @@ static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) {
// back to F32 here, so the GEMM is exact (honoring gemm_compute): BF16 is then a
// storage choice that costs no quality, which the post-norm LayerNorm needs.
// With F32 weights this is the exact path.
-static int linear_ex(embed_cuda_ctx_t *ctx,
+static int linear_ex(ffwd_cuda_ctx_t *ctx,
const cuda_matrix_t *w,
const void *x,
int x_is_bf16,
@@ -789,7 +783,7 @@ static int linear_ex(embed_cuda_ctx_t *ctx,
float beta) {
const float alpha = 1.0f;
if (w->bf16) {
- if (!x_is_bf16 && ctx->config.family == EMBED_FAMILY_BERT) {
+ if (!x_is_bf16 && ctx->config.family == FFWD_FAMILY_BERT) {
size_t welems = (size_t)out_dim * (size_t)in_dim;
if (ensure_weight_f32(ctx, welems) != 0)
return -1;
@@ -814,19 +808,18 @@ static int linear_ex(embed_cuda_ctx_t *ctx,
return -1;
xb = ctx->act_bf16;
}
- CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim,
- &alpha, w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride,
- &beta, y, CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F,
- CUBLAS_GEMM_DEFAULT));
+ CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha,
+ w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride, &beta, y,
+ CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
return 0;
}
CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha,
- w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y,
- CUDA_R_32F, out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT));
+ w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y, CUDA_R_32F,
+ out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT));
return 0;
}
-static int linear(embed_cuda_ctx_t *ctx,
+static int linear(ffwd_cuda_ctx_t *ctx,
const cuda_matrix_t *w,
const float *x,
float *y,
@@ -837,7 +830,7 @@ static int linear(embed_cuda_ctx_t *ctx,
}
// x points at activations already stored as BF16 (e.g. ctx->act_bf16).
-static int linear_bf16x(embed_cuda_ctx_t *ctx,
+static int linear_bf16x(ffwd_cuda_ctx_t *ctx,
const cuda_matrix_t *w,
const void *x,
float *y,
@@ -849,7 +842,7 @@ static int linear_bf16x(embed_cuda_ctx_t *ctx,
return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta);
}
-static int linear_accum(embed_cuda_ctx_t *ctx,
+static int linear_accum(ffwd_cuda_ctx_t *ctx,
const cuda_matrix_t *w,
const float *x,
float *y,
@@ -879,9 +872,8 @@ static int gemm_strided_batched(cublasHandle_t blas,
int batch_count) {
cublasComputeType_t ct = gemm_compute();
if (ct == CUBLAS_COMPUTE_32F) {
- CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda,
- strideA, B, ldb, strideB, beta, C, ldc, strideC,
- batch_count));
+ CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda, strideA,
+ B, ldb, strideB, beta, C, ldc, strideC, batch_count));
} else {
CUBLAS_CHECK(cublasGemmStridedBatchedEx(
blas, transa, transb, m, n, k, alpha, A, CUDA_R_32F, lda, strideA, B, CUDA_R_32F, ldb,
@@ -901,8 +893,8 @@ static int gemm_strided_batched(cublasHandle_t blas,
// layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged
// batches, single sequences, or when one sequence's scores exceed the
// budget.
-static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) {
- const embed_config_t *c = &ctx->config;
+static int attn_batched_setup(ffwd_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) {
+ const ffwd_config_t *c = &ctx->config;
ctx->attn_G = 0;
if (batch < 2)
return 0;
@@ -991,7 +983,7 @@ static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, in
// out_bf16 != NULL selects the BF16-weight layout: V and the softmax
// probabilities are stored BF16 and the @V GEMM emits BF16 attention output
// at out_bf16 (stride q_dim), ready for the wo projection without a cast.
-static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
+static int cuda_attention_gemm(ffwd_cuda_ctx_t *ctx,
const int *h_offsets,
int batch,
int q_offset,
@@ -1000,12 +992,12 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
int qkv_dim,
float scale,
__nv_bfloat16 *out_bf16) {
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
int q_dim = c->q_dim;
int hd = c->head_dim;
int H = c->n_heads;
int total = h_offsets[batch];
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
+ int causal = c->attention_mode == FFWD_ATTENTION_CAUSAL;
const float alpha = scale, beta = 0.0f, one = 1.0f;
int threads = 256;
@@ -1016,8 +1008,7 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
k_offset, v_offset);
} else {
attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
- ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset,
- v_offset);
+ ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset, v_offset);
}
if (launch_check() != 0)
return -1;
@@ -1038,9 +1029,9 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
size_t off = (size_t)s * H;
/* scores = scale * Q @ K^T, batched over (seq, head) */
CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha,
- K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F,
- qkv_dim, &beta, S + off, CUDA_R_32F, L, n,
- gemm_compute(), CUBLAS_GEMM_DEFAULT));
+ K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F, qkv_dim,
+ &beta, S + off, CUDA_R_32F, L, n, gemm_compute(),
+ CUBLAS_GEMM_DEFAULT));
if (out_bf16) {
if (causal)
attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
@@ -1072,8 +1063,8 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset;
const float *K = ctx->kexp + (size_t)start * q_dim;
/* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */
- if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim,
- hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L,
+ if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim, hd,
+ Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L,
H) != 0)
return -1;
if (out_bf16) {
@@ -1087,10 +1078,10 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
if (launch_check() != 0)
return -1;
/* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- CUBLAS_CHECK(cublasGemmStridedBatchedEx(
- ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, CUDA_R_16BF, q_dim, hd, P,
- CUDA_R_16BF, L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, hd, H,
- CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
+ CUBLAS_CHECK(cublasGemmStridedBatchedEx(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L,
+ &one, V, CUDA_R_16BF, q_dim, hd, P, CUDA_R_16BF,
+ L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim,
+ hd, H, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
} else {
const float *V = ctx->vexp + (size_t)start * q_dim;
float *O = ctx->attn_out + (size_t)start * q_dim;
@@ -1112,8 +1103,8 @@ static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
return 0;
}
-static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) {
- const embed_config_t *c = &ctx->config;
+static int ensure_buffers(ffwd_cuda_ctx_t *ctx, int total, int batch, int max_seq) {
+ const ffwd_config_t *c = &ctx->config;
if (total > ctx->seq_cap) {
cudaFree(ctx->x);
cudaFree(ctx->x_norm);
@@ -1131,8 +1122,7 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
ctx->kexp = ctx->vexp = NULL;
ctx->act_bf16 = NULL;
CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
- CUDA_CHECK(
- cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->qkv,
(size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float)));
@@ -1143,8 +1133,8 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float)));
CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float)));
if (g_weights_bf16)
- CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * 2 * c->intermediate_size *
- sizeof(__nv_bfloat16)));
+ CUDA_CHECK(cudaMalloc(&ctx->act_bf16,
+ (size_t)total * 2 * c->intermediate_size * sizeof(__nv_bfloat16)));
ctx->seq_cap = total;
}
if (batch + 1 > ctx->batch_cap) {
@@ -1198,11 +1188,9 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
cudaMalloc((void **)&ctx->rope_cos, (size_t)max_seq * c->head_dim * sizeof(float)));
CUDA_CHECK(
cudaMalloc((void **)&ctx->rope_sin, (size_t)max_seq * c->head_dim * sizeof(float)));
- CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos,
- (size_t)max_seq * c->head_dim * sizeof(float),
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos, (size_t)max_seq * c->head_dim * sizeof(float),
cudaMemcpyHostToDevice, ctx->stream));
- CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin,
- (size_t)max_seq * c->head_dim * sizeof(float),
+ CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin, (size_t)max_seq * c->head_dim * sizeof(float),
cudaMemcpyHostToDevice, ctx->stream));
free(hcos);
free(hsin);
@@ -1211,19 +1199,18 @@ static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_s
return 0;
}
-static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) {
+static int ensure_pooled_rows(ffwd_cuda_ctx_t *ctx, int rows) {
if (rows <= ctx->pooled_rows_cap)
return 0;
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
cudaFree(ctx->pooled_out);
ctx->pooled_out = NULL;
- CUDA_CHECK(
- cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float)));
+ CUDA_CHECK(cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float)));
ctx->pooled_rows_cap = rows;
return 0;
}
-static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) {
+static int ensure_span_buffers(ffwd_cuda_ctx_t *ctx, int n_spans) {
if (n_spans <= ctx->span_cap)
return 0;
cudaFree(ctx->span_starts);
@@ -1236,7 +1223,7 @@ static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) {
return 0;
}
-static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
+static int load_layer(cuda_layer_t *dst, const ffwd_layer_t *src, const ffwd_config_t *c) {
return load_qkv_matrix(&dst->qkv, src, c) ||
load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
(c->qkv_bias && load_qkv_bias(&dst->qkv_bias, src, c)) ||
@@ -1252,7 +1239,7 @@ static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_c
* kv_dim == hidden); the two block LayerNorm weights load into input_norm
* (post-attention) and post_attn_norm (post-FFN); gate_up_proj holds the single
* up_proj. The added fields carry the biases the Qwen slots lack. */
-static int load_bert_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
+static int load_bert_layer(cuda_layer_t *dst, const ffwd_layer_t *src, const ffwd_config_t *c) {
return load_qkv_matrix(&dst->qkv, src, c) ||
load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
load_qkv_bias(&dst->qkv_bias, src, c) ||
@@ -1285,75 +1272,75 @@ static void free_layer(cuda_layer_t *l) {
}
/* Build a device context from an already-loaded CPU model. own_cpu records
- * whether embed_cuda_free should release that CPU model: embed_cuda_load owns
+ * whether ffwd_cuda_free should release that CPU model: ffwd_cuda_load owns
* the model it loaded; the late path passes a model owned by its CPU late
* model and sets own_cpu = 0. */
-static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) {
+static ffwd_cuda_ctx_t *cuda_ctx_from_model(ffwd_model_t *cpu, int own_cpu) {
if (!cpu)
return NULL;
- embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx));
+ ffwd_cuda_ctx_t *ctx = (ffwd_cuda_ctx_t *)calloc(1, sizeof(*ctx));
if (!ctx) {
if (own_cpu)
- embed_model_free(cpu);
+ ffwd_model_free(cpu);
return NULL;
}
ctx->cpu = cpu;
ctx->own_cpu = own_cpu;
ctx->config = cpu->config;
- /* Without an explicit --cuda-weight-dtype, store weights in the model
+ /* Without an explicit --gpu-weight-dtype, store weights in the model
* file's dtype: BF16 snapshots load as BF16 (bit-exact pass-through),
* F32 snapshots keep the exact F32 default. */
if (!g_weights_bf16_set) {
- g_weights_bf16 = cpu->weights.embed_tokens.dtype == DTYPE_BF16;
+ g_weights_bf16 = cpu->weights.ffwd_tokens.dtype == DTYPE_BF16;
if (g_weights_bf16)
fprintf(stderr, "cuda: BF16 model file; storing weights as BF16 "
- "(use --cuda-weight-dtype f32 to override)\n");
+ "(use --gpu-weight-dtype f32 to override)\n");
}
if (cudaStreamCreate(&ctx->stream) != cudaSuccess ||
cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS ||
cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) {
- embed_cuda_free(ctx);
+ ffwd_cuda_free(ctx);
return NULL;
}
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers));
if (!ctx->layers) {
- embed_cuda_free(ctx);
+ ffwd_cuda_free(ctx);
return NULL;
}
- if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, c->vocab_size,
+ if (load_matrix(&ctx->ffwd_tokens, &cpu->weights.ffwd_tokens, c->vocab_size,
c->hidden_size) != 0) {
- embed_cuda_free(ctx);
+ ffwd_cuda_free(ctx);
return NULL;
}
- if (c->family == EMBED_FAMILY_BERT) {
+ if (c->family == FFWD_FAMILY_BERT) {
/* BERT has no final norm; it adds learned position + token-type[0]
* embeddings and an embedding LayerNorm (only row 0 of token-type is
* ever used, so load just that row). */
- const embed_weights_t *w = &cpu->weights;
+ const ffwd_weights_t *w = &cpu->weights;
if (load_vector(&ctx->position_embeddings, w->position_embeddings,
c->max_position_embeddings * c->hidden_size) != 0 ||
load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) != 0 ||
- load_vector(&ctx->embed_ln_w, w->embed_ln_w, c->hidden_size) != 0 ||
- load_vector(&ctx->embed_ln_b, w->embed_ln_b, c->hidden_size) != 0) {
- embed_cuda_free(ctx);
+ load_vector(&ctx->ffwd_ln_w, w->ffwd_ln_w, c->hidden_size) != 0 ||
+ load_vector(&ctx->ffwd_ln_b, w->ffwd_ln_b, c->hidden_size) != 0) {
+ ffwd_cuda_free(ctx);
return NULL;
}
} else if (load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) {
- embed_cuda_free(ctx);
+ ffwd_cuda_free(ctx);
return NULL;
}
for (int i = 0; i < c->n_layers; i++) {
- int rc = c->family == EMBED_FAMILY_BERT
+ int rc = c->family == FFWD_FAMILY_BERT
? load_bert_layer(&ctx->layers[i], &cpu->weights.layers[i], c)
: load_layer(&ctx->layers[i], &cpu->weights.layers[i], c);
if (rc != 0) {
fprintf(stderr, "cuda: failed to load layer %d\n", i);
- embed_cuda_free(ctx);
+ ffwd_cuda_free(ctx);
return NULL;
}
}
@@ -1361,14 +1348,14 @@ static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) {
return ctx;
}
-embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) {
- return cuda_ctx_from_model(embed_model_load(model_dir), 1);
+ffwd_cuda_ctx_t *ffwd_cuda_load(const char *model_dir) {
+ return cuda_ctx_from_model(ffwd_model_load(model_dir), 1);
}
-void embed_cuda_free(embed_cuda_ctx_t *ctx) {
+void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx) {
if (!ctx)
return;
- cuda_matrix_free(&ctx->embed_tokens);
+ cuda_matrix_free(&ctx->ffwd_tokens);
if (ctx->layers) {
for (int i = 0; i < ctx->config.n_layers; i++)
free_layer(&ctx->layers[i]);
@@ -1377,8 +1364,8 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) {
cudaFree(ctx->norm);
cudaFree(ctx->position_embeddings);
cudaFree(ctx->token_type_embedding);
- cudaFree(ctx->embed_ln_w);
- cudaFree(ctx->embed_ln_b);
+ cudaFree(ctx->ffwd_ln_w);
+ cudaFree(ctx->ffwd_ln_b);
cudaFree(ctx->x);
cudaFree(ctx->x_norm);
cudaFree(ctx->qkv);
@@ -1406,19 +1393,19 @@ void embed_cuda_free(embed_cuda_ctx_t *ctx) {
if (ctx->stream)
cudaStreamDestroy(ctx->stream);
if (ctx->own_cpu)
- embed_model_free(ctx->cpu);
+ ffwd_model_free(ctx->cpu);
free(ctx);
}
-const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) {
+const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx) {
return ctx ? &ctx->config : NULL;
}
// Pack inputs into the device scratch every forward shares: token ids, the
// per-sequence positions 0..L-1, and the batch offsets. Grows scratch via
// ensure_buffers. Returns the packed token count in *out_total, or -1 on error.
-static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx,
- const embed_input_t *inputs,
+static int cuda_upload_packed_inputs(ffwd_cuda_ctx_t *ctx,
+ const ffwd_input_t *inputs,
int batch,
int *out_total) {
int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
@@ -1435,7 +1422,7 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx,
if (inputs[b].n_tokens > max_seq)
max_seq = inputs[b].n_tokens;
}
- if (ctx->config.family == EMBED_FAMILY_BERT &&
+ if (ctx->config.family == FFWD_FAMILY_BERT &&
max_seq > ctx->config.max_position_embeddings - ctx->config.position_id_offset) {
free(h_offsets);
return -1;
@@ -1489,8 +1476,9 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx,
// F32 activations: BERT checkpoints are F32, so the linear/linear_accum F32
// entry points are used throughout (they still handle bf16 weights via an
// internal cast, which keeps a future bf16 BERT correct if not yet optimized).
-static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) {
- const embed_config_t *c = &ctx->config;
+static int
+cuda_forward_batch_bert(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, int batch) {
+ const ffwd_config_t *c = &ctx->config;
int total = 0;
if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0)
return -1;
@@ -1504,8 +1492,8 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i
int threads = 256;
int blocks_hidden = (total * hidden + threads - 1) / threads;
- embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
- ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, hidden,
+ ffwd_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->token_ids, ctx->ffwd_tokens.d, ctx->ffwd_tokens.bf16, total, hidden,
c->vocab_size);
if (launch_check() != 0)
return -1;
@@ -1513,8 +1501,8 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i
ctx->x, ctx->positions, ctx->position_embeddings, ctx->token_type_embedding, total, hidden);
if (launch_check() != 0)
return -1;
- layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->embed_ln_w,
- ctx->embed_ln_b, total, hidden, eps);
+ layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->ffwd_ln_w,
+ ctx->ffwd_ln_b, total, hidden, eps);
if (launch_check() != 0)
return -1;
@@ -1532,15 +1520,15 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i
total, qkv_dim);
if (launch_check() != 0)
return -1;
- if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset,
- qkv_dim, scale, NULL) != 0)
+ if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, qkv_dim,
+ scale, NULL) != 0)
return -1;
// Output dense(+bias), residual into x, then post-attention LayerNorm.
if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, hidden) != 0)
return -1;
- add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>(
- ctx->x, l->o_bias, total, hidden);
+ add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>(ctx->x, l->o_bias,
+ total, hidden);
if (launch_check() != 0)
return -1;
layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->input_norm,
@@ -1556,7 +1544,7 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i
ctx->ffn_gate_up, l->ffn_inter_bias, total, inter);
if (launch_check() != 0)
return -1;
- if (c->ffn_act == EMBED_ACT_GELU_TANH)
+ if (c->ffn_act == FFWD_ACT_GELU_TANH)
gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
ctx->ffn_gate_up, (size_t)inter_count);
else
@@ -1575,15 +1563,15 @@ static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *i
if (launch_check() != 0)
return -1;
}
- // No final norm; embed_cuda_encode_batch pools ctx->x directly.
+ // No final norm; ffwd_cuda_encode_batch pools ctx->x directly.
return 0;
}
-static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) {
+static int cuda_forward_batch(ffwd_cuda_ctx_t *ctx, const ffwd_input_t *inputs, int batch) {
if (!ctx || !inputs || batch <= 0)
return -1;
- const embed_config_t *c = &ctx->config;
- if (c->family == EMBED_FAMILY_BERT)
+ const ffwd_config_t *c = &ctx->config;
+ if (c->family == FFWD_FAMILY_BERT)
return cuda_forward_batch_bert(ctx, inputs, batch);
int total = 0;
if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0)
@@ -1591,9 +1579,9 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
int threads = 256;
int blocks_hidden = (total * c->hidden_size + threads - 1) / threads;
- embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
- ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, c->hidden_size,
- c->vocab_size);
+ ffwd_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
+ ctx->x, ctx->token_ids, ctx->ffwd_tokens.d, ctx->ffwd_tokens.bf16, total,
+ c->hidden_size, c->vocab_size);
if (launch_check() != 0)
return -1;
@@ -1621,8 +1609,8 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
c->hidden_size, 0.0f) != 0)
return -1;
} else {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm,
- total, c->hidden_size, c->rms_norm_eps);
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm, total,
+ c->hidden_size, c->rms_norm_eps);
if (launch_check() != 0)
return -1;
if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, total, c->hidden_size, qkv_dim) != 0)
@@ -1644,8 +1632,8 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
if (launch_check() != 0)
return -1;
- if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset,
- qkv_dim, scale, wbf16 ? act : NULL) != 0)
+ if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset, qkv_dim,
+ scale, wbf16 ? act : NULL) != 0)
return -1;
if (wbf16) {
@@ -1689,40 +1677,39 @@ static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs
if (launch_check() != 0)
return -1;
if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, total,
- c->intermediate_size, c->hidden_size, 2 * c->intermediate_size,
- 1.0f) != 0)
+ c->intermediate_size, c->hidden_size, 2 * c->intermediate_size, 1.0f) != 0)
return -1;
}
}
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total,
- c->hidden_size, c->rms_norm_eps);
+ rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total, c->hidden_size,
+ c->rms_norm_eps);
if (launch_check() != 0)
return -1;
return 0;
}
-int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
+int ffwd_cuda_encode_batch(ffwd_cuda_ctx_t *ctx,
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
if (cuda_forward_batch(ctx, inputs, batch) != 0)
return -1;
if (ensure_pooled_rows(ctx, batch) != 0)
return -1;
- if (c->pooling_mode == EMBED_POOL_LAST_TOKEN)
- last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
- batch, c->hidden_size);
- else if (c->pooling_mode == EMBED_POOL_CLS)
+ if (c->pooling_mode == FFWD_POOL_LAST_TOKEN)
+ last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, batch,
+ c->hidden_size);
+ else if (c->pooling_mode == FFWD_POOL_CLS)
first_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
batch, c->hidden_size);
else
- mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
- batch, c->hidden_size);
+ mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets, batch,
+ c->hidden_size);
if (launch_check() != 0)
return -1;
cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
@@ -1736,7 +1723,7 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
}
if (c->normalize_embeddings) {
for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) !=
+ if (ffwd_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) !=
0)
return -1;
}
@@ -1744,17 +1731,17 @@ int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
return 0;
}
-int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
+int ffwd_cuda_encode_spans_batch(ffwd_cuda_ctx_t *ctx,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
int total_spans = 0;
int total_tokens = 0;
for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
+ const ffwd_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
total_spans > INT_MAX - input->n_spans || total_tokens > INT_MAX - n_tokens)
@@ -1769,7 +1756,7 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
total_tokens += n_tokens;
}
- embed_input_t *packed = (embed_input_t *)malloc((size_t)batch * sizeof(*packed));
+ ffwd_input_t *packed = (ffwd_input_t *)malloc((size_t)batch * sizeof(*packed));
int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int));
int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int));
if (!packed || !h_starts || !h_lens) {
@@ -1782,7 +1769,7 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
int token_offset = 0;
int span_offset = 0;
for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
+ const ffwd_context_input_t *input = &inputs[b];
packed[b] = input->input;
for (int s = 0; s < input->n_spans; s++) {
h_starts[span_offset] = token_offset + input->spans[s].start;
@@ -1812,8 +1799,8 @@ int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
if (launch_check() != 0)
goto cleanup;
ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
- (size_t)total_spans * c->hidden_size * sizeof(float),
- cudaMemcpyDeviceToHost, ctx->stream);
+ (size_t)total_spans * c->hidden_size * sizeof(float), cudaMemcpyDeviceToHost,
+ ctx->stream);
if (ce == cudaSuccess)
ce = cudaStreamSynchronize(ctx->stream);
if (ce != cudaSuccess) {
@@ -1829,22 +1816,22 @@ cleanup:
return rc;
}
-int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- embed_input_t input = {token_ids, n_tokens};
- return embed_cuda_encode_batch(ctx, &input, 1, out_embedding);
+int ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ ffwd_input_t input = {token_ids, n_tokens};
+ return ffwd_cuda_encode_batch(ctx, &input, 1, out_embedding);
}
-float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) {
+float *ffwd_cuda_encode(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) {
if (!ctx)
return NULL;
int hidden = ctx->config.hidden_size;
float *out = (float *)malloc((size_t)hidden * sizeof(float));
if (!out)
return NULL;
- if (embed_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) {
+ if (ffwd_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) {
free(out);
return NULL;
}
@@ -1857,65 +1844,67 @@ float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_toke
* The transformer forward and the 1_Dense projection run on the GPU; MaxSim
* scoring stays on the host, where the grouped-GEMM scorer is far faster than a
* device graph for this small op. So the encoders copy the projected token
- * vectors back to host memory and the caller runs embed_late_maxsim_batch there.
+ * vectors back to host memory and the caller runs ffwd_late_maxsim_batch there.
* ======================================================================== */
-struct embed_cuda_late_ctx {
- embed_cuda_ctx_t *base; // device base model; does not own its CPU model
- embed_late_model_t *cpu_late; // owns the CPU base model + projection host data
- cuda_matrix_t projection; // device [token_dim, hidden]
- float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand
- size_t proj_cap; // capacity of proj_dev in floats
+struct ffwd_cuda_late_ctx {
+ ffwd_cuda_ctx_t *base; // device base model; does not own its CPU model
+ ffwd_late_model_t *cpu_late; // owns the CPU base model + projection host data
+ cuda_matrix_t projection; // device [token_dim, hidden]
+ float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand
+ size_t proj_cap; // capacity of proj_dev in floats
int token_dim;
};
-embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir) {
- embed_late_model_t *cpu_late = embed_late_model_load(model_dir);
+ffwd_cuda_late_ctx_t *ffwd_cuda_late_load(const char *model_dir) {
+ ffwd_late_model_t *cpu_late = ffwd_late_model_load(model_dir);
if (!cpu_late)
return NULL;
- embed_cuda_late_ctx_t *ctx = (embed_cuda_late_ctx_t *)calloc(1, sizeof(*ctx));
+ ffwd_cuda_late_ctx_t *ctx = (ffwd_cuda_late_ctx_t *)calloc(1, sizeof(*ctx));
if (!ctx) {
- embed_late_model_free(cpu_late);
+ ffwd_late_model_free(cpu_late);
return NULL;
}
ctx->cpu_late = cpu_late;
- ctx->token_dim = embed_late_model_token_dim(cpu_late);
+ ctx->token_dim = ffwd_late_model_token_dim(cpu_late);
// Upload the base transformer weights; the CPU late model keeps ownership.
- ctx->base = cuda_ctx_from_model(embed_late_model_base(cpu_late), 0);
+ ctx->base = cuda_ctx_from_model(ffwd_late_model_base(cpu_late), 0);
if (!ctx->base) {
- embed_cuda_late_free(ctx);
+ ffwd_cuda_late_free(ctx);
return NULL;
}
// Upload the 1_Dense projection [token_dim, hidden].
- const embed_weight_ref_t *proj = embed_late_model_projection(cpu_late);
+ const ffwd_weight_ref_t *proj = ffwd_late_model_projection(cpu_late);
if (!proj || ctx->token_dim <= 0 ||
load_matrix(&ctx->projection, proj, ctx->token_dim, ctx->base->config.hidden_size) != 0) {
- embed_cuda_late_free(ctx);
+ ffwd_cuda_late_free(ctx);
return NULL;
}
return ctx;
}
-void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx) {
+void ffwd_cuda_late_free(ffwd_cuda_late_ctx_t *ctx) {
if (!ctx)
return;
cuda_matrix_free(&ctx->projection);
cudaFree(ctx->proj_dev);
- embed_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model
- embed_late_model_free(ctx->cpu_late); // frees the CPU base model + projection
+ ffwd_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model
+ ffwd_late_model_free(ctx->cpu_late); // frees the CPU base model + projection
free(ctx);
}
-const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx) {
+const ffwd_config_t *ffwd_cuda_late_config(const ffwd_cuda_late_ctx_t *ctx) {
return ctx ? &ctx->base->config : NULL;
}
-int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
+int ffwd_cuda_late_token_dim(const ffwd_cuda_late_ctx_t *ctx) {
+ return ctx ? ctx->token_dim : 0;
+}
-static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) {
+static int ensure_late_proj(ffwd_cuda_late_ctx_t *ctx, int total) {
size_t need = (size_t)total * (size_t)ctx->token_dim;
if (need <= ctx->proj_cap)
return 0;
@@ -1932,13 +1921,13 @@ static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) {
* [total, token_dim] F32 result to a freshly malloc'd host buffer (caller frees).
* On success *proj_host and *total_out are set; base->offsets_host holds the
* packed per-document token offsets. */
-static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx,
- const embed_input_t *inputs,
+static int cuda_late_forward_project(ffwd_cuda_late_ctx_t *ctx,
+ const ffwd_input_t *inputs,
int n_docs,
float **proj_host,
int *total_out) {
- embed_cuda_ctx_t *base = ctx->base;
- const embed_config_t *c = &base->config;
+ ffwd_cuda_ctx_t *base = ctx->base;
+ const ffwd_config_t *c = &base->config;
if (cuda_forward_batch(base, inputs, n_docs) != 0)
return -1;
int total = base->offsets_host[n_docs];
@@ -1965,14 +1954,14 @@ static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx,
return 0;
}
-int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
+int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
if (!ctx || !token_ids || n_tokens <= 0 || !out_vectors || ctx->token_dim <= 0)
return -1;
- embed_input_t input = {token_ids, n_tokens};
+ ffwd_input_t input = {token_ids, n_tokens};
float *proj_host = NULL;
int total = 0;
if (cuda_late_forward_project(ctx, &input, 1, &proj_host, &total) != 0)
@@ -1981,27 +1970,27 @@ int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
free(proj_host);
if (normalize) {
for (int i = 0; i < total; i++)
- if (embed_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0)
+ if (ffwd_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0)
return -1;
}
return 0;
}
-int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets) {
+int ffwd_cuda_late_encode_docs(ffwd_cuda_late_ctx_t *ctx,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets) {
if (!ctx || !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors ||
!out_offsets || ctx->token_dim <= 0)
return -1;
int dim = ctx->token_dim;
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
+ ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
if (!inputs)
return -1;
@@ -2049,7 +2038,7 @@ int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
if (normalize) {
for (int i = 0; i < total_keep; i++)
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ if (ffwd_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
return -1;
}
return 0;
diff --git a/src/cuda.h b/libffwd/cuda.h
index aad20ab..8be1dd4 100644
--- a/src/cuda.h
+++ b/libffwd/cuda.h
@@ -1,31 +1,31 @@
/*
- * embed_cuda.h - CUDA/cuBLAS backend
+ * ffwd_cuda.h - CUDA/cuBLAS backend
*/
-#ifndef EMBED_CUDA_H
-#define EMBED_CUDA_H
+#ifndef FFWD_CUDA_H
+#define FFWD_CUDA_H
/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
+ * -fvisibility=hidden, so only declarations carrying FFWD_API are exported
* from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
+#ifndef FFWD_API
+# if defined(__GNUC__)
+# define FFWD_API __attribute__((visibility("default")))
+# else
+# define FFWD_API
+# endif
#endif
-#include "embed.h"
+#include "ffwd.h"
-typedef struct embed_cuda_ctx embed_cuda_ctx_t;
-typedef struct embed_cuda_late_ctx embed_cuda_late_ctx_t;
+typedef struct ffwd_cuda_ctx ffwd_cuda_ctx_t;
+typedef struct ffwd_cuda_late_ctx ffwd_cuda_late_ctx_t;
-EMBED_API int embed_cuda_set_fast_gemm(const char *mode);
-EMBED_API int embed_cuda_set_weights_bf16(int on);
+FFWD_API int ffwd_cuda_set_fast_gemm(const char *mode);
+FFWD_API int ffwd_cuda_set_weights_bf16(int on);
-EMBED_API embed_cuda_ctx_t *embed_cuda_load(const char *model_dir);
-EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx);
+FFWD_API ffwd_cuda_ctx_t *ffwd_cuda_load(const char *model_dir);
+FFWD_API void ffwd_cuda_free(ffwd_cuda_ctx_t *ctx);
/*
* Late-interaction CUDA path. Encoding (transformer forward + projection) runs
@@ -33,13 +33,13 @@ EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx);
* much faster than a device graph for this small op. The encoders therefore copy
* the projected token vectors back to host memory.
*/
-EMBED_API embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir);
-EMBED_API void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx);
-EMBED_API const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx);
-EMBED_API int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx);
+FFWD_API ffwd_cuda_late_ctx_t *ffwd_cuda_late_load(const char *model_dir);
+FFWD_API void ffwd_cuda_late_free(ffwd_cuda_late_ctx_t *ctx);
+FFWD_API const ffwd_config_t *ffwd_cuda_late_config(const ffwd_cuda_late_ctx_t *ctx);
+FFWD_API int ffwd_cuda_late_token_dim(const ffwd_cuda_late_ctx_t *ctx);
/* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */
-EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
+FFWD_API int ffwd_cuda_late_encode_tokens(ffwd_cuda_late_ctx_t *ctx,
const int *token_ids,
int n_tokens,
int normalize,
@@ -49,9 +49,9 @@ EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
* Encode n_docs documents in one packed forward and pack each document's kept
* token vectors back-to-back into out_vectors[total_keep, token_dim] (host) in
* document order. out_offsets[n_docs + 1] receives the prefix sum of kept counts
- * (the layout embed_late_maxsim_batch consumes). Returns 0 on success.
+ * (the layout ffwd_late_maxsim_batch consumes). Returns 0 on success.
*/
-EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
+FFWD_API int ffwd_cuda_late_encode_docs(ffwd_cuda_late_ctx_t *ctx,
const int *const *doc_ids,
const int *n_tokens,
const int *const *keep,
@@ -61,20 +61,20 @@ EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
float *out_vectors,
int *out_offsets);
-EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx);
+FFWD_API const ffwd_config_t *ffwd_cuda_config(const ffwd_cuda_ctx_t *ctx);
-EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
+FFWD_API int ffwd_cuda_encode_into(ffwd_cuda_ctx_t *ctx,
const int *token_ids,
int n_tokens,
float *out_embedding);
-EMBED_API float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens);
-EMBED_API int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
- const embed_input_t *inputs,
+FFWD_API float *ffwd_cuda_encode(ffwd_cuda_ctx_t *ctx, const int *token_ids, int n_tokens);
+FFWD_API int ffwd_cuda_encode_batch(ffwd_cuda_ctx_t *ctx,
+ const ffwd_input_t *inputs,
int batch,
float *out_embeddings);
-EMBED_API int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
- const embed_context_input_t *inputs,
+FFWD_API int ffwd_cuda_encode_spans_batch(ffwd_cuda_ctx_t *ctx,
+ const ffwd_context_input_t *inputs,
int batch,
float *out_embeddings);
-#endif /* EMBED_CUDA_H */
+#endif /* FFWD_CUDA_H */
diff --git a/src/dtype.h b/libffwd/dtype.h
index 2a25cae..7a7a2ad 100644
--- a/src/dtype.h
+++ b/libffwd/dtype.h
@@ -1,5 +1,5 @@
-#ifndef EMBED_DTYPE_H
-#define EMBED_DTYPE_H
+#ifndef FFWD_DTYPE_H
+#define FFWD_DTYPE_H
#include <stdint.h>
#include <string.h>
@@ -8,8 +8,8 @@
* left shift reinterpreted as a float. Shared by every host-side caller (the
* CPU loader, the safetensors reader); the CUDA backend keeps its own
* __nv_bfloat16 device kernels, and kernels.c keeps the SIMD buffer variant
- * embed_bf16_to_f32_buf for hot, contiguous conversions. */
-static inline float embed_bf16_to_f32(uint16_t bf16) {
+ * bf16_to_f32_buf for hot, contiguous conversions. */
+static inline float ffwd_bf16_to_f32(uint16_t bf16) {
uint32_t bits = (uint32_t)bf16 << 16;
float f;
memcpy(&f, &bits, sizeof(f));
@@ -22,7 +22,7 @@ static inline float embed_bf16_to_f32(uint16_t bf16) {
* subnormals, and inf/NaN. Used by the loaders to accept F16 safetensors
* snapshots (e.g. the public multilingual-e5 weights); there is no fused F16
* kernel, so weights are widened to F32 once at load. */
-static inline float embed_f16_to_f32(uint16_t f16) {
+static inline float ffwd_f16_to_f32(uint16_t f16) {
uint32_t sign = (uint32_t)(f16 & 0x8000u) << 16;
uint32_t exp = (f16 >> 10) & 0x1fu;
uint32_t mant = f16 & 0x3ffu;
@@ -52,4 +52,4 @@ static inline float embed_f16_to_f32(uint16_t f16) {
return f;
}
-#endif /* EMBED_DTYPE_H */
+#endif /* FFWD_DTYPE_H */
diff --git a/libffwd/ffwd.h b/libffwd/ffwd.h
new file mode 100644
index 0000000..6f947b6
--- /dev/null
+++ b/libffwd/ffwd.h
@@ -0,0 +1,280 @@
+/*
+ * ffwd.h - ffwd API
+ */
+
+#ifndef FFWD_H
+#define FFWD_H
+
+/* Public-API export annotation. The libraries are built with
+ * -fvisibility=hidden, so only declarations carrying FFWD_API are exported
+ * from the shared library; everything else stays internal. */
+#ifndef FFWD_API
+# if defined(__GNUC__)
+# define FFWD_API __attribute__((visibility("default")))
+# else
+# define FFWD_API
+# endif
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+/* ========================================================================
+ * Shared constants and family defaults
+ * ======================================================================== */
+
+#define FFWD_VOCAB_SIZE 151936
+#define FFWD_HEAD_DIM 128
+#define FFWD_MAX_LAYERS 64 /* upper bound for stack arrays */
+#define FFWD_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */
+
+/* Late-interaction (ColBERT) special-token id fallbacks and the query pad
+ * length, used when a model's tokenizer does not define the tokens itself. */
+#define FFWD_LATE_QUERY_TOKENS 32
+#define FFWD_LATE_MASK_TOKEN_ID 151642
+#define FFWD_LATE_QUERY_PREFIX_ID 151669
+#define FFWD_LATE_DOCUMENT_PREFIX_ID 151670
+
+/* ========================================================================
+ * Model Configuration (populated from config.json at load time)
+ * ======================================================================== */
+
+typedef enum {
+ FFWD_ATTENTION_BIDIRECTIONAL = 0,
+ FFWD_ATTENTION_CAUSAL = 1,
+} ffwd_attention_mode_t;
+
+typedef enum {
+ FFWD_POOL_MEAN = 0,
+ FFWD_POOL_LAST_TOKEN = 1,
+ FFWD_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */
+} ffwd_pooling_mode_t;
+
+typedef enum {
+ FFWD_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (Qwen2/3, pplx-embed) */
+ FFWD_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */
+} ffwd_family_t;
+
+/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf
+ * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or
+ * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */
+typedef enum {
+ FFWD_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */
+ FFWD_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */
+} ffwd_act_t;
+
+typedef struct {
+ ffwd_family_t family;
+ ffwd_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */
+ int hidden_size;
+ int n_layers;
+ int n_heads;
+ int n_kv_heads;
+ int head_dim;
+ int q_dim; /* n_heads * head_dim */
+ int kv_dim; /* n_kv_heads * head_dim */
+ int intermediate_size;
+ int vocab_size;
+ int max_position_embeddings; /* BERT learned position table size */
+ int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */
+ int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */
+ float rms_norm_eps;
+ float layer_norm_eps; /* BERT LayerNorm epsilon */
+ float rope_theta;
+ int qk_norm;
+ int qkv_bias;
+ ffwd_attention_mode_t attention_mode;
+ ffwd_pooling_mode_t pooling_mode;
+ int normalize_embeddings;
+ /* Qwen3-Embedding pools the last token, which is the tokenizer's
+ * <|endoftext|> suffix added by its post-processor - not the model's chat
+ * eos_token_id (<|im_end|>). Text frontends append it and resolve the id
+ * from the tokenizer exactly like the contextual separator
+ * (FFWD_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The
+ * model forward consumes token ids exactly and never appends. */
+ int append_terminal_token;
+} ffwd_config_t;
+
+typedef struct {
+ const int *ids;
+ int n_tokens;
+} ffwd_input_t;
+
+typedef struct {
+ int start;
+ int n_tokens;
+} ffwd_span_t;
+
+typedef struct {
+ ffwd_input_t input;
+ const ffwd_span_t *spans;
+ int n_spans;
+} ffwd_context_input_t;
+
+/* ========================================================================
+ * Engine API — the backend-selected inference handle
+ *
+ * This is the library's primary public interface and the entire supported
+ * surface of the shared library. Exactly one backend (CPU, Apple Silicon GPU,
+ * or NVIDIA GPU) is linked at build time and implements these functions;
+ * callers never branch on the backend. The CLI, the server, and any language
+ * binding use only these functions plus the math helpers below.
+ *
+ * A model loads in two phases so a GPU context can be created on the thread
+ * that will own it. ffwd_open runs on the caller's thread and loads a CPU
+ * model immediately; a GPU build only records what to load. ffwd_activate then
+ * runs on the inference thread and creates the GPU context (a no-op for CPU).
+ * Teardown mirrors this: ffwd_worker_free releases the GPU context from the
+ * inference thread, ffwd_free releases the rest.
+ * ======================================================================== */
+
+/* Opaque inference handle; its layout is private to the linked backend. */
+typedef struct embed ffwd_t;
+
+/* Backend tuning, parsed from the command line. A backend reads the fields it
+ * supports and ignores the rest, so the front-ends parse them with no backend
+ * branch. */
+typedef struct {
+ int n_threads; /* CPU worker threads; <= 0 auto-detects */
+ int gpu_quant_bits; /* GPU load-time weight quantization (0 or 8) */
+ int gpu_quant_group_size; /* GPU quantization group size (> 0) */
+ const char *gpu_gemm_mode; /* GPU GEMM compute mode, or NULL */
+ const char *gpu_weight_dtype; /* GPU weight storage dtype, or NULL */
+ double memory_utilization; /* GPU memory budget fraction; 0 = default */
+} ffwd_options_t;
+
+/* One late-interaction rerank: a query and N candidate documents, each given as
+ * token ids plus the kept-token indices. The vector dimension is the loaded
+ * late model's token_dim. */
+typedef struct {
+ const int *query_ids;
+ int query_n_tokens;
+ int query_n_keep;
+ const int *const *doc_ids;
+ const int *doc_n_tokens;
+ const int *const *doc_keep;
+ const int *doc_n_keep;
+ int n_docs;
+} ffwd_rerank_input_t;
+
+/* Apply process-wide settings before any load (CPU thread count, GPU GEMM and
+ * weight settings, GPU option checks) and print the startup banner. Returns -1 and
+ * writes a reason into err on a rejected option value. */
+FFWD_API int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen);
+
+/* First load phase, on the caller's thread: load a CPU model now, or record what
+ * a GPU build will load in ffwd_activate. is_late selects a late-interaction
+ * model. NULL on failure with a reason in err. */
+FFWD_API ffwd_t *ffwd_open(
+ const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen);
+/* Second load phase, on the inference thread: create the GPU context. CPU
+ * no-op. Returns -1 with a reason in err on failure. */
+FFWD_API int ffwd_activate(ffwd_t *e, char *err, size_t errlen);
+/* Release the inference-thread-owned GPU context; CPU no-op. Idempotent. */
+FFWD_API void ffwd_worker_free(ffwd_t *e);
+/* Release CPU state and the handle; call after ffwd_worker_free. */
+FFWD_API void ffwd_free(ffwd_t *e);
+
+/* Available once the model is loaded (after open for CPU, after activate for
+ * GPU); config is NULL until then. */
+FFWD_API const ffwd_config_t *ffwd_config(const ffwd_t *e);
+/* Late projection dimension, or 0 for a standard model. */
+FFWD_API int ffwd_token_dim(const ffwd_t *e);
+/* Whether the backend pads batches to a common length rather than packing them. */
+FFWD_API int ffwd_uses_dense_batches(const ffwd_t *e);
+
+/* Encode a packed batch into out[batch, hidden_size]. Returns 0 on success. */
+FFWD_API int
+ffwd_encode_batch(ffwd_t *e, const ffwd_input_t *inputs, int batch, float *out);
+/* Encode a contextual document batch, pooling each selected span in order. */
+FFWD_API int ffwd_encode_spans_batch(ffwd_t *e,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out);
+/* Score one query against its candidate documents into scores[n_docs]. */
+FFWD_API int ffwd_rerank(ffwd_t *e, const ffwd_rerank_input_t *in, float *scores);
+/* Scratch memory the model currently holds, for diagnostics; 0 on GPU. */
+FFWD_API size_t ffwd_scratch_nbytes(const ffwd_t *e);
+
+/* Capability label for --build-info and the startup banner: "CPU", "Apple
+ * Silicon GPU", or "NVIDIA GPU". A function, not a macro, so a shared object
+ * does not bake in one backend's label and stays correct across a rebuild. */
+FFWD_API const char *ffwd_capability(void);
+
+/* First-arrival micro-batch wait the backend prefers, in microseconds. */
+FFWD_API int ffwd_default_batch_wait_us(void);
+/* Reject a model set that would not fit host memory (GPU backends); 0 otherwise. On
+ * rejection returns -1 with a reason in err. paths are the model directories. */
+FFWD_API int ffwd_preflight(
+ const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen);
+
+/*
+ * L2-normalize vec[dim] in place.
+ *
+ * Returns 0 on success. Returns -1 for invalid arguments or a zero-length
+ * vector, which cannot be normalized.
+ */
+FFWD_API int ffwd_l2_normalize(float *vec, int dim);
+
+/*
+ * Cosine similarity between two vectors.
+ * Returns 0 for invalid arguments or if either vector has zero length.
+ */
+FFWD_API float ffwd_cosine_similarity(const float *a, const float *b, int dim);
+
+/* ========================================================================
+ * Tokenization — text into the token ids a model expects
+ *
+ * The tokenizer is backend-independent (text -> ids is identical on CPU or
+ * GPU), so it is its own handle, separate from ffwd_t. ffwd_tok_open picks the
+ * tokenizer by the files in model_dir (WordPiece vocab.txt, SentencePiece
+ * .model, or byte-level vocab.json) and resolves the model's special-token ids.
+ * ffwd_tokenize then applies that model's layout: BERT and XLM-R wrap the ids
+ * with [CLS]..[SEP] (<s>..</s>), Qwen appends its terminal token.
+ * ======================================================================== */
+
+/* Opaque tokenizer handle; owns the tokenizer and its resolved special tokens. */
+typedef struct ffwd_tok ffwd_tok_t;
+
+/* Late-interaction tokens: ids, plus the document positions to score. For a
+ * query, keep is NULL (every position is kept) and n_keep == n_tokens. */
+typedef struct {
+ int *ids;
+ int n_tokens;
+ int *keep;
+ int n_keep;
+} ffwd_late_tokens_t;
+
+/* Load the tokenizer for the model in model_dir and resolve its special-token
+ * ids. is_late additionally resolves the late-interaction [Q]/[D]/[MASK] ids.
+ * NULL on failure with a reason in err. */
+FFWD_API ffwd_tok_t *
+ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen);
+FFWD_API void ffwd_tok_free(ffwd_tok_t *t);
+
+/* Tokenize text for this model: applies the model's special-token layout and
+ * optionally prepends a query instruction. Returns a malloc'd id array (caller
+ * frees) and sets *n_out. NULL on error. */
+FFWD_API int *
+ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out);
+
+/* Late-interaction tokenization: [Q]/[D] prefix, [MASK]-pad queries to a fixed
+ * length, and mark the document positions to score. Returns 0 on success. */
+FFWD_API int
+ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out);
+FFWD_API void ffwd_late_tokens_free(ffwd_late_tokens_t *out);
+
+/* The contextual chunk-separator id (the tokenizer's <|endoftext|>), for
+ * frontends that split a document into chunks. */
+FFWD_API int ffwd_tok_context_separator_id(const ffwd_tok_t *t);
+
+/* Whether ffwd_tokenize appends the model's terminal token (<|endoftext|>) for
+ * last-token pooling. Off until set; the caller sets it from the loaded model's
+ * config (ffwd_config(...)->append_terminal_token), since it is model-specific
+ * and only known after the backend is open. */
+FFWD_API void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on);
+
+/* Verbose level: 0=quiet, 1=info, 2=debug */
+FFWD_API extern int ffwd_verbose;
+
+#endif /* FFWD_H */
diff --git a/src/forward.c b/libffwd/forward.c
index 28243af..e8a5cdf 100644
--- a/src/forward.c
+++ b/libffwd/forward.c
@@ -1,4 +1,4 @@
-#include "embed.h"
+#include "ffwd.h"
#include "model_internal.h"
#include "dtype.h"
#include "kernels.h"
@@ -11,8 +11,8 @@
/* Below these packed-sequence lengths the fused BF16 row kernels beat widening
* the weight to F32 and dispatching through BLAS; above them, widen once. */
-#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16
-#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4
+#define FFWD_BF16_QKV_FUSE_MAX_SEQ 16
+#define FFWD_BF16_PAIR_FUSE_MAX_SEQ 4
/* ========================================================================
* Linear projections (weight-dtype dispatch) and workspace scratch growth
@@ -20,10 +20,10 @@
static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) {
for (int i = 0; i < n; i++)
- dst[i] = embed_bf16_to_f32(src[i]);
+ dst[i] = ffwd_bf16_to_f32(src[i]);
}
-static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) {
+static void copy_weight_row(float *dst, const ffwd_weight_ref_t *w, size_t row, int row_width) {
if (w->dtype == DTYPE_F32) {
const float *src = (const float *)w->data + row * (size_t)row_width;
memcpy(dst, src, (size_t)row_width * sizeof(float));
@@ -33,7 +33,7 @@ static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row,
}
}
-static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) {
+static int ensure_bf16_widen(ffwd_workspace_t *ws, size_t count) {
if (ws->bf16_widen_count >= count)
return 0;
float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float));
@@ -44,15 +44,15 @@ static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) {
return 0;
}
-void linear_nobias_weight(embed_workspace_t *ws,
+void linear_nobias_weight(ffwd_workspace_t *ws,
float *y,
const float *x,
- const embed_weight_ref_t *w,
+ const ffwd_weight_ref_t *w,
int seq_len,
int in_dim,
int out_dim) {
if (w->dtype == DTYPE_F32) {
- embed_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim);
+ linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim);
return;
}
if (seq_len > 16) {
@@ -60,31 +60,30 @@ void linear_nobias_weight(embed_workspace_t *ws,
* BLAS-backed F32 path; below that the fused BF16 row kernels win. */
size_t count = (size_t)out_dim * (size_t)in_dim;
if (ensure_bf16_widen(ws, count) == 0) {
- embed_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count);
- embed_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim);
+ bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count);
+ linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim);
return;
}
}
- embed_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim);
+ linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim);
}
-static void linear_qkv_weight(embed_workspace_t *ws,
+static void linear_qkv_weight(ffwd_workspace_t *ws,
float *q,
float *k,
float *v,
const float *x,
- const embed_weight_ref_t *wq,
- const embed_weight_ref_t *wk,
- const embed_weight_ref_t *wv,
+ const ffwd_weight_ref_t *wq,
+ const ffwd_weight_ref_t *wk,
+ const ffwd_weight_ref_t *wv,
int seq_len,
int in_dim,
int q_dim,
int kv_dim) {
if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 &&
- seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) {
- embed_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data,
- (const uint16_t *)wk->data, (const uint16_t *)wv->data,
- seq_len, in_dim, q_dim, kv_dim);
+ seq_len <= FFWD_BF16_QKV_FUSE_MAX_SEQ) {
+ linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, (const uint16_t *)wk->data,
+ (const uint16_t *)wv->data, seq_len, in_dim, q_dim, kv_dim);
return;
}
@@ -103,20 +102,20 @@ static void add_bias_rows(float *x, const float *bias, int rows, int dim) {
}
}
-static void linear_pair_weight(embed_workspace_t *ws,
+static void linear_pair_weight(ffwd_workspace_t *ws,
float *a,
float *b,
const float *x,
- const embed_weight_ref_t *wa,
- const embed_weight_ref_t *wb,
+ const ffwd_weight_ref_t *wa,
+ const ffwd_weight_ref_t *wb,
int seq_len,
int in_dim,
int a_dim,
int b_dim) {
if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 &&
- seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) {
- embed_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data,
- (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim);
+ seq_len <= FFWD_BF16_PAIR_FUSE_MAX_SEQ) {
+ linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, (const uint16_t *)wb->data,
+ seq_len, in_dim, a_dim, b_dim);
return;
}
@@ -128,11 +127,18 @@ static void linear_pair_weight(embed_workspace_t *ws,
* Packed forward pass
*
* The workspace-growth helpers these call (ensure_buffers, ensure_rope_cache,
- * ensure_attention_scores, ensure_offsets) live in model.c beside the workspace
- * lifecycle and embed_workspace_nbytes, whose size accounting mirrors them.
+ * ensure_attention_scores, ensure_offsets) live in model_blas.c beside the
+ * workspace lifecycle and ffwd_workspace_nbytes, whose size accounting mirrors
+ * them.
* ======================================================================== */
-int build_offsets(const embed_input_t *inputs,
+static ffwd_attention_fn attention_for_config(const ffwd_config_t *cfg) {
+ return cfg->attention_mode == FFWD_ATTENTION_CAUSAL
+ ? causal_gqa_attention_packed_with_scratch
+ : bidirectional_gqa_attention_packed_with_scratch;
+}
+
+int build_offsets(const ffwd_input_t *inputs,
int batch,
int require_ids,
int *offsets,
@@ -159,9 +165,9 @@ int build_offsets(const embed_input_t *inputs,
return 0;
}
-static int forward_packed_slice_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
+static int forward_packed_slice_inplace(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
int batch,
const int *offsets,
int total_seq,
@@ -170,12 +176,12 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
int layer_start,
int layer_end,
int apply_final_norm) {
- if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets ||
- total_seq <= 0 || max_seq <= 0)
+ if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || total_seq <= 0 ||
+ max_seq <= 0)
return -1;
- const embed_config_t *cfg = &model->config;
- const embed_weights_t *w = &model->weights;
+ const ffwd_config_t *cfg = &model->config;
+ const ffwd_weights_t *w = &model->weights;
if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers ||
layer_start < model->layer_start || layer_end > model->layer_end ||
(layer_start == 0 && input_states) || (layer_start != 0 && !input_states) ||
@@ -219,7 +225,7 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
for (int i = 0; i < inputs[b].n_tokens; i++) {
int id = inputs[b].ids[i];
if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b,
+ fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b,
i);
return -1;
}
@@ -236,53 +242,54 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
/* 2. Transformer layers. Linear/MLP ops run over all packed tokens;
* attention is explicitly block-diagonal by sequence offsets. */
for (int layer = layer_start; layer < layer_end; layer++) {
- const embed_layer_t *l = &w->layers[layer];
+ const ffwd_layer_t *l = &w->layers[layer];
- embed_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
+ rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
- linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq,
- hidden, q_dim, kv_dim);
+ linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, hidden,
+ q_dim, kv_dim);
add_bias_rows(q_buf, l->q_bias, total_seq, q_dim);
add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim);
add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim);
if (cfg->qk_norm) {
- embed_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
- embed_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
+ rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
+ rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
}
for (int b = 0; b < batch; b++) {
int start = offsets[b];
int len = offsets[b + 1] - start;
- embed_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads,
- head_dim);
- embed_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len,
- n_kv_heads, head_dim);
+ apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads,
+ head_dim);
+ apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, n_kv_heads,
+ head_dim);
}
- model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads,
- head_dim, scale, ws->attn_scores, ws->attn_scores_bytes);
+ ffwd_attention_fn attention = attention_for_config(cfg);
+ attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, head_dim, scale,
+ ws->attn_scores, ws->attn_scores_bytes);
linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
+ add_inplace(x, proj_out, total_seq * hidden);
- embed_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
+ rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq,
hidden, inter, inter);
- embed_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
+ silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
+ add_inplace(x, proj_out, total_seq * hidden);
}
if (apply_final_norm)
- embed_rms_norm(x, x, w->norm, total_seq, hidden, eps);
+ rms_norm(x, x, w->norm, total_seq, hidden, eps);
return 0;
}
-int forward_packed_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
+int forward_packed_inplace(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
int batch,
const int *offsets,
int total_seq,
@@ -299,19 +306,19 @@ int forward_packed_inplace(const embed_model_t *model,
* cfg->position_id_offset (0 for BERT, pad_id + 1 for RoBERTa/XLM-R). Leaves the
* final hidden states in ws->x; pooling reads them as-is because the last
* layer's LayerNorm is already the model's final normalization. */
-static int forward_packed_bert(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
+static int forward_packed_bert(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
int batch,
const int *offsets,
int total_seq,
int max_seq) {
- if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets ||
- total_seq <= 0 || max_seq <= 0)
+ if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets || total_seq <= 0 ||
+ max_seq <= 0)
return -1;
- const embed_config_t *cfg = &model->config;
- const embed_weights_t *w = &model->weights;
+ const ffwd_config_t *cfg = &model->config;
+ const ffwd_weights_t *w = &model->weights;
int hidden = cfg->hidden_size;
int n_heads = cfg->n_heads;
int head_dim = cfg->head_dim;
@@ -319,11 +326,12 @@ static int forward_packed_bert(const embed_model_t *model,
float eps = cfg->layer_norm_eps;
if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings ||
- !w->embed_ln_w || !w->embed_ln_b)
+ !w->ffwd_ln_w || !w->ffwd_ln_b)
return -1;
if (max_seq > cfg->max_position_embeddings - cfg->position_id_offset) {
fprintf(stderr,
- "embed: sequence length %d plus position offset %d exceeds max_position_embeddings %d\n",
+ "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings "
+ "%d\n",
max_seq, cfg->position_id_offset, cfg->max_position_embeddings);
return -1;
}
@@ -342,22 +350,22 @@ static int forward_packed_bert(const embed_model_t *model,
for (int i = 0; i < inputs[b].n_tokens; i++) {
int id = inputs[b].ids[i];
if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, i);
+ fprintf(stderr, "ffwd: invalid token id %d at input %d position %d\n", id, b, i);
return -1;
}
float *row = x + (size_t)(start + i) * hidden;
copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden);
int pos = cfg->position_id_offset + i;
- embed_add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden);
- embed_add_inplace(row, w->token_type_embeddings, hidden);
+ add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden);
+ add_inplace(row, w->token_type_embeddings, hidden);
}
}
- embed_layer_norm(x, x, w->embed_ln_w, w->embed_ln_b, total_seq, hidden, eps);
+ layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, total_seq, hidden, eps);
float scale = 1.0f / sqrtf((float)head_dim);
for (int layer = 0; layer < cfg->n_layers; layer++) {
- const embed_layer_t *l = &w->layers[layer];
+ const ffwd_layer_t *l = &w->layers[layer];
/* Self-attention (post-norm): projections read x directly. */
linear_qkv_weight(ws, q_buf, k_buf, v_buf, x, &l->wq, &l->wk, &l->wv, total_seq, hidden,
@@ -366,35 +374,36 @@ static int forward_packed_bert(const embed_model_t *model,
add_bias_rows(k_buf, l->k_bias, total_seq, hidden);
add_bias_rows(v_buf, l->v_bias, total_seq, hidden);
- model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim,
- scale, ws->attn_scores, ws->attn_scores_bytes);
+ ffwd_attention_fn attention = attention_for_config(cfg);
+ attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim, scale,
+ ws->attn_scores, ws->attn_scores_bytes);
linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, hidden, hidden);
add_bias_rows(proj_out, l->o_bias, total_seq, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
- embed_layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps);
+ add_inplace(x, proj_out, total_seq * hidden);
+ layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps);
/* Feed-forward (post-norm): dense -> GeLU -> dense. */
linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter);
add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter);
- if (cfg->ffn_act == EMBED_ACT_GELU_TANH)
- embed_gelu_tanh_inplace(ffn_up, total_seq * inter);
+ if (cfg->ffn_act == FFWD_ACT_GELU_TANH)
+ gelu_tanh_inplace(ffn_up, total_seq * inter);
else
- embed_gelu_inplace(ffn_up, total_seq * inter);
+ gelu_inplace(ffn_up, total_seq * inter);
linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden);
add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
- embed_layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps);
+ add_inplace(x, proj_out, total_seq * hidden);
+ layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps);
}
return 0;
}
-static int pool_embeddings(const embed_model_t *model,
- const embed_workspace_t *ws,
+static int pool_embeddings(const ffwd_model_t *model,
+ const ffwd_workspace_t *ws,
const int *offsets,
int batch,
float *out_embeddings) {
- const embed_config_t *cfg = &model->config;
+ const ffwd_config_t *cfg = &model->config;
int hidden = cfg->hidden_size;
float eps = cfg->rms_norm_eps;
/* Qwen pools pre-final-norm states and applies the final RMSNorm here; BERT
@@ -410,10 +419,10 @@ static int pool_embeddings(const embed_model_t *model,
int len = end - start;
float *emb = out_embeddings + (size_t)b * hidden;
- if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) {
+ if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) {
/* Single-token pooling: CLS takes the first token, last-token the
* final one. */
- int idx = cfg->pooling_mode == EMBED_POOL_CLS ? start : end - 1;
+ int idx = cfg->pooling_mode == FFWD_POOL_CLS ? start : end - 1;
const float *row = x + (size_t)idx * hidden;
if (norm_weight) {
float sum_sq = 0.0f;
@@ -446,7 +455,7 @@ static int pool_embeddings(const embed_model_t *model,
emb[d] *= inv_len;
}
- if (cfg->normalize_embeddings && embed_l2_normalize(emb, hidden) != 0)
+ if (cfg->normalize_embeddings && ffwd_l2_normalize(emb, hidden) != 0)
return -1;
}
return 0;
@@ -456,11 +465,11 @@ static int pool_embeddings(const embed_model_t *model,
* Public forward/embed APIs
* ======================================================================== */
-int embed_model_forward_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_states) {
+int ffwd_model_forward_into(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_states) {
if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states)
return -1;
@@ -469,7 +478,7 @@ int embed_model_forward_into(const embed_model_t *model,
mul_size(count, sizeof(float), &bytes) != 0)
return -1;
- embed_input_t input = {token_ids, n_tokens};
+ ffwd_input_t input = {token_ids, n_tokens};
int offsets[2] = {0, n_tokens};
if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0)
return -1;
@@ -478,12 +487,12 @@ int embed_model_forward_into(const embed_model_t *model,
return 0;
}
-int embed_pool_spans(const embed_config_t *cfg,
- const float *states,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings) {
+int ffwd_pool_spans(const ffwd_config_t *cfg,
+ const float *states,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings)
return -1;
@@ -514,19 +523,19 @@ int embed_pool_spans(const embed_config_t *cfg,
return 0;
}
-int embed_model_encode_spans_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
+int ffwd_model_encode_spans_batch(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
return -1;
if (ensure_offsets(ws, batch) != 0)
return -1;
- if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t))
+ if ((size_t)batch > SIZE_MAX / sizeof(ffwd_input_t))
return -1;
- embed_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs));
+ ffwd_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs));
if (!packed_inputs)
return -1;
int *offsets = ws->offsets;
@@ -534,7 +543,7 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
int max_seq = 0;
int total_spans = 0;
for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
+ const ffwd_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) {
@@ -568,35 +577,35 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
int span_offset = 0;
int hidden = model->config.hidden_size;
for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
- if (embed_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden,
- input->input.n_tokens, input->spans, input->n_spans,
- out_embeddings + (size_t)span_offset * hidden) != 0)
+ const ffwd_context_input_t *input = &inputs[b];
+ if (ffwd_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden,
+ input->input.n_tokens, input->spans, input->n_spans,
+ out_embeddings + (size_t)span_offset * hidden) != 0)
return -1;
span_offset += input->n_spans;
}
return 0;
}
-int embed_model_encode_spans(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings) {
- embed_context_input_t input = {
+int ffwd_model_encode_spans(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
+ ffwd_context_input_t input = {
.input = {token_ids, n_tokens},
.spans = spans,
.n_spans = n_spans,
};
- return embed_model_encode_spans_batch(model, ws, &input, 1, out_embeddings);
+ return ffwd_model_encode_spans_batch(model, ws, &input, 1, out_embeddings);
}
-float *embed_model_forward(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens) {
+float *ffwd_model_forward(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
if (!model || !ws || !token_ids || n_tokens <= 0)
return NULL;
@@ -607,18 +616,18 @@ float *embed_model_forward(const embed_model_t *model,
if (!out)
return NULL;
- if (embed_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) {
+ if (ffwd_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) {
free(out);
return NULL;
}
return out;
}
-int embed_model_encode_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
+int ffwd_model_encode_batch(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
return -1;
@@ -631,7 +640,7 @@ int embed_model_encode_batch(const embed_model_t *model,
if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0)
return -1;
- int rc = model->config.family == EMBED_FAMILY_BERT
+ int rc = model->config.family == FFWD_FAMILY_BERT
? forward_packed_bert(model, ws, inputs, batch, offsets, total_seq, max_seq)
: forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0);
if (rc == 0)
@@ -640,11 +649,11 @@ int embed_model_encode_batch(const embed_model_t *model,
return rc;
}
-int embed_pool_batch(const embed_config_t *cfg,
- const float *states,
- const int *seq_lengths,
- int batch,
- float *out_embeddings) {
+int ffwd_pool_batch(const ffwd_config_t *cfg,
+ const float *states,
+ const int *seq_lengths,
+ int batch,
+ float *out_embeddings) {
if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0)
return -1;
@@ -655,10 +664,10 @@ int embed_pool_batch(const embed_config_t *cfg,
return -1;
float *emb = out_embeddings + (size_t)b * cfg->hidden_size;
- if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) {
+ if (cfg->pooling_mode == FFWD_POOL_LAST_TOKEN || cfg->pooling_mode == FFWD_POOL_CLS) {
/* CLS takes the first token, last-token the final one; these states
* already carry the model's final RMSNorm. */
- int idx = cfg->pooling_mode == EMBED_POOL_CLS ? offset : offset + len - 1;
+ int idx = cfg->pooling_mode == FFWD_POOL_CLS ? offset : offset + len - 1;
const float *row = states + (size_t)idx * cfg->hidden_size;
memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float));
} else {
@@ -672,26 +681,26 @@ int embed_pool_batch(const embed_config_t *cfg,
for (int d = 0; d < cfg->hidden_size; d++)
emb[d] *= inv_len;
}
- if (cfg->normalize_embeddings && embed_l2_normalize(emb, cfg->hidden_size) != 0)
+ if (cfg->normalize_embeddings && ffwd_l2_normalize(emb, cfg->hidden_size) != 0)
return -1;
offset += len;
}
return 0;
}
-int embed_model_encode_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- embed_input_t input = {token_ids, n_tokens};
- return embed_model_encode_batch(model, ws, &input, 1, out_embedding);
+int ffwd_model_encode_into(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ ffwd_input_t input = {token_ids, n_tokens};
+ return ffwd_model_encode_batch(model, ws, &input, 1, out_embedding);
}
-float *embed_model_encode(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens) {
+float *ffwd_model_encode(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
if (!model || !ws || !token_ids || n_tokens <= 0)
return NULL;
@@ -702,7 +711,7 @@ float *embed_model_encode(const embed_model_t *model,
if (!emb)
return NULL;
- if (embed_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) {
+ if (ffwd_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) {
free(emb);
return NULL;
}
diff --git a/src/kernels_gemm.c b/libffwd/gemm.c
index aa8dadf..bb54fb9 100644
--- a/src/kernels_gemm.c
+++ b/libffwd/gemm.c
@@ -1,5 +1,10 @@
+/*
+ * gemm.c - general Matrix Multiplication
+ */
+
#include "kernels.h"
-#include "kernels_impl.h"
+#include "impl.h"
+#include "threadpool.h"
#include <math.h>
#include <stdlib.h>
@@ -7,22 +12,22 @@
#if (defined(__AVX512F__) || defined(__AVX2__)) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
+# include <immintrin.h>
#endif
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
+# include <arm_neon.h>
#endif
#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
+# ifdef __APPLE__
+# include <Accelerate/Accelerate.h>
+# else
+# include <cblas.h>
+# endif
#endif
#ifndef M_PI
-#define M_PI 3.14159265358979323846
+# define M_PI 3.14159265358979323846
#endif
/* Matrix and vector ops: GEMM, the dtype-dispatch linear kernels, and the
@@ -32,7 +37,7 @@
* Basic Element-wise Operations
* ======================================================================== */
-void embed_add_inplace(float *a, const float *b, int n) {
+void add_inplace(float *a, const float *b, int n) {
for (int i = 0; i < n; i++)
a[i] += b[i];
}
@@ -41,7 +46,7 @@ void embed_add_inplace(float *a, const float *b, int n) {
* Matrix Operations
* ======================================================================== */
-void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) {
+void matmul_t(float *C, const float *A, const float *B, int M, int K, int N) {
#ifdef USE_BLAS
cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N);
#else
@@ -57,17 +62,11 @@ void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int
#endif
}
-void embed_linear(float *y,
- const float *x,
- const float *W,
- const float *b,
- int seq_len,
- int in_dim,
- int out_dim) {
+void linear(
+ float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim) {
#ifdef USE_BLAS
if (seq_len == 1) {
- cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y,
- 1);
+ cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y, 1);
} else {
cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x,
in_dim, W, in_dim, 0.0f, y, out_dim);
@@ -95,13 +94,12 @@ void embed_linear(float *y,
#endif
}
-void embed_linear_nobias(
- float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) {
- embed_linear(y, x, W, NULL, seq_len, in_dim, out_dim);
+void linear_nobias(float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) {
+ linear(y, x, W, NULL, seq_len, in_dim, out_dim);
}
/* Convert bf16 buffer to f32 buffer */
-void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
+void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
size_t i = 0;
uint32_t *d = (uint32_t *)(void *)dst;
@@ -127,7 +125,7 @@ void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
*/
static void bf16_matvec_fused(
float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- embed_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim);
+ bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim);
}
/* Threaded matvec: split output rows across threads */
@@ -156,7 +154,7 @@ static void matvec_worker(int tid, int n_threads, void *arg) {
static void bf16_matvec_threaded(
float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- if (embed_num_threads() <= 1) {
+ if (tp_num_threads() <= 1) {
bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim);
return;
}
@@ -207,7 +205,7 @@ static void bf16_linear_rows(float *y,
.out_dim = out_dim,
};
- if (embed_num_threads() > 1 && seq_len >= 2)
+ if (tp_num_threads() > 1 && seq_len >= 2)
parallel_for(bf16_linear_rows_worker, &task);
else
bf16_linear_rows_worker(0, 1, &task);
@@ -272,19 +270,19 @@ static void pair_matvec_worker(int tid, int n_threads, void *arg) {
}
}
-void embed_linear_nobias_bf16_pair(float *a,
- float *b,
- const float *x,
- const uint16_t *Wa_bf16,
- const uint16_t *Wb_bf16,
- int seq_len,
- int in_dim,
- int a_dim,
- int b_dim) {
+void linear_nobias_bf16_pair(float *a,
+ float *b,
+ const float *x,
+ const uint16_t *Wa_bf16,
+ const uint16_t *Wb_bf16,
+ int seq_len,
+ int in_dim,
+ int a_dim,
+ int b_dim) {
if (seq_len <= 0)
return;
- if (embed_num_threads() <= 1) {
+ if (tp_num_threads() <= 1) {
for (int s = 0; s < seq_len; s++) {
const float *x_row = x + (size_t)s * in_dim;
bf16_matvec_fused(a + (size_t)s * a_dim, x_row, Wa_bf16, NULL, in_dim, a_dim);
@@ -380,21 +378,21 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
}
}
-void embed_linear_nobias_bf16_qkv(float *q,
- float *k,
- float *v,
- const float *x,
- const uint16_t *Wq_bf16,
- const uint16_t *Wk_bf16,
- const uint16_t *Wv_bf16,
- int seq_len,
- int in_dim,
- int q_dim,
- int kv_dim) {
+void linear_nobias_bf16_qkv(float *q,
+ float *k,
+ float *v,
+ const float *x,
+ const uint16_t *Wq_bf16,
+ const uint16_t *Wk_bf16,
+ const uint16_t *Wv_bf16,
+ int seq_len,
+ int in_dim,
+ int q_dim,
+ int kv_dim) {
if (seq_len <= 0)
return;
- if (embed_num_threads() <= 1) {
+ if (tp_num_threads() <= 1) {
for (int s = 0; s < seq_len; s++) {
const float *x_row = x + (size_t)s * in_dim;
bf16_matvec_fused(q + (size_t)s * q_dim, x_row, Wq_bf16, NULL, in_dim, q_dim);
@@ -421,14 +419,13 @@ void embed_linear_nobias_bf16_qkv(float *q,
parallel_for(qkv_matvec_worker, &task);
}
-void embed_linear_nobias_bf16(
+void linear_nobias_bf16(
float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) {
if (seq_len == 1) {
bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim);
return;
}
/* Callers route longer sequences through an F32-widened weight matrix in
- * caller-owned scratch (see embed.c); this per-row path stays correct for
- * any length. */
+ * caller-owned scratch; this per-row path stays correct for any length. */
bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim);
}
diff --git a/src/kernels_generic.c b/libffwd/generic.c
index 7e85298..8798107 100644
--- a/src/kernels_generic.c
+++ b/libffwd/generic.c
@@ -1,12 +1,12 @@
/*
- * embed_kernels_generic.c - architecture-generic hot kernels
+ * generic.c - architecture-generic hot kernels / non-SIMD fallback
*/
-#include "kernels_impl.h"
+#include "impl.h"
#include <string.h>
-void embed_bf16_matvec_fused_generic(
+void bf16_matvec_fused_generic(
float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
for (int o = 0; o < out_dim; o++) {
const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
@@ -21,24 +21,24 @@ void embed_bf16_matvec_fused_generic(
}
}
-float embed_dot_f32_generic(const float *a, const float *b, int n) {
+float dot_f32_generic(const float *a, const float *b, int n) {
float sum = 0.0f;
for (int i = 0; i < n; i++)
sum += a[i] * b[i];
return sum;
}
-void embed_vec_scale_inplace_generic(float *dst, float scale, int n) {
+void vec_scale_inplace_generic(float *dst, float scale, int n) {
for (int i = 0; i < n; i++)
dst[i] *= scale;
}
-void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) {
+void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) {
for (int i = 0; i < n; i++)
dst[i] += alpha * src[i];
}
-void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n) {
+void vec_scale_add_generic(float *dst, const float *src, float correction, int n) {
for (int i = 0; i < n; i++)
dst[i] = dst[i] * correction + src[i];
}
diff --git a/libffwd/impl.h b/libffwd/impl.h
new file mode 100644
index 0000000..39baa40
--- /dev/null
+++ b/libffwd/impl.h
@@ -0,0 +1,279 @@
+/*
+ * impl.h - internal architecture dispatch for hot kernels
+ *
+ * The small vector primitives (dot product, scale, axpy, scale-add, scaled
+ * multiply) are defined here as static inline so they fold directly into the
+ * caller's loop: a row of RMSNorm or attention issues no call and the compiler
+ * keeps the row in registers across the reduction and the write-back. The
+ * generic scalar versions stay out of line in generic.c as a reference for the
+ * kernel tests and as the fallback on platforms with no SIMD path.
+ *
+ * bf16_matvec_fused is large (a full matrix-vector product) and is called once
+ * per output tile rather than inside a tight loop, so it stays out of line.
+ */
+
+#ifndef KERNELS_IMPL_H
+#define KERNELS_IMPL_H
+
+#include <stdint.h>
+
+/* Scalar references (generic.c): the kernel tests compare the SIMD results
+ * against these, and the no-SIMD build dispatches to them directly. */
+void bf16_matvec_fused_generic(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
+float dot_f32_generic(const float *a, const float *b, int n);
+void vec_scale_inplace_generic(float *dst, float scale, int n);
+void vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
+void vec_scale_add_generic(float *dst, const float *src, float correction, int n);
+
+#ifdef __ARM_NEON
+
+# include <arm_neon.h>
+
+void bf16_matvec_fused_neon(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
+# define bf16_matvec_fused_impl bf16_matvec_fused_neon
+
+static inline float dot_f32_impl(const float *a, const float *b, int n) {
+ int i = 0;
+ float32x4_t acc0 = vdupq_n_f32(0.0f);
+ float32x4_t acc1 = vdupq_n_f32(0.0f);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t a0 = vld1q_f32(a + i);
+ float32x4_t b0 = vld1q_f32(b + i);
+ float32x4_t a1 = vld1q_f32(a + i + 4);
+ float32x4_t b1 = vld1q_f32(b + i + 4);
+ acc0 = vfmaq_f32(acc0, a0, b0);
+ acc1 = vfmaq_f32(acc1, a1, b1);
+ }
+ float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
+ for (; i < n; i++)
+ sum += a[i] * b[i];
+ return sum;
+}
+
+static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
+ int i = 0;
+ float32x4_t s = vdupq_n_f32(scale);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
+ vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
+ }
+ for (; i < n; i++)
+ dst[i] *= scale;
+}
+
+static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
+ int i = 0;
+ float32x4_t a = vdupq_n_f32(alpha);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t s0 = vld1q_f32(src + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ float32x4_t s1 = vld1q_f32(src + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
+ vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
+ }
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
+}
+
+static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
+ int i = 0;
+ float32x4_t c = vdupq_n_f32(correction);
+ for (; i + 8 <= n; i += 8) {
+ float32x4_t d0 = vld1q_f32(dst + i);
+ float32x4_t s0 = vld1q_f32(src + i);
+ float32x4_t d1 = vld1q_f32(dst + i + 4);
+ float32x4_t s1 = vld1q_f32(src + i + 4);
+ vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
+ vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
+ }
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
+}
+
+#elif defined(__AVX2__) && defined(__FMA__)
+
+# include <immintrin.h>
+
+void bf16_matvec_fused_avx(
+ float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
+# define bf16_matvec_fused_impl bf16_matvec_fused_avx
+
+static inline float dot_f32_impl(const float *a, const float *b, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 acc0 = _mm512_setzero_ps();
+ __m512 acc1 = _mm512_setzero_ps();
+ __m512 acc2 = _mm512_setzero_ps();
+ __m512 acc3 = _mm512_setzero_ps();
+ for (; i + 64 <= n; i += 64) {
+ acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
+ acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
+ acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
+ acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
+ }
+ __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3));
+ for (; i + 16 <= n; i += 16) {
+ acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
+ }
+ float sum = _mm512_reduce_add_ps(acc);
+ for (; i < n; i++)
+ sum += a[i] * b[i];
+ return sum;
+# else
+ int i = 0;
+ __m256 acc0 = _mm256_setzero_ps();
+ __m256 acc1 = _mm256_setzero_ps();
+ __m256 acc2 = _mm256_setzero_ps();
+ __m256 acc3 = _mm256_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
+ acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1);
+ acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2);
+ acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3);
+ }
+ acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
+ for (; i + 8 <= n; i += 8) {
+ acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
+ }
+ __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
+ r = _mm_hadd_ps(r, r);
+ r = _mm_hadd_ps(r, r);
+ float sum = _mm_cvtss_f32(r);
+ for (; i < n; i++)
+ sum += a[i] * b[i];
+ return sum;
+# endif
+}
+
+static inline void vec_scale_inplace_impl(float *dst, float scale, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 s = _mm512_set1_ps(scale);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
+ _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
+ _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
+ _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
+ }
+ for (; i < n; i++)
+ dst[i] *= scale;
+# else
+ int i = 0;
+ __m256 s = _mm256_set1_ps(scale);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
+ _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s));
+ _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s));
+ _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
+ }
+ for (; i < n; i++)
+ dst[i] *= scale;
+# endif
+}
+
+static inline void vec_axpy_inplace_impl(float *dst, const float *src, float alpha, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 a = _mm512_set1_ps(alpha);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
+ _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a,
+ _mm512_loadu_ps(dst + i + 16)));
+ _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a,
+ _mm512_loadu_ps(dst + i + 32)));
+ _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a,
+ _mm512_loadu_ps(dst + i + 48)));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
+ }
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
+# else
+ int i = 0;
+ __m256 a = _mm256_set1_ps(alpha);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
+ _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a,
+ _mm256_loadu_ps(dst + i + 8)));
+ _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a,
+ _mm256_loadu_ps(dst + i + 16)));
+ _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a,
+ _mm256_loadu_ps(dst + i + 24)));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
+ }
+ for (; i < n; i++)
+ dst[i] += alpha * src[i];
+# endif
+}
+
+static inline void vec_scale_add_impl(float *dst, const float *src, float correction, int n) {
+# if defined(__AVX512F__)
+ int i = 0;
+ __m512 c = _mm512_set1_ps(correction);
+ for (; i + 64 <= n; i += 64) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
+ _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c,
+ _mm512_loadu_ps(src + i + 16)));
+ _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c,
+ _mm512_loadu_ps(src + i + 32)));
+ _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c,
+ _mm512_loadu_ps(src + i + 48)));
+ }
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(dst + i,
+ _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
+ }
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
+# else
+ int i = 0;
+ __m256 c = _mm256_set1_ps(correction);
+ for (; i + 32 <= n; i += 32) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
+ _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c,
+ _mm256_loadu_ps(src + i + 8)));
+ _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c,
+ _mm256_loadu_ps(src + i + 16)));
+ _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c,
+ _mm256_loadu_ps(src + i + 24)));
+ }
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(dst + i,
+ _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
+ }
+ for (; i < n; i++)
+ dst[i] = dst[i] * correction + src[i];
+# endif
+}
+
+#else /* no SIMD path: dispatch to the scalar references */
+
+# define bf16_matvec_fused_impl bf16_matvec_fused_generic
+# define dot_f32_impl dot_f32_generic
+# define vec_scale_inplace_impl vec_scale_inplace_generic
+# define vec_axpy_inplace_impl vec_axpy_inplace_generic
+# define vec_scale_add_impl vec_scale_add_generic
+
+#endif
+
+#endif /* KERNELS_IMPL_H */
diff --git a/libffwd/internal.h b/libffwd/internal.h
new file mode 100644
index 0000000..0338bc2
--- /dev/null
+++ b/libffwd/internal.h
@@ -0,0 +1,134 @@
+/*
+ * internal.h - non-exported, in-tree API shared by the frontends,
+ * backends, tests, and benchmarks. The public ABI is ffwd.h; none of these
+ * symbols carry FFWD_API visibility or appear in the export maps.
+ */
+
+#ifndef FFWD_INTERNAL_H
+#define FFWD_INTERNAL_H
+
+#include "ffwd.h"
+
+#include <stddef.h>
+#include <stdio.h>
+
+typedef struct ffwd_model ffwd_model_t;
+typedef struct ffwd_workspace ffwd_workspace_t;
+typedef struct ffwd_late_model ffwd_late_model_t;
+typedef struct ffwd_late_workspace ffwd_late_workspace_t;
+
+/* Backend-specific help text used by the in-tree frontends. */
+void ffwd_cli_help(FILE *f);
+void ffwd_server_help(FILE *f);
+
+/*
+ * Lower-level CPU model API. These functions are used by backend glue, tests,
+ * benchmarks, and parity harnesses; they are not exported from libffwd.
+ */
+ffwd_model_t *ffwd_model_load(const char *model_dir);
+void ffwd_model_free(ffwd_model_t *model);
+
+ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model);
+void ffwd_workspace_free(ffwd_workspace_t *ws);
+
+const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model);
+size_t ffwd_workspace_nbytes(const ffwd_workspace_t *ws);
+
+float *ffwd_model_encode(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens);
+
+int ffwd_model_encode_into(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
+
+int ffwd_model_encode_batch(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings);
+
+int ffwd_pool_batch(const ffwd_config_t *cfg,
+ const float *states,
+ const int *seq_lengths,
+ int batch,
+ float *out_embeddings);
+
+float *ffwd_model_forward(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens);
+
+int ffwd_model_forward_into(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_states);
+
+int ffwd_pool_spans(const ffwd_config_t *cfg,
+ const float *states,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
+
+int ffwd_model_encode_spans(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings);
+
+int ffwd_model_encode_spans_batch(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings);
+
+/*
+ * Late-interaction CPU model API. The public engine path exposes reranking via
+ * ffwd_rerank(); these entry points are for in-tree tests/backends only.
+ */
+ffwd_late_model_t *ffwd_late_model_load(const char *model_dir);
+void ffwd_late_model_free(ffwd_late_model_t *model);
+
+ffwd_late_workspace_t *ffwd_late_workspace_new(const ffwd_late_model_t *model);
+void ffwd_late_workspace_free(ffwd_late_workspace_t *ws);
+
+const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model);
+int ffwd_late_model_token_dim(const ffwd_late_model_t *model);
+
+int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model,
+ ffwd_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors);
+
+int ffwd_late_model_encode_docs(const ffwd_late_model_t *model,
+ ffwd_late_workspace_t *ws,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets);
+
+float ffwd_late_maxsim(
+ const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim);
+
+int ffwd_late_maxsim_batch(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores);
+
+#endif /* FFWD_INTERNAL_H */
diff --git a/src/kernels.h b/libffwd/kernels.h
index 17c45c0..f36d0d3 100644
--- a/src/kernels.h
+++ b/libffwd/kernels.h
@@ -1,22 +1,11 @@
/*
- * embed_kernels.h - Math kernels for Qwen3-architecture inference
+ * kernels.h - tkern public API: transformer inference kernels (CPU, float32)
*
* Low-level math operations. All operate on float32 tensors in row-major order.
*/
-#ifndef EMBED_KERNELS_H
-#define EMBED_KERNELS_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
+#ifndef KERNELS_H
+#define KERNELS_H
#include <stddef.h>
#include <stdint.h>
@@ -25,30 +14,30 @@
* Basic Operations
* ======================================================================== */
-void embed_add_inplace(float *a, const float *b, int n);
+void add_inplace(float *a, const float *b, int n);
/* ========================================================================
* Matrix Operations
* ======================================================================== */
/* C = A @ B^T: A[M,K], B[N,K], C[M,N] */
-void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N);
+void matmul_t(float *C, const float *A, const float *B, int M, int K, int N);
/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */
-void embed_linear(
+void linear(
float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim);
-void embed_linear_nobias(
+void linear_nobias(
float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim);
/* bf16 weight variants */
-void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n);
-void embed_linear_nobias_bf16(
+void bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n);
+void linear_nobias_bf16(
float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim);
/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */
-void embed_linear_nobias_bf16_pair(float *a,
+void linear_nobias_bf16_pair(float *a,
float *b,
const float *x,
const uint16_t *Wa_bf16,
@@ -59,7 +48,7 @@ void embed_linear_nobias_bf16_pair(float *a,
int b_dim);
/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */
-void embed_linear_nobias_bf16_qkv(float *q,
+void linear_nobias_bf16_qkv(float *q,
float *k,
float *v,
const float *x,
@@ -72,26 +61,26 @@ void embed_linear_nobias_bf16_qkv(float *q,
int kv_dim);
/* Dot product using the best available local SIMD implementation. */
-float embed_dot_f32(const float *a, const float *b, int n);
+float dot_f32(const float *a, const float *b, int n);
/* ========================================================================
* Normalization
* ======================================================================== */
/* RMS Normalization: out = x / rms(x) * weight */
-void embed_rms_norm(
+void rms_norm(
float *out, const float *x, const float *weight, int seq_len, int hidden, float eps);
/* Per-head RMS Normalization for Q/K norms in decoder
* x: [seq, n_heads, head_dim], weight: [head_dim]
* Normalizes each head independently */
-void embed_rms_norm_per_head(
+void rms_norm_per_head(
float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps);
/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps)
* + beta, with mean and biased (population) variance over the hidden axis per
* row. Unlike RMSNorm this subtracts the mean and adds a bias. */
-void embed_layer_norm(float *out,
+void layer_norm(float *out,
const float *x,
const float *gamma,
const float *beta,
@@ -103,17 +92,17 @@ void embed_layer_norm(float *out,
* Activation Functions
* ======================================================================== */
-void embed_softmax(float *x, int rows, int cols);
+void softmax(float *x, int rows, int cols);
/* gate = SiLU(gate) * up */
-void embed_silu_mul_inplace(float *gate, const float *up, int n);
+void silu_mul_inplace(float *gate, const float *up, int n);
/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the
* "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */
-void embed_gelu_inplace(float *x, int n);
+void gelu_inplace(float *x, int n);
/* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh):
* x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */
-void embed_gelu_tanh_inplace(float *x, int n);
+void gelu_tanh_inplace(float *x, int n);
/* ========================================================================
* Attention Operations
@@ -129,7 +118,7 @@ void embed_gelu_tanh_inplace(float *x, int n);
* Attention is block-diagonal: tokens only attend within their own
* offsets[b]..offsets[b+1] span.
*/
-void embed_bidirectional_gqa_attention_packed(float *out,
+void bidirectional_gqa_attention_packed(float *out,
const float *Q,
const float *K,
const float *V,
@@ -141,8 +130,8 @@ void embed_bidirectional_gqa_attention_packed(float *out,
float scale);
/* Optional reusable scratch for the BLAS-tiled packed attention path. */
-size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
-void embed_bidirectional_gqa_attention_packed_with_scratch(float *out,
+size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
+void bidirectional_gqa_attention_packed_with_scratch(float *out,
const float *Q,
const float *K,
const float *V,
@@ -159,7 +148,7 @@ void embed_bidirectional_gqa_attention_packed_with_scratch(float *out,
* Causal GQA attention over a packed/ragged batch. Query position i attends
* only to positions at or before i within its sequence.
*/
-void embed_causal_gqa_attention_packed(float *out,
+void causal_gqa_attention_packed(float *out,
const float *Q,
const float *K,
const float *V,
@@ -169,7 +158,7 @@ void embed_causal_gqa_attention_packed(float *out,
int n_kv_heads,
int head_dim,
float scale);
-void embed_causal_gqa_attention_packed_with_scratch(float *out,
+void causal_gqa_attention_packed_with_scratch(float *out,
const float *Q,
const float *K,
const float *V,
@@ -191,7 +180,7 @@ void embed_causal_gqa_attention_packed_with_scratch(float *out,
* cos_out, sin_out: [seq, head_dim]
* cos[d] and cos[half+d] are the same (duplicated for full head_dim).
*/
-void embed_compute_rope_neox(
+void compute_rope_neox(
float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta);
/*
@@ -199,18 +188,7 @@ void embed_compute_rope_neox(
* x: [seq, n_heads * head_dim] (in-place)
* cos_vals, sin_vals: [seq, head_dim]
*/
-void embed_apply_rope_neox(
+void apply_rope_neox(
float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim);
-/* ========================================================================
- * Threading
- * ======================================================================== */
-
-/* The persistent worker pool lives in threadpool.{c,h}; included here so kernel
- * callers keep getting embed_set_threads / embed_get_num_cpus from one header. */
-#include "threadpool.h"
-
-/* Global verbose flag */
-EMBED_API extern int embed_verbose;
-
-#endif /* EMBED_KERNELS_H */
+#endif /* KERNELS_H */
diff --git a/libffwd/late.c b/libffwd/late.c
new file mode 100644
index 0000000..05d0341
--- /dev/null
+++ b/libffwd/late.c
@@ -0,0 +1,218 @@
+#include "ffwd.h"
+#include "model_internal.h"
+#include "kernels.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include <float.h>
+
+/* ========================================================================
+ * Late-interaction model
+ *
+ * A late-interaction model is a pooled base model plus a 1_Dense linear
+ * projection from hidden states to token vectors. Instead of pooling to one
+ * vector, it keeps one projected vector per kept token and scores query/doc
+ * pairs with MaxSim. It reuses the base model's packed forward pass and
+ * workspace; only the projection, the token-vector buffer, and the scoring
+ * live here.
+ * ======================================================================== */
+
+struct ffwd_late_model {
+ ffwd_model_t *base;
+ safetensors_file_t *dense_sf;
+ ffwd_weight_ref_t projection; /* [token_dim, hidden] */
+ int token_dim;
+};
+
+static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
+ if (!sf || !name)
+ return NULL;
+ for (int i = 0; i < sf->num_tensors; i++) {
+ if (strcmp(sf->tensors[i].name, name) == 0)
+ return &sf->tensors[i];
+ }
+ return NULL;
+}
+
+ffwd_late_model_t *ffwd_late_model_load(const char *model_dir) {
+ if (!model_dir_has_late_projection(model_dir)) {
+ fprintf(stderr, "ffwd_late_model_load: missing 1_Dense projection\n");
+ return NULL;
+ }
+
+ ffwd_late_model_t *late = (ffwd_late_model_t *)calloc(1, sizeof(*late));
+ if (!late)
+ return NULL;
+
+ late->base = model_load_range_ex(model_dir, 0, -1, 1);
+ if (!late->base)
+ goto fail;
+
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ fprintf(stderr, "ffwd_late_model_load: model path too long\n");
+ goto fail;
+ }
+
+ late->dense_sf = safetensors_open(path);
+ if (!late->dense_sf) {
+ fprintf(stderr, "ffwd_late_model_load: cannot open %s\n", path);
+ goto fail;
+ }
+
+ const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
+ if (!t) {
+ fprintf(stderr, "ffwd_late_model_load: missing linear.weight\n");
+ goto fail;
+ }
+ if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) {
+ fprintf(stderr, "ffwd_late_model_load: bad projection shape\n");
+ goto fail;
+ }
+
+ int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
+ if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
+ goto fail;
+
+ /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */
+ late->projection = weight_ref_from_tensor(late->dense_sf, t);
+ if (!late->projection.data)
+ goto fail;
+ late->token_dim = (int)t->shape[0];
+ return late;
+
+fail:
+ ffwd_late_model_free(late);
+ return NULL;
+}
+
+void ffwd_late_model_free(ffwd_late_model_t *model) {
+ if (!model)
+ return;
+ ffwd_model_free(model->base);
+ /* Free the projection before the mmap: only an F16-widened ref owns its
+ * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */
+ if (model->projection.owned)
+ free((void *)model->projection.data);
+ safetensors_close(model->dense_sf);
+ free(model);
+}
+
+const ffwd_config_t *ffwd_late_model_config(const ffwd_late_model_t *model) {
+ return model && model->base ? &model->base->config : NULL;
+}
+
+int ffwd_late_model_token_dim(const ffwd_late_model_t *model) {
+ return model ? model->token_dim : 0;
+}
+
+/* Internal (internal.h): expose the base model and projection so GPU
+ * backends can upload them to device without seeing struct ffwd_late_model. */
+ffwd_model_t *ffwd_late_model_base(const ffwd_late_model_t *model) {
+ return model ? model->base : NULL;
+}
+
+const ffwd_weight_ref_t *ffwd_late_model_projection(const ffwd_late_model_t *model) {
+ return model ? &model->projection : NULL;
+}
+
+float ffwd_late_maxsim(
+ const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim) {
+ if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
+ return 0.0f;
+
+ float score = 0.0f;
+ for (int qi = 0; qi < query_tokens; qi++) {
+ const float *q = query_vectors + (size_t)qi * dim;
+ float best = -FLT_MAX;
+ for (int di = 0; di < doc_tokens; di++) {
+ const float *d = doc_vectors + (size_t)di * dim;
+ float dot = dot_f32(q, d, dim);
+ if (dot > best)
+ best = dot;
+ }
+ score += best;
+ }
+ return score;
+}
+
+int ffwd_late_maxsim_batch(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores) {
+ if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 || docs <= 0 ||
+ dim <= 0 || doc_offsets[0] != 0)
+ return -1;
+
+ for (int i = 0; i < docs; i++) {
+ if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i])
+ return -1;
+ }
+
+#ifdef USE_BLAS
+ /* One similarity GEMM per group of whole candidates:
+ * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document
+ * row-max plus sum. The group token budget bounds the scratch matrix;
+ * one oversized document still runs alone in a group of its own. */
+ enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
+ int total = doc_offsets[docs];
+ int budget = LATE_GROUP_TOKEN_BUDGET;
+ size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
+ int largest = 0;
+ for (int i = 0; i < docs; i++) {
+ int len = doc_offsets[i + 1] - doc_offsets[i];
+ if (len > largest)
+ largest = len;
+ }
+ if ((size_t)query_tokens * (size_t)largest > cap)
+ cap = (size_t)query_tokens * (size_t)largest;
+ float *sim = (float *)malloc(cap * sizeof(float));
+ if (sim) {
+ int doc = 0;
+ while (doc < docs) {
+ int first = doc;
+ int start = doc_offsets[first];
+ doc++;
+ while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
+ doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
+ doc++;
+ int group_tokens = doc_offsets[doc] - start;
+
+ matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
+ group_tokens);
+
+ for (int i = first; i < doc; i++) {
+ int s0 = doc_offsets[i] - start;
+ int s1 = doc_offsets[i + 1] - start;
+ float score = 0.0f;
+ for (int qi = 0; qi < query_tokens; qi++) {
+ const float *row = sim + (size_t)qi * group_tokens;
+ float best = row[s0];
+ for (int t = s0 + 1; t < s1; t++)
+ if (row[t] > best)
+ best = row[t];
+ score += best;
+ }
+ scores[i] = score;
+ }
+ }
+ free(sim);
+ return 0;
+ }
+ /* Allocation failure: fall through to the scalar path. */
+#endif
+
+ for (int i = 0; i < docs; i++) {
+ int start = doc_offsets[i];
+ int end = doc_offsets[i + 1];
+ scores[i] = ffwd_late_maxsim(query_vectors, query_tokens,
+ doc_vectors + (size_t)start * dim, end - start, dim);
+ }
+ return 0;
+}
diff --git a/libffwd/late_blas.c b/libffwd/late_blas.c
new file mode 100644
index 0000000..cc57596
--- /dev/null
+++ b/libffwd/late_blas.c
@@ -0,0 +1,173 @@
+#include "ffwd.h"
+#include "model_internal.h"
+
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* CPU late-interaction encoding. The late model loader and MaxSim scoring live
+ * in late.c because CUDA reuses those without linking the CPU transformer. */
+
+struct ffwd_late_workspace {
+ const ffwd_late_model_t *model;
+ ffwd_workspace_t *base_ws;
+ float *states;
+ int states_seq_cap;
+};
+
+static int ensure_late_state_buffer(ffwd_late_workspace_t *ws, int n_tokens) {
+ if (!ws || !ws->model || n_tokens <= 0)
+ return -1;
+ if (n_tokens <= ws->states_seq_cap)
+ return 0;
+ ffwd_model_t *base = ffwd_late_model_base(ws->model);
+ if (!base)
+ return -1;
+ int hidden = base->config.hidden_size;
+ if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
+ return -1;
+ ws->states_seq_cap = n_tokens;
+ return 0;
+}
+
+ffwd_late_workspace_t *ffwd_late_workspace_new(const ffwd_late_model_t *model) {
+ ffwd_model_t *base = ffwd_late_model_base(model);
+ if (!model || !base)
+ return NULL;
+ ffwd_late_workspace_t *ws = (ffwd_late_workspace_t *)calloc(1, sizeof(*ws));
+ if (!ws)
+ return NULL;
+ ws->model = model;
+ ws->base_ws = ffwd_workspace_new(base);
+ if (!ws->base_ws) {
+ free(ws);
+ return NULL;
+ }
+ return ws;
+}
+
+void ffwd_late_workspace_free(ffwd_late_workspace_t *ws) {
+ if (!ws)
+ return;
+ ffwd_workspace_free(ws->base_ws);
+ free(ws->states);
+ free(ws);
+}
+
+int ffwd_late_model_encode_tokens(const ffwd_late_model_t *model,
+ ffwd_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
+ int dim = ffwd_late_model_token_dim(model);
+ ffwd_model_t *base = ffwd_late_model_base(model);
+ const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model);
+ if (!model || !ws || ws->model != model || !base || !projection || !projection->data ||
+ !token_ids || n_tokens <= 0 || !out_vectors || dim <= 0)
+ return -1;
+
+ int hidden = base->config.hidden_size;
+ if (ensure_late_state_buffer(ws, n_tokens) != 0)
+ return -1;
+ if (ffwd_model_forward_into(base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
+ return -1;
+
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states, projection, n_tokens, hidden, dim);
+ if (normalize) {
+ for (int i = 0; i < n_tokens; i++) {
+ if (ffwd_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ return -1;
+ }
+ }
+ return 0;
+}
+
+/* Late-interaction batch encoder. Reuses the base model's packed/ragged batch
+ * forward pass, then gathers kept rows and projects them in one GEMM. */
+int ffwd_late_model_encode_docs(const ffwd_late_model_t *model,
+ ffwd_late_workspace_t *ws,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ float *out_vectors,
+ int *out_offsets) {
+ int dim = ffwd_late_model_token_dim(model);
+ ffwd_model_t *base = ffwd_late_model_base(model);
+ const ffwd_weight_ref_t *projection = ffwd_late_model_projection(model);
+ if (!model || !ws || ws->model != model || !base || !projection || !projection->data ||
+ !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors || !out_offsets ||
+ dim <= 0)
+ return -1;
+
+ int hidden = base->config.hidden_size;
+
+ ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
+ int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets));
+ if (!inputs || !offsets) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+
+ int total_keep = 0;
+ out_offsets[0] = 0;
+ for (int d = 0; d < n_docs; d++) {
+ if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
+ n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+ for (int k = 0; k < n_keep[d]; k++) {
+ if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
+ free(inputs);
+ free(offsets);
+ return -1;
+ }
+ }
+ inputs[d].ids = doc_ids[d];
+ inputs[d].n_tokens = n_tokens[d];
+ total_keep += n_keep[d];
+ out_offsets[d + 1] = total_keep;
+ }
+
+ int total_seq = 0, max_seq = 0;
+ int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq);
+ if (rc == 0)
+ rc =
+ forward_packed_inplace(base, ws->base_ws, inputs, n_docs, offsets, total_seq, max_seq, 1);
+ free(inputs);
+ if (rc != 0) {
+ free(offsets);
+ return -1;
+ }
+
+ if (ensure_late_state_buffer(ws, total_keep) != 0) {
+ free(offsets);
+ return -1;
+ }
+ const float *states = ws->base_ws->x;
+ int pos = 0;
+ for (int d = 0; d < n_docs; d++) {
+ int start = offsets[d];
+ for (int k = 0; k < n_keep[d]; k++) {
+ memcpy(ws->states + (size_t)pos * hidden, states + (size_t)(start + keep[d][k]) * hidden,
+ (size_t)hidden * sizeof(float));
+ pos++;
+ }
+ }
+ free(offsets);
+
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states, projection, total_keep, hidden, dim);
+ if (normalize) {
+ for (int i = 0; i < total_keep; i++) {
+ if (ffwd_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
+ return -1;
+ }
+ }
+ return 0;
+}
diff --git a/src/mlx.c b/libffwd/mlx.c
index 7d05e91..10299c1 100644
--- a/src/mlx.c
+++ b/libffwd/mlx.c
@@ -1,7 +1,7 @@
#include "mlx.h"
#include "config.h"
#include "dtype.h"
-#include "embed.h"
+#include "ffwd.h"
#include "safetensors.h"
#include <mlx/c/mlx.h>
@@ -33,19 +33,19 @@ typedef struct {
mlx_array q_norm, k_norm;
mlx_array input_norm, post_attn_norm;
mlx_linear_t gate_proj, up_proj, down_proj;
- /* BERT family (unset for Qwen3; see embed_internal.h). wo/up_proj/down_proj
+ /* BERT family (unset for Qwen3; see internal.h). wo/up_proj/down_proj
* and input_norm/post_attn_norm are reused as the BERT attention output,
* FFN matrices, and the two block LayerNorm weights. */
mlx_array o_bias, attn_ln_bias, ffn_inter_bias, ffn_out_bias, ffn_ln_bias;
} mlx_layer_t;
-struct embed_mlx_ctx {
- embed_config_t config;
+struct ffwd_mlx_ctx {
+ ffwd_config_t config;
mlx_array embed_tokens;
mlx_layer_t *layers; /* heap-allocated [n_layers] */
mlx_array norm;
/* BERT family (unset for Qwen3): learned positions, token-type, embed LN. */
- mlx_array position_embeddings, token_type_embeddings, embed_ln_w, embed_ln_b;
+ mlx_array position_embeddings, token_type_embeddings, ffwd_ln_w, ffwd_ln_b;
mlx_stream stream;
multi_safetensors_t *ms;
int layer_start; /* loaded transformer range, inclusive */
@@ -55,15 +55,15 @@ struct embed_mlx_ctx {
mlx_dtype weight_dtype;
};
-struct embed_mlx_late_ctx {
- embed_mlx_ctx_t *base;
+struct ffwd_mlx_late_ctx {
+ ffwd_mlx_ctx_t *base;
multi_safetensors_t *dense_ms;
mlx_linear_t projection;
int token_dim;
};
-struct embed_mlx_late_vectors {
- embed_mlx_late_ctx_t *owner;
+struct ffwd_mlx_late_vectors {
+ ffwd_mlx_late_ctx_t *owner;
mlx_array vectors; /* [tokens, dim] */
int tokens;
int dim;
@@ -82,7 +82,7 @@ static int mlx_mul_size(size_t a, size_t b, size_t *out) {
return 0;
}
-static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; }
+static mlx_dtype mlx_weight_dtype(const ffwd_mlx_ctx_t *ctx) { return ctx->weight_dtype; }
static size_t safetensor_dtype_size(safetensor_dtype_t dtype) {
switch (dtype) {
@@ -148,8 +148,7 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static mlx_array
-load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
+static mlx_array load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
safetensors_file_t *sf = NULL;
const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
@@ -169,7 +168,7 @@ load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndi
return (mlx_array){0};
const uint16_t *src = (const uint16_t *)safetensors_data(sf, t);
for (int64_t i = 0; i < n; i++)
- tmp[i] = embed_f16_to_f32(src[i]);
+ tmp[i] = ffwd_f16_to_f32(src[i]);
mlx_array a = mlx_array_new_data(tmp, shape, ndim, MLX_FLOAT32);
free(tmp);
return a;
@@ -295,8 +294,8 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) {
mlx_optional_int gs = {.value = W->group_size, .has_value = true};
mlx_optional_int bits = {.value = W->bits, .has_value = true};
mlx_array null_arr = (mlx_array){0};
- mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr,
- true, gs, bits, "affine", s);
+ mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr, true,
+ gs, bits, "affine", s);
return res;
}
mlx_array Wt;
@@ -328,7 +327,7 @@ static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) {
return res;
}
-const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) {
+const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx) {
return ctx ? &ctx->config : NULL;
}
@@ -365,12 +364,12 @@ static int mlx_model_dir_has_late_projection(const char *model_dir) {
mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors");
}
-static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) {
- embed_mlx_options_t out = {0};
+static ffwd_mlx_options_t normalize_mlx_options(const ffwd_mlx_options_t *opts) {
+ ffwd_mlx_options_t out = {0};
if (opts)
out = *opts;
if (out.quantize_bits != 0 && out.quantize_bits != 8) {
- fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n");
+ fprintf(stderr, "mlx: --gpu-quant-bits supports only 8\n");
out.quantize_bits = -1;
}
if (out.quantize_bits && out.quantize_group_size <= 0)
@@ -400,8 +399,8 @@ static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) {
/* BERT-family MLX loader, mirroring load_bert_weights on the CPU side. The
* shared post-load below (stream, transposes, optional BF16 upcast) runs after
* either family fills ctx->layers. */
-static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int layer_end) {
- const embed_config_t *c = &ctx->config;
+static int load_mlx_bert_weights(ffwd_mlx_ctx_t *ctx, int layer_start, int layer_end) {
+ const ffwd_config_t *c = &ctx->config;
int h = c->hidden_size, inter = c->intermediate_size;
char name[256];
@@ -429,12 +428,12 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye
ctx->token_type_embeddings = load_tensor(ctx->ms, name, type_shape, 2);
int ln_shape[] = {h};
snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p);
- ctx->embed_ln_w = load_tensor(ctx->ms, name, ln_shape, 1);
+ ctx->ffwd_ln_w = load_tensor(ctx->ms, name, ln_shape, 1);
snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p);
- ctx->embed_ln_b = load_tensor(ctx->ms, name, ln_shape, 1);
+ ctx->ffwd_ln_b = load_tensor(ctx->ms, name, ln_shape, 1);
if (!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->position_embeddings) ||
- !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->embed_ln_w) ||
- !arr_ok(ctx->embed_ln_b))
+ !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->ffwd_ln_w) ||
+ !arr_ok(ctx->ffwd_ln_b))
return -1;
}
@@ -483,12 +482,12 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye
return 0;
}
-static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
- int layer_start,
- int layer_end,
- const embed_mlx_options_t *opts,
- int allow_late) {
- embed_mlx_options_t options = normalize_mlx_options(opts);
+static ffwd_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
+ int layer_start,
+ int layer_end,
+ const ffwd_mlx_options_t *opts,
+ int allow_late) {
+ ffwd_mlx_options_t options = normalize_mlx_options(opts);
if (options.quantize_bits < 0)
return NULL;
if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) {
@@ -498,18 +497,18 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
return NULL;
}
- embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t));
+ ffwd_mlx_ctx_t *ctx = calloc(1, sizeof(ffwd_mlx_ctx_t));
if (!ctx)
return NULL;
ctx->quantize_bits = options.quantize_bits;
ctx->quantize_group_size = options.quantize_group_size;
- if (embed_config_parse(&ctx->config, model_dir) != 0) {
+ if (ffwd_config_parse(&ctx->config, model_dir) != 0) {
free(ctx);
return NULL;
}
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
if (layer_end == -1)
layer_end = c->n_layers;
if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) {
@@ -529,7 +528,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
return NULL;
}
- if (c->family == EMBED_FAMILY_BERT) {
+ if (c->family == FFWD_FAMILY_BERT) {
if (load_mlx_bert_weights(ctx, layer_start, layer_end) != 0)
goto fail;
goto loaded;
@@ -614,7 +613,7 @@ static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
LD(down_proj, "%slayers.%d.mlp.down_proj.weight", h, inter);
#undef LD
- if (embed_verbose >= 2)
+ if (ffwd_verbose >= 2)
fprintf(stderr, " mlx layer %d loaded\n", i);
}
@@ -655,8 +654,8 @@ loaded:
* embeddings and the embedding LayerNorm all feed the F32 residual. */
upcast_f32_inplace(&ctx->position_embeddings, ctx->stream);
upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream);
- upcast_f32_inplace(&ctx->embed_ln_w, ctx->stream);
- upcast_f32_inplace(&ctx->embed_ln_b, ctx->stream);
+ upcast_f32_inplace(&ctx->ffwd_ln_w, ctx->stream);
+ upcast_f32_inplace(&ctx->ffwd_ln_b, ctx->stream);
}
if (ctx->quantize_bits) {
for (int i = layer_start; i < layer_end; i++) {
@@ -665,7 +664,7 @@ loaded:
goto fail;
}
mlx_clear_cache();
- if (embed_verbose >= 1)
+ if (ffwd_verbose >= 1)
fprintf(stderr,
"mlx: quantized linear weights to %d-bit "
"(group_size=%d)\n",
@@ -677,7 +676,7 @@ loaded:
}
}
- if (embed_verbose >= 2) {
+ if (ffwd_verbose >= 2) {
size_t active = 0, cache = 0, peak = 0;
mlx_get_active_memory(&active);
mlx_get_cache_memory(&cache);
@@ -690,22 +689,22 @@ loaded:
(double)peak / (1024.0 * 1024.0 * 1024.0));
}
- if (embed_verbose >= 1)
+ if (ffwd_verbose >= 1)
fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h);
return ctx;
fail:
- embed_mlx_free(ctx);
+ ffwd_mlx_free(ctx);
return NULL;
}
-embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) {
+ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir) {
return mlx_load_range_ex(model_dir, 0, -1, NULL, 0);
}
-embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts) {
+ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir,
+ const ffwd_mlx_options_t *opts) {
return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
}
@@ -735,7 +734,7 @@ static void free_mlx_layer(mlx_layer_t *l) {
mlx_array_free(l->ffn_ln_bias);
}
-void embed_mlx_free(embed_mlx_ctx_t *ctx) {
+void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx) {
if (!ctx)
return;
mlx_array_free(ctx->embed_tokens);
@@ -747,8 +746,8 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx) {
mlx_array_free(ctx->norm);
mlx_array_free(ctx->position_embeddings);
mlx_array_free(ctx->token_type_embeddings);
- mlx_array_free(ctx->embed_ln_w);
- mlx_array_free(ctx->embed_ln_b);
+ mlx_array_free(ctx->ffwd_ln_w);
+ mlx_array_free(ctx->ffwd_ln_b);
if (ctx->stream.ctx)
mlx_stream_free(ctx->stream);
if (ctx->ms)
@@ -756,14 +755,14 @@ void embed_mlx_free(embed_mlx_ctx_t *ctx) {
free(ctx);
}
-embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts) {
+ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir,
+ const ffwd_mlx_options_t *opts) {
if (!mlx_model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "mlx late: missing 1_Dense projection\n");
return NULL;
}
- embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late));
+ ffwd_mlx_late_ctx_t *late = calloc(1, sizeof(*late));
if (!late)
return NULL;
@@ -803,34 +802,34 @@ embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
goto fail;
late->token_dim = shape[0];
- if (embed_verbose >= 1)
+ if (ffwd_verbose >= 1)
fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim);
return late;
fail:
- embed_mlx_late_free(late);
+ ffwd_mlx_late_free(late);
return NULL;
}
-embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) {
- return embed_mlx_late_load_with_options(model_dir, NULL);
+ffwd_mlx_late_ctx_t *ffwd_mlx_late_load(const char *model_dir) {
+ return ffwd_mlx_late_load_with_options(model_dir, NULL);
}
-void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) {
+void ffwd_mlx_late_free(ffwd_mlx_late_ctx_t *ctx) {
if (!ctx)
return;
free_linear(&ctx->projection);
if (ctx->dense_ms)
multi_safetensors_close(ctx->dense_ms);
- embed_mlx_free(ctx->base);
+ ffwd_mlx_free(ctx->base);
free(ctx);
}
-const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) {
+const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx) {
return ctx && ctx->base ? &ctx->base->config : NULL;
}
-int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
+int ffwd_mlx_late_token_dim(const ffwd_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
/* ========================================================================
* Forward pass
@@ -904,7 +903,7 @@ static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) {
/* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU
* feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the
* x it is given and returns the new hidden states. */
-static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
+static mlx_array mlx_forward_bert(ffwd_mlx_ctx_t *ctx,
mlx_array x,
int batch,
int max_seq,
@@ -912,7 +911,7 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
mlx_array attn_mask,
int layer_start,
int layer_end) {
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
int n_heads = c->n_heads, head_dim = c->head_dim, q_dim = c->q_dim;
float scale = 1.0f / sqrtf((float)head_dim);
@@ -990,8 +989,8 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
mlx_array mid_b = mlx_array_new();
mlx_add(&mid_b, mid, l->ffn_inter_bias, S);
mlx_array_free(mid);
- mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S)
- : mlx_gelu_erf(mid_b, S);
+ mlx_array g = ctx->config.ffn_act == FFWD_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S)
+ : mlx_gelu_erf(mid_b, S);
mlx_array_free(mid_b);
mlx_array out = linear(g, &l->down_proj, S);
mlx_array_free(g);
@@ -1009,7 +1008,7 @@ static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
return x;
}
-static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
+static mlx_array mlx_forward_layers(ffwd_mlx_ctx_t *ctx,
mlx_array x,
int batch,
int max_seq,
@@ -1017,7 +1016,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
mlx_array attn_mask,
int layer_start,
int layer_end) {
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
int n_heads = c->n_heads;
int n_kv_heads = c->n_kv_heads;
@@ -1025,7 +1024,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
int q_dim = c->q_dim;
float scale = 1.0f / sqrtf((float)head_dim);
mlx_array null_arr = (mlx_array){0};
- const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
+ const char *mask_mode = c->attention_mode == FFWD_ATTENTION_CAUSAL ? "causal" : "";
/* For non-quantized BF16 weights, run the residual stream, norms, and
* attention in F32; matmuls downcast their operands to BF16 internally
@@ -1034,8 +1033,7 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
* models are unaffected. */
mlx_array local_mask = attn_mask;
int free_mask = 0;
- int mixed_bf16 =
- mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized;
+ int mixed_bf16 = mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized;
if (mixed_bf16) {
if (mlx_array_dtype(x) != MLX_FLOAT32) {
mlx_array xf = mlx_array_new();
@@ -1172,20 +1170,20 @@ static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
return x;
}
-embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize) {
+ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize) {
if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 ||
!linear_ok(ctx->projection))
return NULL;
- embed_mlx_ctx_t *base = ctx->base;
+ ffwd_mlx_ctx_t *base = ctx->base;
if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
!arr_ok(base->embed_tokens) || !arr_ok(base->norm))
return NULL;
- const embed_config_t *c = &base->config;
+ const ffwd_config_t *c = &base->config;
mlx_stream S = base->stream;
for (int i = 0; i < n_tokens; i++) {
if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
@@ -1209,7 +1207,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx
if (!arr_ok(ids))
return NULL;
- embed_mlx_late_vectors_t *result = NULL;
+ ffwd_mlx_late_vectors_t *result = NULL;
mlx_array x = mlx_array_new();
mlx_array x_normed = (mlx_array){0};
mlx_array projected = (mlx_array){0};
@@ -1276,7 +1274,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx
if (!arr_ok(reshaped))
goto cleanup;
- result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
+ result = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*result));
if (!result)
goto cleanup;
result->owner = ctx;
@@ -1293,22 +1291,22 @@ cleanup:
return result;
}
-void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) {
+void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs) {
if (!vecs)
return;
mlx_array_free(vecs->vectors);
free(vecs);
}
-int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) {
+int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs) {
return vecs ? vecs->tokens : 0;
}
-int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) {
+int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs) {
return vecs ? vecs->dim : 0;
}
-int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) {
+int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors) {
if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0)
return -1;
@@ -1343,7 +1341,7 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out
* document's kept token vectors into a single packed [total_keep, token_dim]
* array in document order. out_offsets receives the prefix sum of kept counts
* (out_offsets[i + 1] - out_offsets[i] == n_keep[i]), the exact layout
- * embed_mlx_late_maxsim_batch_device consumes. This collapses N per-candidate
+ * ffwd_mlx_late_maxsim_batch_device consumes. This collapses N per-candidate
* transformer calls into one forward.
*
* Right-padding keeps real tokens at positions 0..n_tokens[d]-1 so RoPE
@@ -1352,26 +1350,26 @@ int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out
* batch path uses). Kept-token vectors therefore match per-document encoding to
* within MLX's batched-GEMM rounding.
*/
-embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- int *out_offsets) {
+ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx,
+ const int *const *doc_ids,
+ const int *n_tokens,
+ const int *const *keep,
+ const int *n_keep,
+ int n_docs,
+ int normalize,
+ int *out_offsets) {
if (!ctx || !ctx->base || !doc_ids || !n_tokens || !keep || !n_keep || !out_offsets ||
n_docs <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection))
return NULL;
- embed_mlx_ctx_t *base = ctx->base;
+ ffwd_mlx_ctx_t *base = ctx->base;
if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
!arr_ok(base->embed_tokens) || !arr_ok(base->norm))
return NULL;
- const embed_config_t *c = &base->config;
+ const ffwd_config_t *c = &base->config;
mlx_stream S = base->stream;
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
+ int causal = c->attention_mode == FFWD_ATTENTION_CAUSAL;
/* Validate inputs and size the batch: longest sequence and total kept
* tokens. out_offsets is the running prefix sum of kept counts. */
@@ -1476,7 +1474,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t
}
}
- embed_mlx_late_vectors_t *result = NULL;
+ ffwd_mlx_late_vectors_t *result = NULL;
mlx_array x = mlx_array_new();
mlx_array x_normed = (mlx_array){0};
mlx_array projected = (mlx_array){0};
@@ -1555,7 +1553,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t
goto cleanup;
}
- result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
+ result = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*result));
if (!result)
goto cleanup;
result->owner = ctx;
@@ -1576,17 +1574,17 @@ cleanup:
}
/* Device-resident select/concat primitives. The server rerank path batches
- * candidates through embed_mlx_late_encode_docs_device, but the late-interaction
- * verification harness (scripts/check_late_interaction.py) drives these directly
+ * candidates through ffwd_mlx_late_encode_docs_device, but the late-interaction
+ * verification harness (tests/integration/check_late_interaction.py) drives these directly
* to validate per-document gather and concat against the CPU MaxSim reference. */
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) {
+ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(
+ ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count) {
if (!ctx || !ctx->base || !items || count <= 0)
return NULL;
int dim = 0;
int total = 0;
for (int i = 0; i < count; i++) {
- const embed_mlx_late_vectors_t *v = items[i];
+ const ffwd_mlx_late_vectors_t *v = items[i];
if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0)
return NULL;
if (i == 0)
@@ -1616,7 +1614,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
return NULL;
}
- embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ ffwd_mlx_late_vectors_t *out = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*out));
if (!out) {
mlx_array_free(joined);
return NULL;
@@ -1628,10 +1626,10 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
return out;
}
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *vecs,
- const int *token_indices,
- int count) {
+ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx,
+ const ffwd_mlx_late_vectors_t *vecs,
+ const int *token_indices,
+ int count) {
if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) ||
!token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0)
return NULL;
@@ -1664,7 +1662,7 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ct
return NULL;
}
- embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
+ ffwd_mlx_late_vectors_t *out = (ffwd_mlx_late_vectors_t *)calloc(1, sizeof(*out));
if (!out) {
mlx_array_free(selected);
return NULL;
@@ -1676,12 +1674,12 @@ embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ct
return out;
}
-int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs,
- const int *doc_offsets,
- int docs_count,
- float *scores) {
+int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx,
+ const ffwd_mlx_late_vectors_t *query,
+ const ffwd_mlx_late_vectors_t *docs,
+ const int *doc_offsets,
+ int docs_count,
+ float *scores) {
if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores ||
query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim ||
query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) ||
@@ -1884,39 +1882,39 @@ cleanup:
return rc;
}
-int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- embed_mlx_late_vectors_t *vecs =
- embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize);
+int ffwd_mlx_late_encode_tokens(ffwd_mlx_late_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
+ ffwd_mlx_late_vectors_t *vecs =
+ ffwd_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize);
if (!vecs)
return -1;
- int rc = embed_mlx_late_vectors_copy(vecs, out_vectors);
- embed_mlx_late_vectors_free(vecs);
+ int rc = ffwd_mlx_late_vectors_copy(vecs, out_vectors);
+ ffwd_mlx_late_vectors_free(vecs);
return rc;
}
-static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
+static int ffwd_mlx_encode_batch_dense(ffwd_mlx_ctx_t *ctx,
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
ctx->layer_end != ctx->config.n_layers)
return -1;
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
int hidden = c->hidden_size;
int max_seq = 0;
int has_padding = 0;
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
- int cls_pool = c->pooling_mode == EMBED_POOL_CLS;
+ int causal = c->attention_mode == FFWD_ATTENTION_CAUSAL;
+ int cls_pool = c->pooling_mode == FFWD_POOL_CLS;
/* CLS and last-token are both single-token pooling: gather one row per
* sequence (the first for CLS, the last otherwise) rather than mean-pool. */
- int single_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN || cls_pool;
+ int single_token_pool = c->pooling_mode == FFWD_POOL_LAST_TOKEN || cls_pool;
if (hidden <= 0 || c->vocab_size <= 0)
return -1;
@@ -2023,9 +2021,9 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
: (mlx_array){0};
mlx_array lengths =
needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0};
- mlx_array last_indices =
- single_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
- : (mlx_array){0};
+ mlx_array last_indices = single_token_pool
+ ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
+ : (mlx_array){0};
free(padded_ids);
free(pool_mask_data);
free(attn_mask_data);
@@ -2065,7 +2063,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
* already final, so pooling uses them directly. Qwen runs the decoder block
* and applies the final RMSNorm here. */
mlx_array x_normed;
- if (c->family == EMBED_FAMILY_BERT) {
+ if (c->family == FFWD_FAMILY_BERT) {
int start2[] = {0, 0}, stride2[] = {1, 1};
int pos_start[] = {c->position_id_offset, 0};
int pos_stop[] = {c->position_id_offset + max_seq, hidden}, type_stop[] = {1, hidden};
@@ -2091,7 +2089,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array_free(xp);
mlx_array_free(tt);
mlx_array xe = mlx_array_new();
- mlx_fast_layer_norm(&xe, xt, ctx->embed_ln_w, ctx->embed_ln_b, c->layer_norm_eps, S);
+ mlx_fast_layer_norm(&xe, xt, ctx->ffwd_ln_w, ctx->ffwd_ln_b, c->layer_norm_eps, S);
mlx_array_free(xt);
x_normed =
mlx_forward_bert(ctx, xe, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
@@ -2149,7 +2147,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
if (c->normalize_embeddings) {
rc = 0;
for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) {
+ if (ffwd_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) {
rc = -1;
break;
}
@@ -2164,7 +2162,7 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
typedef struct {
int index;
- const embed_input_t *input;
+ const ffwd_input_t *input;
} mlx_batch_item_t;
static int mlx_batch_item_cmp(const void *a, const void *b) {
@@ -2181,7 +2179,7 @@ static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) {
return padded * 5 > tokens * 11;
}
-static int mlx_batch_should_split(const embed_input_t *inputs,
+static int mlx_batch_should_split(const ffwd_input_t *inputs,
int batch,
int *out_total_tokens,
int *out_max_seq) {
@@ -2210,7 +2208,7 @@ static int mlx_batch_should_split(const embed_input_t *inputs,
return mlx_padding_exceeds_limit(padded, (size_t)total_tokens);
}
-static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
+static int mlx_should_add_to_group(const ffwd_mlx_ctx_t *ctx,
int group_count,
int group_max,
int next_tokens) {
@@ -2232,19 +2230,19 @@ static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
return merged <= split_total;
}
-int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
+int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx,
+ const ffwd_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
if (!mlx_batch_should_split(inputs, batch, NULL, NULL))
- return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings);
+ return ffwd_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings);
const int hidden = ctx->config.hidden_size;
mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items));
- embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
+ ffwd_input_t *group_inputs = (ffwd_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
size_t group_values, group_bytes;
if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 ||
mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
@@ -2290,7 +2288,7 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
int group_count = end - start;
for (int i = 0; i < group_count; i++)
group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out);
+ rc = ffwd_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out);
if (rc != 0)
break;
for (int i = 0; i < group_count; i++) {
@@ -2307,18 +2305,18 @@ int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
return rc;
}
-int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- embed_input_t input = {token_ids, n_tokens};
- return embed_mlx_encode_batch(ctx, &input, 1, out_embedding);
+int ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ ffwd_input_t input = {token_ids, n_tokens};
+ return ffwd_mlx_encode_batch(ctx, &input, 1, out_embedding);
}
typedef struct {
int index;
int span_offset;
- const embed_context_input_t *input;
+ const ffwd_context_input_t *input;
} mlx_context_batch_item_t;
static int mlx_context_batch_item_cmp(const void *a, const void *b) {
@@ -2335,7 +2333,7 @@ static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) {
return padded * 4 > tokens * 5;
}
-static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
+static int mlx_context_batch_should_split(const ffwd_context_input_t *inputs,
int batch,
int *out_total_spans) {
int max_seq = 0;
@@ -2363,22 +2361,22 @@ static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens);
}
-static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
+static int ffwd_mlx_encode_spans_batch_dense(ffwd_mlx_ctx_t *ctx,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
return -1;
- const embed_config_t *c = &ctx->config;
+ const ffwd_config_t *c = &ctx->config;
mlx_stream S = ctx->stream;
int hidden = c->hidden_size;
int max_seq = 0;
int total_spans = 0;
int has_padding = 0;
for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
+ const ffwd_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
total_spans > INT_MAX - input->n_spans)
@@ -2424,7 +2422,7 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
return -1;
}
for (int b = 0; b < batch; b++) {
- const embed_input_t *input = &inputs[b].input;
+ const ffwd_input_t *input = &inputs[b].input;
for (int t = 0; t < max_seq; t++) {
size_t row = (size_t)b * max_seq + t;
if (t < input->n_tokens) {
@@ -2513,7 +2511,7 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
int pooled_count = 0;
for (int b = 0; b < batch; b++) {
for (int s = 0; s < inputs[b].n_spans; s++) {
- const embed_span_t *span = &inputs[b].spans[s];
+ const ffwd_span_t *span = &inputs[b].spans[s];
int start[] = {b, span->start, 0};
int stop[] = {b + 1, span->start + span->n_tokens, hidden};
int strides[] = {1, 1, 1};
@@ -2551,7 +2549,7 @@ static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
int span_count = 0;
for (int b = 0; b < batch; b++) {
for (int s = 0; s < inputs[b].n_spans; s++) {
- const embed_span_t *span = &inputs[b].spans[s];
+ const ffwd_span_t *span = &inputs[b].spans[s];
int base = b * max_seq;
int start = span->start;
int stop = span->start + span->n_tokens;
@@ -2664,22 +2662,22 @@ cleanup:
return rc;
}
-int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
+int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!ctx || !inputs || batch <= 0 || !out_embeddings)
return -1;
int total_spans = 0;
if (!mlx_context_batch_should_split(inputs, batch, &total_spans))
- return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings);
+ return ffwd_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings);
const int hidden = ctx->config.hidden_size;
mlx_context_batch_item_t *items =
(mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items));
- embed_context_input_t *group_inputs =
- (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
+ ffwd_context_input_t *group_inputs =
+ (ffwd_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
size_t group_values, group_bytes;
if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 ||
mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
@@ -2729,15 +2727,15 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
int group_count = end - start;
for (int i = 0; i < group_count; i++)
group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out);
+ rc = ffwd_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out);
if (rc != 0)
break;
int src_span = 0;
for (int i = 0; i < group_count; i++) {
int n_spans = group_inputs[i].n_spans;
int dst_span = items[start + i].span_offset;
- memcpy(out_embeddings + (size_t)dst_span * hidden,
- group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float));
+ memcpy(out_embeddings + (size_t)dst_span * hidden, group_out + (size_t)src_span * hidden,
+ (size_t)n_spans * hidden * sizeof(float));
src_span += n_spans;
}
start = end;
@@ -2749,21 +2747,21 @@ int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
return rc;
}
-int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings) {
- embed_context_input_t input = {
+int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx,
+ const int *token_ids,
+ int n_tokens,
+ const ffwd_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
+ ffwd_context_input_t input = {
.input = {token_ids, n_tokens},
.spans = spans,
.n_spans = n_spans,
};
- return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings);
+ return ffwd_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings);
}
-float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) {
+float *ffwd_mlx_encode(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) {
if (!ctx || !token_ids || n_tokens <= 0)
return NULL;
@@ -2775,7 +2773,7 @@ float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens
if (!out)
return NULL;
- if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) {
+ if (ffwd_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) {
free(out);
return NULL;
}
diff --git a/src/mlx.h b/libffwd/mlx.h
index 55f5666..d8a915f 100644
--- a/src/mlx.h
+++ b/libffwd/mlx.h
@@ -1,59 +1,59 @@
/*
- * embed_mlx.h - MLX GPU backend for Apple Silicon
+ * ffwd_mlx.h - MLX GPU backend for Apple Silicon
*
* Uses Apple's mlx-c to run the transformer on Metal.
*/
-#ifndef EMBED_MLX_H
-#define EMBED_MLX_H
+#ifndef FFWD_MLX_H
+#define FFWD_MLX_H
/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
+ * -fvisibility=hidden, so only declarations carrying FFWD_API are exported
* from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
+#ifndef FFWD_API
+# if defined(__GNUC__)
+# define FFWD_API __attribute__((visibility("default")))
+# else
+# define FFWD_API
+# endif
#endif
#include <stddef.h>
#include <stdint.h>
-#include "embed.h"
+#include "ffwd.h"
/* Opaque MLX model context */
-typedef struct embed_mlx_ctx embed_mlx_ctx_t;
-typedef struct embed_mlx_late_ctx embed_mlx_late_ctx_t;
-typedef struct embed_mlx_late_vectors embed_mlx_late_vectors_t;
+typedef struct ffwd_mlx_ctx ffwd_mlx_ctx_t;
+typedef struct ffwd_mlx_late_ctx ffwd_mlx_late_ctx_t;
+typedef struct ffwd_mlx_late_vectors ffwd_mlx_late_vectors_t;
typedef struct {
int quantize_bits; /* 0 disables; currently 8 */
int quantize_group_size; /* default 64 when quantize_bits is set */
-} embed_mlx_options_t;
+} ffwd_mlx_options_t;
/*
* Load model into MLX arrays from safetensors + config.json.
* Returns NULL on error.
*/
-EMBED_API embed_mlx_ctx_t *embed_mlx_load(const char *model_dir);
-EMBED_API embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts);
+FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load(const char *model_dir);
+FFWD_API ffwd_mlx_ctx_t *ffwd_mlx_load_with_options(const char *model_dir,
+ const ffwd_mlx_options_t *opts);
-EMBED_API void embed_mlx_free(embed_mlx_ctx_t *ctx);
+FFWD_API void ffwd_mlx_free(ffwd_mlx_ctx_t *ctx);
/*
* Late-interaction MLX path. This is separate from pooled embeddings because
* late snapshots produce one projected token vector per input token for MaxSim
* scoring, not one pooled document vector.
*/
-EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir);
-EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts);
-EMBED_API void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx);
+FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load(const char *model_dir);
+FFWD_API ffwd_mlx_late_ctx_t *ffwd_mlx_late_load_with_options(const char *model_dir,
+ const ffwd_mlx_options_t *opts);
+FFWD_API void ffwd_mlx_late_free(ffwd_mlx_late_ctx_t *ctx);
-EMBED_API const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx);
-EMBED_API int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx);
+FFWD_API const ffwd_config_t *ffwd_mlx_late_config(const ffwd_mlx_late_ctx_t *ctx);
+FFWD_API int ffwd_mlx_late_token_dim(const ffwd_mlx_late_ctx_t *ctx);
/*
* Encode token_ids into out_vectors[n_tokens, token_dim].
@@ -61,11 +61,8 @@ EMBED_API int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx);
* If normalize is non-zero, every token vector is L2-normalized on device
* before copying back to CPU memory.
*/
-EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors);
+FFWD_API int ffwd_mlx_late_encode_tokens(
+ ffwd_mlx_late_ctx_t *ctx, const int *token_ids, int n_tokens, int normalize, float *out_vectors);
/*
* Device-resident late-interaction token vectors.
@@ -74,22 +71,22 @@ EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
* outlive ctx. Copying is optional: MaxSim can consume these handles directly
* and copy back only final candidate scores.
*/
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
+FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_tokens_device(ffwd_mlx_late_ctx_t *ctx,
const int *token_ids,
int n_tokens,
int normalize);
-EMBED_API void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs);
-EMBED_API int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs);
-EMBED_API int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs);
-EMBED_API int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors);
+FFWD_API void ffwd_mlx_late_vectors_free(ffwd_mlx_late_vectors_t *vecs);
+FFWD_API int ffwd_mlx_late_vectors_token_count(const ffwd_mlx_late_vectors_t *vecs);
+FFWD_API int ffwd_mlx_late_vectors_dim(const ffwd_mlx_late_vectors_t *vecs);
+FFWD_API int ffwd_mlx_late_vectors_copy(const ffwd_mlx_late_vectors_t *vecs, float *out_vectors);
/*
* Encode n_docs documents in one padded transformer pass and pack each
* document's kept token vectors back-to-back into [total_keep, token_dim] in
* document order. out_offsets[n_docs + 1] receives the prefix sum of kept
- * counts, so the result feeds embed_mlx_late_maxsim_batch_device directly.
+ * counts, so the result feeds ffwd_mlx_late_maxsim_batch_device directly.
* Returns NULL on error; the handle must not outlive ctx.
*/
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx,
+FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_encode_docs_device(ffwd_mlx_late_ctx_t *ctx,
const int *const *doc_ids,
const int *n_tokens,
const int *const *keep,
@@ -98,18 +95,18 @@ EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_
int normalize,
int *out_offsets);
/* Device-resident select/concat primitives, retained for the late-interaction
- * verification harness (scripts/check_late_interaction.py); the server rerank
- * path uses embed_mlx_late_encode_docs_device. */
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count);
-EMBED_API embed_mlx_late_vectors_t *
-embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *vecs,
+ * verification harness (tests/integration/check_late_interaction.py); the server rerank
+ * path uses ffwd_mlx_late_encode_docs_device. */
+FFWD_API ffwd_mlx_late_vectors_t *ffwd_mlx_late_vectors_concat(
+ ffwd_mlx_late_ctx_t *ctx, const ffwd_mlx_late_vectors_t *const *items, int count);
+FFWD_API ffwd_mlx_late_vectors_t *
+ffwd_mlx_late_vectors_select(ffwd_mlx_late_ctx_t *ctx,
+ const ffwd_mlx_late_vectors_t *vecs,
const int *token_indices,
int count);
-EMBED_API int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs,
+FFWD_API int ffwd_mlx_late_maxsim_batch_device(ffwd_mlx_late_ctx_t *ctx,
+ const ffwd_mlx_late_vectors_t *query,
+ const ffwd_mlx_late_vectors_t *docs,
const int *doc_offsets,
int docs_count,
float *scores);
@@ -118,16 +115,14 @@ EMBED_API int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
* Compute embedding for token_ids[0..n_tokens-1].
* Returns malloc'd float[hidden_size] (caller frees). NULL on error.
*/
-EMBED_API float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens);
+FFWD_API float *ffwd_mlx_encode(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens);
/*
* Compute one embedding into caller-provided out[hidden_size].
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
+FFWD_API int
+ffwd_mlx_encode_into(ffwd_mlx_ctx_t *ctx, const int *token_ids, int n_tokens, float *out_embedding);
/*
* Compute a true padded dense batch on MLX.
@@ -136,8 +131,8 @@ EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
* Pooling and attention masking follow the loaded model configuration.
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
+FFWD_API int ffwd_mlx_encode_batch(ffwd_mlx_ctx_t *ctx,
+ const ffwd_input_t *inputs,
int batch,
float *out_embeddings);
@@ -145,10 +140,10 @@ EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
* Run one contextual sequence and pool selected token spans.
* Returns 0 on success, -1 on error.
*/
-EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
+FFWD_API int ffwd_mlx_encode_spans(ffwd_mlx_ctx_t *ctx,
const int *token_ids,
int n_tokens,
- const embed_span_t *spans,
+ const ffwd_span_t *spans,
int n_spans,
float *out_embeddings);
@@ -156,12 +151,12 @@ EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
* Run a padded dense contextual document batch and pool every selected span.
* out_embeddings contains spans in document order.
*/
-EMBED_API int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
+FFWD_API int ffwd_mlx_encode_spans_batch(ffwd_mlx_ctx_t *ctx,
+ const ffwd_context_input_t *inputs,
int batch,
float *out_embeddings);
-/* Get the config (so embed_cli.c can read hidden_size etc.) */
-EMBED_API const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx);
+/* Get the config (so ffwd_cli.c can read hidden_size etc.) */
+FFWD_API const ffwd_config_t *ffwd_mlx_config(const ffwd_mlx_ctx_t *ctx);
-#endif /* EMBED_MLX_H */
+#endif /* FFWD_MLX_H */
diff --git a/src/model.c b/libffwd/model.c
index c39c526..faecfd4 100644
--- a/src/model.c
+++ b/libffwd/model.c
@@ -1,19 +1,12 @@
-#include "embed.h"
+#include "ffwd.h"
#include "model_internal.h"
#include "config.h"
-#include "kernels.h"
+#include "dtype.h"
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
-#include <math.h>
-#include <limits.h>
-
-/* ========================================================================
- * Globals
- * ======================================================================== */
-
-int embed_verbose = 0;
+#include <string.h>
/* ========================================================================
* Model-directory probes
@@ -81,12 +74,12 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
* to F32 by the caller (BF16 lazily per matmul, F16 once at load). */
int sixteen_ok = bf16_ok && (t->dtype == DTYPE_BF16 || t->dtype == DTYPE_F16);
if (t->dtype != DTYPE_F32 && !sixteen_ok) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name,
+ fprintf(stderr, "ffwd: unsupported dtype for %s: got %s, expected %s\n", name,
dtype_name(t->dtype), bf16_ok ? "F32, BF16, or F16" : "F32");
return 0;
}
if (t->ndim != ndim) {
- fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
+ fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
return 0;
}
@@ -94,13 +87,13 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
for (int i = 0; i < ndim; i++) {
if (t->shape[i] != shape[i]) {
fprintf(stderr,
- "embed: bad shape for %s at dim %d: "
+ "ffwd: bad shape for %s at dim %d: "
"got %lld, expected %lld\n",
name, i, (long long)t->shape[i], (long long)shape[i]);
return 0;
}
if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) {
- fprintf(stderr, "embed: shape too large for %s\n", name);
+ fprintf(stderr, "ffwd: shape too large for %s\n", name);
return 0;
}
numel *= (size_t)shape[i];
@@ -108,17 +101,17 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
size_t elem_size = dtype_size(t->dtype);
if (elem_size == 0) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype));
+ fprintf(stderr, "ffwd: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype));
return 0;
}
size_t bytes;
if (mul_size(numel, elem_size, &bytes) != 0) {
- fprintf(stderr, "embed: tensor data too large for %s\n", name);
+ fprintf(stderr, "ffwd: tensor data too large for %s\n", name);
return 0;
}
if (t->data_size != bytes) {
- fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
+ fprintf(stderr, "ffwd: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
bytes);
return 0;
}
@@ -126,14 +119,47 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
size_t data_start = 8 + sf->header_size;
if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
t->data_size > sf->file_size - data_start - t->data_offset) {
- fprintf(stderr, "embed: tensor data out of bounds for %s\n", name);
+ fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name);
return 0;
}
return 1;
}
-embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t) {
- embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
+/* Widen a safetensors tensor to f32 (caller frees). libsafetensors is a pure
+ * reader now, so the dtype conversion lives here, next to its only callers. */
+static float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) {
+ int64_t n = safetensor_numel(t);
+ if (n <= 0)
+ return NULL;
+ float *out = malloc((size_t)n * sizeof(float));
+ if (!out)
+ return NULL;
+ const void *data = safetensors_data(sf, t);
+ switch (t->dtype) {
+ case DTYPE_F32:
+ memcpy(out, data, (size_t)n * sizeof(float));
+ break;
+ case DTYPE_BF16: {
+ const uint16_t *s = (const uint16_t *)data;
+ for (int64_t i = 0; i < n; i++)
+ out[i] = ffwd_bf16_to_f32(s[i]);
+ break;
+ }
+ case DTYPE_F16: {
+ const uint16_t *s = (const uint16_t *)data;
+ for (int64_t i = 0; i < n; i++)
+ out[i] = ffwd_f16_to_f32(s[i]);
+ break;
+ }
+ default:
+ free(out);
+ return NULL;
+ }
+ return out;
+}
+
+ffwd_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t) {
+ ffwd_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
/* F32/BF16 are consumed directly by the forward pass, so the ref borrows the
* mmap (zero copy; BF16 is widened lazily per matmul). F16 has neither a
* fused kernel nor a zero-copy F32 view, so widen it once here into an owned
@@ -152,13 +178,13 @@ embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const sa
return ref;
}
-static embed_weight_ref_t
+static ffwd_weight_ref_t
load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
- embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
+ ffwd_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
safetensors_file_t *sf = NULL;
const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
- fprintf(stderr, "embed: tensor not found: %s\n", name);
+ fprintf(stderr, "ffwd: tensor not found: %s\n", name);
return ref;
}
if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1))
@@ -172,7 +198,7 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i
safetensors_file_t *sf = NULL;
const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
- fprintf(stderr, "embed: tensor not found: %s\n", name);
+ fprintf(stderr, "ffwd: tensor not found: %s\n", name);
return NULL;
}
if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1))
@@ -186,13 +212,13 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i
/* BERT-family weight loader. The tensor names differ entirely from the Qwen
* block (encoder.layer.N.*, embeddings.*), so the family gets its own entry
- * point. Reused embed_layer_t slots carry their BERT meaning: wo is the
+ * point. Reused ffwd_layer_t slots carry their BERT meaning: wo is the
* attention output dense, input_norm/post_attn_norm are the two block LayerNorm
- * weights, up_proj/down_proj are the two FFN matrices (see embed_internal.h). */
+ * weights, up_proj/down_proj are the two FFN matrices (see internal.h). */
static int
-load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) {
- const embed_config_t *cfg = &model->config;
- embed_weights_t *w = &model->weights;
+load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) {
+ const ffwd_config_t *cfg = &model->config;
+ ffwd_weights_t *w = &model->weights;
char name[256];
/* sentence-transformers checkpoints store encoder tensors unprefixed; raw
@@ -205,7 +231,7 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start
else if (multi_safetensors_find(ms, "roberta.embeddings.word_embeddings.weight", NULL))
p = "roberta.";
else {
- fprintf(stderr, "embed_model_load: BERT-family word embeddings not found\n");
+ fprintf(stderr, "ffwd_model_load: BERT-family word embeddings not found\n");
return -1;
}
@@ -223,20 +249,20 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start
w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2);
const int64_t ln_shape[1] = {hidden};
snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p);
- w->embed_ln_w = load_norm_f32(ms, name, ln_shape, 1);
+ w->ffwd_ln_w = load_norm_f32(ms, name, ln_shape, 1);
snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p);
- w->embed_ln_b = load_norm_f32(ms, name, ln_shape, 1);
+ w->ffwd_ln_b = load_norm_f32(ms, name, ln_shape, 1);
if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings ||
- !w->embed_ln_w || !w->embed_ln_b)
+ !w->ffwd_ln_w || !w->ffwd_ln_b)
return -1;
}
- w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t));
+ w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t));
if (!w->layers)
return -1;
for (int i = layer_start; i < layer_end; i++) {
- embed_layer_t *l = &w->layers[i];
+ ffwd_layer_t *l = &w->layers[i];
#define BERT_W(field, suffix, d0, d1) \
do { \
@@ -279,31 +305,31 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start
return 0;
}
-embed_model_t *
+ffwd_model_t *
model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) {
if (!allow_late && model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "embed_model_load: late-interaction models require "
+ fprintf(stderr, "ffwd_model_load: late-interaction models require "
"token-level MaxSim support and are not valid pooled "
"embedding models yet\n");
return NULL;
}
- embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t));
+ ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t));
if (!model)
return NULL;
/* Parse config.json */
- if (embed_config_parse(&model->config, model_dir) != 0) {
+ if (ffwd_config_parse(&model->config, model_dir) != 0) {
free(model);
return NULL;
}
- const embed_config_t *cfg = &model->config;
+ const ffwd_config_t *cfg = &model->config;
if (layer_end == -1)
layer_end = cfg->n_layers;
if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) {
fprintf(stderr,
- "embed_model_load: invalid layer range [%d, %d) "
+ "ffwd_model_load: invalid layer range [%d, %d) "
"for %d-layer model\n",
layer_start, layer_end, cfg->n_layers);
free(model);
@@ -311,42 +337,39 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a
}
model->layer_start = layer_start;
model->layer_end = layer_end;
- model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL
- ? embed_causal_gqa_attention_packed_with_scratch
- : embed_bidirectional_gqa_attention_packed_with_scratch;
/* Open safetensors (handles single file or multi-shard) */
multi_safetensors_t *ms = multi_safetensors_open(model_dir);
if (!ms) {
- fprintf(stderr, "embed_model_load: failed to open safetensors in %s\n", model_dir);
+ fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir);
free(model);
return NULL;
}
model->safetensors = ms;
- if (cfg->family == EMBED_FAMILY_BERT) {
+ if (cfg->family == FFWD_FAMILY_BERT) {
if (load_bert_weights(model, ms, layer_start, layer_end) != 0)
goto fail;
- if (embed_verbose >= 1)
- fprintf(stderr, "embed_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start,
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "ffwd_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start,
layer_end, cfg->hidden_size);
return model;
}
const char *weight_prefix = multi_safetensors_weight_prefix(ms, "embed_tokens.weight");
if (!weight_prefix) {
- fprintf(stderr, "embed_model_load: neither embed_tokens.weight nor "
+ fprintf(stderr, "ffwd_model_load: neither embed_tokens.weight nor "
"model.embed_tokens.weight was found\n");
goto fail;
}
- if (embed_verbose >= 1)
- fprintf(stderr, "embed_model_load: loading weights from %s\n", model_dir);
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "ffwd_model_load: loading weights from %s\n", model_dir);
char name[256];
/* Only the first stage needs token embeddings. */
- embed_weights_t *w = &model->weights;
+ ffwd_weights_t *w = &model->weights;
if (layer_start == 0) {
const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size};
snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix);
@@ -356,13 +379,13 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a
}
/* Allocate layer array */
- w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t));
+ w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t));
if (!w->layers)
goto fail;
/* Per-layer weights */
for (int i = layer_start; i < layer_end; i++) {
- embed_layer_t *l = &w->layers[i];
+ ffwd_layer_t *l = &w->layers[i];
#define LOAD_NORM(field, fmt, d0) \
do { \
@@ -406,7 +429,7 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a
#undef LOAD2
#undef LOAD_NORM
- if (embed_verbose >= 2)
+ if (ffwd_verbose >= 2)
fprintf(stderr, " layer %d loaded\n", i);
}
@@ -419,38 +442,38 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a
goto fail;
}
- if (embed_verbose >= 1)
+ if (ffwd_verbose >= 1)
fprintf(stderr,
- "embed_model_load: layers [%d, %d) loaded "
+ "ffwd_model_load: layers [%d, %d) loaded "
"(%d-dim embeddings)\n",
layer_start, layer_end, cfg->hidden_size);
return model;
fail:
- embed_model_free(model);
+ ffwd_model_free(model);
return NULL;
}
-embed_model_t *embed_model_load(const char *model_dir) {
+ffwd_model_t *ffwd_model_load(const char *model_dir) {
return model_load_range_ex(model_dir, 0, -1, 0);
}
/* Release a weight ref's backing store. Only F16-widened refs own their buffer;
* borrowed F32/BF16 views into the mmap are a no-op (see weight_ref_from_tensor). */
-static void free_weight_ref(embed_weight_ref_t *w) {
+static void free_weight_ref(ffwd_weight_ref_t *w) {
if (w->owned)
free((void *)w->data);
w->data = NULL;
w->owned = 0;
}
-void embed_model_free(embed_model_t *model) {
+void ffwd_model_free(ffwd_model_t *model) {
if (!model)
return;
if (model->weights.layers) {
for (int i = 0; i < model->config.n_layers; i++) {
- embed_layer_t *l = &model->weights.layers[i];
+ ffwd_layer_t *l = &model->weights.layers[i];
free_weight_ref(&l->wq);
free_weight_ref(&l->wk);
free_weight_ref(&l->wv);
@@ -476,213 +499,14 @@ void embed_model_free(embed_model_t *model) {
free((void *)model->weights.norm);
free((void *)model->weights.position_embeddings);
free((void *)model->weights.token_type_embeddings);
- free((void *)model->weights.embed_ln_w);
- free((void *)model->weights.embed_ln_b);
+ free((void *)model->weights.ffwd_ln_w);
+ free((void *)model->weights.ffwd_ln_b);
if (model->safetensors)
multi_safetensors_close((multi_safetensors_t *)model->safetensors);
free(model->weights.layers);
free(model);
}
-embed_workspace_t *embed_workspace_new(const embed_model_t *model) {
- if (!model)
- return NULL;
- embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t));
- if (!ws)
- return NULL;
- ws->model = model;
- return ws;
-}
-
-void embed_workspace_free(embed_workspace_t *ws) {
- if (!ws)
- return;
- free(ws->x);
- free(ws->x_norm);
- free(ws->q);
- free(ws->k);
- free(ws->v);
- free(ws->attn_out);
- free(ws->proj_out);
- free(ws->ffn_gate);
- free(ws->ffn_up);
- free(ws->attn_scores);
- free(ws->offsets);
- free(ws->rope_cos);
- free(ws->rope_sin);
- free(ws->bf16_widen);
- free(ws);
-}
-
-const embed_config_t *embed_model_config(const embed_model_t *model) {
+const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) {
return model ? &model->config : NULL;
}
-
-/* ========================================================================
- * Workspace buffer growth
- *
- * Lazy per-call growth of the forward-pass scratch. Lives here, beside the
- * workspace lifecycle and embed_workspace_nbytes below: nbytes reproduces these
- * allocation sizes, so the allocator and the accountant stay in one file.
- * ======================================================================== */
-
-int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) {
- if (seq <= ws->buf_seq_cap)
- return 0;
-
- int cap;
- if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0)
- return -1;
-
-#define R(ptr, n) \
- do { \
- if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \
- return -1; \
- } while (0)
-
- R(ws->x, c->hidden_size);
- R(ws->x_norm, c->hidden_size);
- R(ws->q, c->q_dim);
- R(ws->k, c->kv_dim);
- R(ws->v, c->kv_dim);
- R(ws->attn_out, c->q_dim);
- R(ws->proj_out, c->hidden_size);
- R(ws->ffn_gate, c->intermediate_size);
- R(ws->ffn_up, c->intermediate_size);
-
-#undef R
-
- ws->buf_seq_cap = cap;
- return 0;
-}
-
-void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) {
- size_t bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (bytes <= ws->attn_scores_bytes)
- return;
-
- void *p = realloc(ws->attn_scores, bytes);
- if (!p)
- return;
- ws->attn_scores = (float *)p;
- ws->attn_scores_bytes = bytes;
-}
-
-int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) {
- if (n_pos <= ws->rope_cache_cap)
- return 0;
-
- int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512;
- while (cap < n_pos) {
- if (cap > INT_MAX / 2)
- return -1;
- cap *= 2;
- }
-
- int head_dim = cfg->head_dim;
- int half = head_dim / 2;
- float theta = cfg->rope_theta;
-
- size_t count;
- if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0)
- return -1;
- if (realloc_floats(&ws->rope_cos, count) != 0)
- return -1;
- if (realloc_floats(&ws->rope_sin, count) != 0)
- return -1;
-
- for (int pos = ws->rope_cache_cap; pos < cap; pos++) {
- float fp = (float)pos;
- float *c = ws->rope_cos + (size_t)pos * head_dim;
- float *s = ws->rope_sin + (size_t)pos * head_dim;
- for (int d = 0; d < half; d++) {
- float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
- float angle = fp * inv;
- float cv = cosf(angle), sv = sinf(angle);
- c[d] = cv;
- c[half + d] = cv;
- s[d] = sv;
- s[half + d] = sv;
- }
- }
-
- ws->rope_cache_cap = cap;
- return 0;
-}
-
-int ensure_offsets(embed_workspace_t *ws, int batch) {
- if (!ws || batch <= 0 || batch == INT_MAX)
- return -1;
-
- int needed = batch + 1;
- if (needed <= ws->offsets_cap)
- return 0;
-
- if (realloc_ints(&ws->offsets, (size_t)needed) != 0)
- return -1;
- ws->offsets_cap = needed;
- return 0;
-}
-
-size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
- if (!ws)
- return 0;
-
- size_t total = sizeof(*ws);
- const embed_model_t *model = ws->model;
- if (!model)
- return total;
-
- const embed_config_t *c = &model->config;
- size_t row_floats = 0;
- const int dims[] = {
- c->hidden_size, /* x */
- c->hidden_size, /* x_norm */
- c->q_dim, /* q */
- c->kv_dim, /* k */
- c->kv_dim, /* v */
- c->q_dim, /* attn_out */
- c->hidden_size, /* proj_out */
- c->intermediate_size, /* ffn_gate */
- c->intermediate_size, /* ffn_up */
- };
-
- for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) {
- if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0)
- return SIZE_MAX;
- }
-
- size_t buf_floats = 0;
- if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0)
- return SIZE_MAX;
-
- size_t rope_floats = 0;
- if (ws->rope_cache_cap > 0) {
- size_t rope_rows = 0;
- if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 ||
- mul_size(rope_rows, 2, &rope_floats) != 0)
- return SIZE_MAX;
- }
-
- size_t all_floats = 0, bytes = 0;
- if (add_size(buf_floats, rope_floats, &all_floats) != 0 ||
- mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0)
- return SIZE_MAX;
-
- size_t offset_bytes = 0;
- if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0)
- return SIZE_MAX;
- if (add_size(total, offset_bytes, &total) != 0)
- return SIZE_MAX;
- if (add_size(total, ws->attn_scores_bytes, &total) != 0)
- return SIZE_MAX;
-
- size_t widen_bytes = 0;
- if (ws->bf16_widen_count > 0 &&
- mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0)
- return SIZE_MAX;
- if (add_size(total, widen_bytes, &total) != 0)
- return SIZE_MAX;
-
- return total;
-}
diff --git a/libffwd/model_blas.c b/libffwd/model_blas.c
new file mode 100644
index 0000000..aae2a44
--- /dev/null
+++ b/libffwd/model_blas.c
@@ -0,0 +1,201 @@
+#include "ffwd.h"
+#include "model_internal.h"
+#include "kernels.h"
+
+#include <limits.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/* CPU workspace ownership and scratch growth. GPU builds use their own
+ * backend contexts and do not link this translation unit. */
+
+ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model) {
+ if (!model)
+ return NULL;
+ ffwd_workspace_t *ws = (ffwd_workspace_t *)calloc(1, sizeof(ffwd_workspace_t));
+ if (!ws)
+ return NULL;
+ ws->model = model;
+ return ws;
+}
+
+void ffwd_workspace_free(ffwd_workspace_t *ws) {
+ if (!ws)
+ return;
+ free(ws->x);
+ free(ws->x_norm);
+ free(ws->q);
+ free(ws->k);
+ free(ws->v);
+ free(ws->attn_out);
+ free(ws->proj_out);
+ free(ws->ffn_gate);
+ free(ws->ffn_up);
+ free(ws->attn_scores);
+ free(ws->offsets);
+ free(ws->rope_cos);
+ free(ws->rope_sin);
+ free(ws->bf16_widen);
+ free(ws);
+}
+
+int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) {
+ if (seq <= ws->buf_seq_cap)
+ return 0;
+
+ int cap;
+ if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0)
+ return -1;
+
+#define R(ptr, n) \
+ do { \
+ if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \
+ return -1; \
+ } while (0)
+
+ R(ws->x, c->hidden_size);
+ R(ws->x_norm, c->hidden_size);
+ R(ws->q, c->q_dim);
+ R(ws->k, c->kv_dim);
+ R(ws->v, c->kv_dim);
+ R(ws->attn_out, c->q_dim);
+ R(ws->proj_out, c->hidden_size);
+ R(ws->ffn_gate, c->intermediate_size);
+ R(ws->ffn_up, c->intermediate_size);
+
+#undef R
+
+ ws->buf_seq_cap = cap;
+ return 0;
+}
+
+void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch) {
+ size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (bytes <= ws->attn_scores_bytes)
+ return;
+
+ void *p = realloc(ws->attn_scores, bytes);
+ if (!p)
+ return;
+ ws->attn_scores = (float *)p;
+ ws->attn_scores_bytes = bytes;
+}
+
+int ensure_rope_cache(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int n_pos) {
+ if (n_pos <= ws->rope_cache_cap)
+ return 0;
+
+ int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512;
+ while (cap < n_pos) {
+ if (cap > INT_MAX / 2)
+ return -1;
+ cap *= 2;
+ }
+
+ int head_dim = cfg->head_dim;
+ int half = head_dim / 2;
+ float theta = cfg->rope_theta;
+
+ size_t count;
+ if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0)
+ return -1;
+ if (realloc_floats(&ws->rope_cos, count) != 0)
+ return -1;
+ if (realloc_floats(&ws->rope_sin, count) != 0)
+ return -1;
+
+ for (int pos = ws->rope_cache_cap; pos < cap; pos++) {
+ float fp = (float)pos;
+ float *c = ws->rope_cos + (size_t)pos * head_dim;
+ float *s = ws->rope_sin + (size_t)pos * head_dim;
+ for (int d = 0; d < half; d++) {
+ float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
+ float angle = fp * inv;
+ float cv = cosf(angle), sv = sinf(angle);
+ c[d] = cv;
+ c[half + d] = cv;
+ s[d] = sv;
+ s[half + d] = sv;
+ }
+ }
+
+ ws->rope_cache_cap = cap;
+ return 0;
+}
+
+int ensure_offsets(ffwd_workspace_t *ws, int batch) {
+ if (!ws || batch <= 0 || batch == INT_MAX)
+ return -1;
+
+ int needed = batch + 1;
+ if (needed <= ws->offsets_cap)
+ return 0;
+
+ if (realloc_ints(&ws->offsets, (size_t)needed) != 0)
+ return -1;
+ ws->offsets_cap = needed;
+ return 0;
+}
+
+size_t ffwd_workspace_nbytes(const ffwd_workspace_t *ws) {
+ if (!ws)
+ return 0;
+
+ size_t total = sizeof(*ws);
+ const ffwd_model_t *model = ws->model;
+ if (!model)
+ return total;
+
+ const ffwd_config_t *c = &model->config;
+ size_t row_floats = 0;
+ const int dims[] = {
+ c->hidden_size, /* x */
+ c->hidden_size, /* x_norm */
+ c->q_dim, /* q */
+ c->kv_dim, /* k */
+ c->kv_dim, /* v */
+ c->q_dim, /* attn_out */
+ c->hidden_size, /* proj_out */
+ c->intermediate_size, /* ffn_gate */
+ c->intermediate_size, /* ffn_up */
+ };
+
+ for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) {
+ if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0)
+ return SIZE_MAX;
+ }
+
+ size_t buf_floats = 0;
+ if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0)
+ return SIZE_MAX;
+
+ size_t rope_floats = 0;
+ if (ws->rope_cache_cap > 0) {
+ size_t rope_rows = 0;
+ if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 ||
+ mul_size(rope_rows, 2, &rope_floats) != 0)
+ return SIZE_MAX;
+ }
+
+ size_t all_floats = 0, bytes = 0;
+ if (add_size(buf_floats, rope_floats, &all_floats) != 0 ||
+ mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0)
+ return SIZE_MAX;
+
+ size_t offset_bytes = 0;
+ if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0)
+ return SIZE_MAX;
+ if (add_size(total, offset_bytes, &total) != 0)
+ return SIZE_MAX;
+ if (add_size(total, ws->attn_scores_bytes, &total) != 0)
+ return SIZE_MAX;
+
+ size_t widen_bytes = 0;
+ if (ws->bf16_widen_count > 0 && mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0)
+ return SIZE_MAX;
+ if (add_size(total, widen_bytes, &total) != 0)
+ return SIZE_MAX;
+
+ return total;
+}
diff --git a/src/model_internal.h b/libffwd/model_internal.h
index acbba07..97de064 100644
--- a/src/model_internal.h
+++ b/libffwd/model_internal.h
@@ -1,13 +1,13 @@
-#ifndef EMBED_MODEL_INTERNAL_H
-#define EMBED_MODEL_INTERNAL_H
+#ifndef FFWD_MODEL_INTERNAL_H
+#define FFWD_MODEL_INTERNAL_H
-#include "internal.h"
+#include "model_types.h"
#include "alloc.h"
/* Cross-file function declarations private to the CPU model code. The companion
- * to internal.h: that header holds the shared struct definitions, this one the
- * shared functions. None are part of the public API (embed.h) and none carry
- * EMBED_API visibility; they are shared between the translation units (model
+ * to model_types.h: that header holds the shared struct definitions, this one the
+ * shared functions. None are part of the public API (ffwd.h) and none carry
+ * FFWD_API visibility; they are shared between the translation units (model
* loading, the forward pass, and late interaction) that used to live in one
* file. The frontends and the GPU backends do not include this header. */
@@ -25,37 +25,37 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
* widened to an owned F32 buffer (no fused F16 kernel). Shared by the model
* loader and the late-interaction projection loader; the owner frees ref.data
* when ref.owned is set. Returns ref.data == NULL on allocation failure. */
-embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t);
+ffwd_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t);
/* True if model_dir carries a 1_Dense projection (a late-interaction model). */
int model_dir_has_late_projection(const char *model_dir);
/* Load (a layer range of) a model. allow_late lets the late loader accept a
* 1_Dense snapshot that the pooled loader rejects. layer_end == -1 means all. */
-embed_model_t *
+ffwd_model_t *
model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late);
/* Workspace scratch growth, lazy per forward call. Defined in model.c beside the
- * workspace lifecycle (embed_workspace_nbytes mirrors their sizing); called from
+ * workspace lifecycle (ffwd_workspace_nbytes mirrors their sizing); called from
* the forward pass in forward.c. */
-int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq);
-void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch);
-int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos);
-int ensure_offsets(embed_workspace_t *ws, int batch);
+int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq);
+void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch);
+int ensure_rope_cache(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int n_pos);
+int ensure_offsets(ffwd_workspace_t *ws, int batch);
/* y[seq,out] = x[seq,in] @ w^T, no bias; dispatches F32/BF16 paths. Shared by
* the forward pass and the late projection. */
-void linear_nobias_weight(embed_workspace_t *ws,
+void linear_nobias_weight(ffwd_workspace_t *ws,
float *y,
const float *x,
- const embed_weight_ref_t *w,
+ const ffwd_weight_ref_t *w,
int seq_len,
int in_dim,
int out_dim);
/* Compute packed sequence offsets [batch+1] plus total and max sequence length
* over a batch of inputs. require_ids rejects inputs with a NULL id pointer. */
-int build_offsets(const embed_input_t *inputs,
+int build_offsets(const ffwd_input_t *inputs,
int batch,
int require_ids,
int *offsets,
@@ -64,13 +64,13 @@ int build_offsets(const embed_input_t *inputs,
/* Run the full transformer forward over a packed batch in ws, leaving hidden
* states in ws->x. Used by pooled encoding and by late-interaction encoding. */
-int forward_packed_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
+int forward_packed_inplace(const ffwd_model_t *model,
+ ffwd_workspace_t *ws,
+ const ffwd_input_t *inputs,
int batch,
const int *offsets,
int total_seq,
int max_seq,
int apply_final_norm);
-#endif /* EMBED_MODEL_INTERNAL_H */
+#endif /* FFWD_MODEL_INTERNAL_H */
diff --git a/src/internal.h b/libffwd/model_types.h
index c1429de..2719a5b 100644
--- a/src/internal.h
+++ b/libffwd/model_types.h
@@ -1,7 +1,15 @@
-#ifndef EMBED_INTERNAL_H
-#define EMBED_INTERNAL_H
+/*
+ * model_types.h - in-memory layout of a loaded model: the model and workspace
+ * structs, the per-layer weight references, and the accessors that expose them
+ * to the GPU backends. Shared by the CPU model code and the CUDA backend; not
+ * part of the public ABI (ffwd.h). Companion to model_internal.h, which
+ * declares the CPU model's private functions.
+ */
-#include "embed.h"
+#ifndef FFWD_MODEL_TYPES_H
+#define FFWD_MODEL_TYPES_H
+
+#include "internal.h"
#include "safetensors.h"
typedef struct {
@@ -11,13 +19,13 @@ typedef struct {
* case). 1: data is a heap buffer owned by the model and freed on unload.
* Set when an F16 tensor is widened to F32 at load (no fused F16 kernel). */
int owned;
-} embed_weight_ref_t;
+} ffwd_weight_ref_t;
typedef struct {
- embed_weight_ref_t wq; /* [q_dim, hidden] */
- embed_weight_ref_t wk; /* [kv_dim, hidden] */
- embed_weight_ref_t wv; /* [kv_dim, hidden] */
- embed_weight_ref_t wo; /* [hidden, q_dim] */
+ ffwd_weight_ref_t wq; /* [q_dim, hidden] */
+ ffwd_weight_ref_t wk; /* [kv_dim, hidden] */
+ ffwd_weight_ref_t wv; /* [kv_dim, hidden] */
+ ffwd_weight_ref_t wo; /* [hidden, q_dim] */
const float *q_bias; /* [q_dim], optional */
const float *k_bias; /* [kv_dim], optional */
const float *v_bias; /* [kv_dim], optional */
@@ -25,9 +33,9 @@ typedef struct {
const float *k_norm; /* [head_dim] */
const float *input_norm; /* [hidden] */
const float *post_attn_norm; /* [hidden] */
- embed_weight_ref_t gate_proj; /* [intermediate, hidden] */
- embed_weight_ref_t up_proj; /* [intermediate, hidden] */
- embed_weight_ref_t down_proj; /* [hidden, intermediate] */
+ ffwd_weight_ref_t gate_proj; /* [intermediate, hidden] */
+ ffwd_weight_ref_t up_proj; /* [intermediate, hidden] */
+ ffwd_weight_ref_t down_proj; /* [hidden, intermediate] */
/* BERT family (all NULL for Qwen3). The attention projections reuse
* wq/wk/wv/wo and q/k/v_bias; the two FFN matrices reuse up_proj
* (intermediate.dense) and down_proj (output.dense); the two block
@@ -38,21 +46,21 @@ typedef struct {
const float *ffn_inter_bias; /* intermediate.dense bias [intermediate] */
const float *ffn_out_bias; /* output.dense bias [hidden] */
const float *ffn_ln_bias; /* post-FFN LayerNorm bias [hidden] */
-} embed_layer_t;
+} ffwd_layer_t;
typedef struct {
- embed_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
- embed_layer_t *layers; /* [n_layers] heap-allocated */
+ ffwd_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
+ ffwd_layer_t *layers; /* [n_layers] heap-allocated */
const float *norm; /* [hidden]; NULL for BERT (no final norm) */
/* BERT family (NULL for Qwen3): learned absolute position embeddings,
* token-type (segment) embeddings, and the embedding LayerNorm. */
const float *position_embeddings; /* [max_position_embeddings, hidden] */
const float *token_type_embeddings; /* [type_vocab_size, hidden] */
- const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */
- const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */
-} embed_weights_t;
+ const float *ffwd_ln_w; /* embedding LayerNorm weight [hidden] */
+ const float *ffwd_ln_b; /* embedding LayerNorm bias [hidden] */
+} ffwd_weights_t;
-typedef void (*embed_attention_fn)(float *out,
+typedef void (*ffwd_attention_fn)(float *out,
const float *Q,
const float *K,
const float *V,
@@ -65,17 +73,16 @@ typedef void (*embed_attention_fn)(float *out,
float *scratch,
size_t scratch_bytes);
-struct embed_model {
- embed_config_t config;
- embed_weights_t weights;
+struct ffwd_model {
+ ffwd_config_t config;
+ ffwd_weights_t weights;
void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
int layer_start; /* loaded transformer range, inclusive */
int layer_end; /* loaded transformer range, exclusive */
- embed_attention_fn attention;
};
-struct embed_workspace {
- const embed_model_t *model; /* immutable model this workspace belongs to */
+struct ffwd_workspace {
+ const ffwd_model_t *model; /* immutable model this workspace belongs to */
int buf_seq_cap;
float *x; /* [seq, hidden] */
float *x_norm; /* [seq, hidden] */
@@ -107,8 +114,8 @@ struct embed_workspace {
/* Internal accessors for the late-interaction model. GPU backends build a device
* context from an already-loaded CPU late model because the model dir is a late
* snapshot that the pooled loader rejects; these expose the base model and the
- * projection weight without making struct embed_late_model public. */
-embed_model_t *embed_late_model_base(const embed_late_model_t *model);
-const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model);
+ * projection weight without making struct ffwd_late_model public. */
+ffwd_model_t *ffwd_late_model_base(const ffwd_late_model_t *model);
+const ffwd_weight_ref_t *ffwd_late_model_projection(const ffwd_late_model_t *model);
-#endif /* EMBED_INTERNAL_H */
+#endif /* FFWD_MODEL_TYPES_H */
diff --git a/src/kernels_neon.c b/libffwd/neon.c
index 3e2a423..7f44d82 100644
--- a/src/kernels_neon.c
+++ b/libffwd/neon.c
@@ -1,15 +1,15 @@
/*
- * embed_kernels_neon.c - ARM NEON hot kernels
+ * neon.c - ARM NEON hot kernels
*/
-#include "kernels_impl.h"
+#include "impl.h"
#ifdef __ARM_NEON
-#include <arm_neon.h>
-#include <string.h>
+# include <arm_neon.h>
+# include <string.h>
-void embed_bf16_matvec_fused_neon(
+void bf16_matvec_fused_neon(
float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
int o = 0;
@@ -115,65 +115,4 @@ void embed_bf16_matvec_fused_neon(
}
}
-float embed_dot_f32_neon(const float *a, const float *b, int n) {
- int i = 0;
- float32x4_t acc0 = vdupq_n_f32(0.0f);
- float32x4_t acc1 = vdupq_n_f32(0.0f);
- for (; i + 8 <= n; i += 8) {
- float32x4_t a0 = vld1q_f32(a + i);
- float32x4_t b0 = vld1q_f32(b + i);
- float32x4_t a1 = vld1q_f32(a + i + 4);
- float32x4_t b1 = vld1q_f32(b + i + 4);
- acc0 = vfmaq_f32(acc0, a0, b0);
- acc1 = vfmaq_f32(acc1, a1, b1);
- }
- float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-}
-
-void embed_vec_scale_inplace_neon(float *dst, float scale, int n) {
- int i = 0;
- float32x4_t s = vdupq_n_f32(scale);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
- vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-}
-
-void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) {
- int i = 0;
- float32x4_t a = vdupq_n_f32(alpha);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
- vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-}
-
-void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n) {
- int i = 0;
- float32x4_t c = vdupq_n_f32(correction);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
- vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-}
-
#endif /* __ARM_NEON */
diff --git a/libffwd/norm.c b/libffwd/norm.c
new file mode 100644
index 0000000..27fd4c4
--- /dev/null
+++ b/libffwd/norm.c
@@ -0,0 +1,142 @@
+/*
+ * norm.c - Normalization
+ */
+
+#include "kernels.h"
+#include "threadpool.h"
+
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef USE_BLAS
+# ifdef __APPLE__
+# include <Accelerate/Accelerate.h>
+# else
+# include <cblas.h>
+# endif
+#endif
+
+#ifndef M_PI
+# define M_PI 3.14159265358979323846
+#endif
+
+/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the
+ * per-head RMSNorm used for QK-norm. */
+
+/* ========================================================================
+ * Normalization
+ * ======================================================================== */
+
+/* Rows split across the pool only when the tensor is big enough to pay
+ * for the dispatch. */
+#define RMS_NORM_PARALLEL_ELEMS 262144
+
+/* The sum of squares and the scaled write-back are plain loops: the compiler
+ * auto-vectorizes both in place, which is faster than reducing through a
+ * two-input dot kernel that reloads the row and is reduction-latency bound. */
+static void rms_norm_range(
+ float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) {
+ for (int s = start; s < end; s++) {
+ const float *x_row = x + (size_t)s * hidden;
+ float *out_row = out + (size_t)s * hidden;
+
+ float sum_sq = 0.0f;
+ for (int i = 0; i < hidden; i++)
+ sum_sq += x_row[i] * x_row[i];
+ float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps);
+ for (int i = 0; i < hidden; i++)
+ out_row[i] = x_row[i] * rms_inv * weight[i];
+ }
+}
+
+typedef struct {
+ float *out;
+ const float *x;
+ const float *weight;
+ int seq_len;
+ int hidden;
+ float eps;
+} rms_norm_task_t;
+
+static void rms_norm_worker(int tid, int n_threads, void *arg) {
+ rms_norm_task_t *t = (rms_norm_task_t *)arg;
+ int chunk = (t->seq_len + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > t->seq_len)
+ end = t->seq_len;
+ if (start >= end)
+ return;
+ rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps);
+}
+
+void rms_norm(float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) {
+ if (seq_len <= 0 || hidden <= 0)
+ return;
+
+ long long elems = (long long)seq_len * hidden;
+ if (tp_num_threads() > 1 && elems >= RMS_NORM_PARALLEL_ELEMS) {
+ rms_norm_task_t task = {
+ .out = out,
+ .x = x,
+ .weight = weight,
+ .seq_len = seq_len,
+ .hidden = hidden,
+ .eps = eps,
+ };
+ parallel_for(rms_norm_worker, &task);
+ } else {
+ rms_norm_range(out, x, weight, 0, seq_len, hidden, eps);
+ }
+}
+
+void rms_norm_per_head(
+ float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) {
+ /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */
+ int hidden = n_heads * head_dim;
+ for (int s = 0; s < seq_len; s++) {
+ for (int h = 0; h < n_heads; h++) {
+ float *vec = x + s * hidden + h * head_dim;
+
+ float sum_sq = 0.0f;
+ for (int d = 0; d < head_dim; d++)
+ sum_sq += vec[d] * vec[d];
+ float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps);
+ for (int d = 0; d < head_dim; d++)
+ vec[d] = vec[d] * rms_inv * weight[d];
+ }
+ }
+}
+
+/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each
+ * row's mean and variance are reduced before the row is rewritten. Scalar for
+ * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */
+void layer_norm(float *out,
+ const float *x,
+ const float *gamma,
+ const float *beta,
+ int seq_len,
+ int hidden,
+ float eps) {
+ if (seq_len <= 0 || hidden <= 0)
+ return;
+ float inv_h = 1.0f / (float)hidden;
+ for (int s = 0; s < seq_len; s++) {
+ const float *row = x + (size_t)s * hidden;
+ float *dst = out + (size_t)s * hidden;
+ float mean = 0.0f;
+ for (int d = 0; d < hidden; d++)
+ mean += row[d];
+ mean *= inv_h;
+ float var = 0.0f;
+ for (int d = 0; d < hidden; d++) {
+ float diff = row[d] - mean;
+ var += diff * diff;
+ }
+ var *= inv_h;
+ float inv_std = 1.0f / sqrtf(var + eps);
+ for (int d = 0; d < hidden; d++)
+ dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d];
+ }
+}
diff --git a/libffwd/platform.h b/libffwd/platform.h
new file mode 100644
index 0000000..3c92c98
--- /dev/null
+++ b/libffwd/platform.h
@@ -0,0 +1,27 @@
+/* platform.h - compile-time backend selection.
+ *
+ * The build passes -DUSE_GPU for a GPU build. This header maps that flag and the
+ * host platform to a concrete backend: Apple Silicon uses MLX, any other GPU
+ * uses CUDA, and a non-GPU build is CPU. Use USE_GPU for the CPU/GPU split;
+ * USE_MLX and USE_CUDA only where a specific accelerator API is called. */
+#ifndef FFWD_PLATFORM_H
+#define FFWD_PLATFORM_H
+
+#if defined(USE_GPU)
+# if defined(__APPLE__)
+# define USE_MLX 1
+# else
+# define USE_CUDA 1
+# endif
+#endif
+
+/* Hardware-class label for --build-info and the startup banner. */
+#if defined(USE_MLX)
+# define FFWD_BACKEND_LABEL "Apple Silicon GPU"
+#elif defined(USE_CUDA)
+# define FFWD_BACKEND_LABEL "NVIDIA GPU"
+#else
+# define FFWD_BACKEND_LABEL "CPU"
+#endif
+
+#endif /* FFWD_PLATFORM_H */
diff --git a/src/kernels_rope.c b/libffwd/rope.c
index ef1709a..12057b2 100644
--- a/src/kernels_rope.c
+++ b/libffwd/rope.c
@@ -1,5 +1,9 @@
+/*
+ * rope.c - rotary position embeddings
+ */
+
#include "kernels.h"
-#include "kernels_impl.h"
+#include "impl.h"
#include <math.h>
#include <stdlib.h>
@@ -7,22 +11,22 @@
#if (defined(__AVX512F__) || defined(__AVX2__)) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
+# include <immintrin.h>
#endif
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
+# include <arm_neon.h>
#endif
#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
+# ifdef __APPLE__
+# include <Accelerate/Accelerate.h>
+# else
+# include <cblas.h>
+# endif
#endif
#ifndef M_PI
-#define M_PI 3.14159265358979323846
+# define M_PI 3.14159265358979323846
#endif
/* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */
@@ -31,7 +35,7 @@
* Position Embeddings
* ======================================================================== */
-void embed_compute_rope_neox(
+void compute_rope_neox(
float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) {
int half = head_dim / 2;
@@ -51,7 +55,7 @@ void embed_compute_rope_neox(
}
}
-void embed_apply_rope_neox(
+void apply_rope_neox(
float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) {
/*
* NeoX split-half style:
diff --git a/src/safetensors.c b/libffwd/safetensors.c
index 65d0d7c..0efbd57 100644
--- a/src/safetensors.c
+++ b/libffwd/safetensors.c
@@ -1,13 +1,13 @@
#include "safetensors.h"
-#include "dtype.h"
+
+#include <dirent.h>
+#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <fcntl.h>
-#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
-#include <dirent.h>
+#include <unistd.h>
/* ========================================================================
* Minimal JSON parser for safetensors header
@@ -343,40 +343,6 @@ int64_t safetensor_numel(const safetensor_t *t) {
return n;
}
-float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) {
- int64_t n = safetensor_numel(t);
- if (n <= 0)
- return NULL;
-
- float *out = malloc(n * sizeof(float));
- if (!out)
- return NULL;
-
- const void *data = safetensors_data(sf, t);
-
- switch (t->dtype) {
- case DTYPE_F32:
- memcpy(out, data, n * sizeof(float));
- break;
- case DTYPE_BF16: {
- const uint16_t *src = (const uint16_t *)data;
- for (int64_t i = 0; i < n; i++)
- out[i] = embed_bf16_to_f32(src[i]);
- break;
- }
- case DTYPE_F16: {
- const uint16_t *src = (const uint16_t *)data;
- for (int64_t i = 0; i < n; i++)
- out[i] = embed_f16_to_f32(src[i]);
- break;
- }
- default:
- free(out);
- return NULL;
- }
- return out;
-}
-
/* ========================================================================
* Multi-shard operations
* ======================================================================== */
@@ -530,9 +496,8 @@ int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nby
return 0;
}
-const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
- const char *name,
- safetensors_file_t **out_sf) {
+const safetensor_t *
+multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) {
for (int s = 0; s < ms->num_shards; s++) {
safetensors_file_t *sf = ms->shards[s];
for (int i = 0; i < sf->num_tensors; i++) {
@@ -546,8 +511,7 @@ const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
return NULL;
}
-const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms,
- const char *tensor_name) {
+const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms, const char *tensor_name) {
if (multi_safetensors_find(ms, tensor_name, NULL))
return "";
diff --git a/src/safetensors.h b/libffwd/safetensors.h
index 8c9b33f..13d4cba 100644
--- a/src/safetensors.h
+++ b/libffwd/safetensors.h
@@ -1,10 +1,9 @@
/*
- * embed_safetensors.h - Safetensors file format reader (multi-shard support)
- * Adapted from voxtral-realtime project.
+ * safetensors.h - Safetensors file format reader (multi-shard support)
*/
-#ifndef EMBED_SAFETENSORS_H
-#define EMBED_SAFETENSORS_H
+#ifndef SAFETENSORS_H
+#define SAFETENSORS_H
#include <stddef.h>
#include <stdint.h>
@@ -41,7 +40,8 @@ typedef struct {
safetensor_t tensors[SAFETENSORS_MAX_TENSORS];
} safetensors_file_t;
-/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */
+/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup
+ */
typedef struct {
safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS];
int num_shards;
@@ -57,9 +57,8 @@ void multi_safetensors_close(multi_safetensors_t *ms);
int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes);
/* Find a tensor by name across all shards */
-const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
- const char *name,
- safetensors_file_t **out_sf);
+const safetensor_t *
+multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf);
/* Resolve an optional Hugging Face "model." namespace for a backbone tensor.
* Returns "", "model.", or NULL when neither form exists. */
@@ -68,9 +67,6 @@ const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms, const
/* Get raw pointer to tensor data (within mmap'd region) */
const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
-/* Get tensor data as float32 (allocates, caller must free) */
-float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t);
-
int64_t safetensor_numel(const safetensor_t *t);
-#endif /* EMBED_SAFETENSORS_H */
+#endif /* SAFETENSORS_H */
diff --git a/src/tokenizer_sentencepiece.c b/libffwd/sentencepiece.c
index 657e8c9..2372972 100644
--- a/src/tokenizer_sentencepiece.c
+++ b/libffwd/sentencepiece.c
@@ -1,5 +1,5 @@
/*
- * tokenizer_sentencepiece.c - SentencePiece Unigram tokenizer.
+ * sentencepiece.c - SentencePiece Unigram tokenizer.
*
* The implementation reads the small subset of ModelProto needed at runtime:
* pieces, scores, piece types, trainer model_type, and normalizer options. The
@@ -8,7 +8,7 @@
* over UTF-8 byte offsets.
*/
-#include "tokenizer_sentencepiece.h"
+#include "sentencepiece.h"
#include <errno.h>
#include <float.h>
@@ -39,7 +39,7 @@ typedef struct {
double score;
} sp_best_t;
-struct sentencepiece_workspace {
+struct tok_spm_workspace {
char *norm;
size_t norm_len;
size_t norm_cap;
@@ -127,7 +127,7 @@ static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, siz
return 0;
}
-static int reserve_best(sentencepiece_workspace_t *ws, int need) {
+static int reserve_best(tok_spm_workspace_t *ws, int need) {
if (need <= ws->best_cap)
return 0;
int nc = ws->best_cap ? ws->best_cap : 128;
@@ -144,7 +144,7 @@ static int reserve_best(sentencepiece_workspace_t *ws, int need) {
return 0;
}
-static int reserve_ids(sentencepiece_workspace_t *ws, int need) {
+static int reserve_ids(tok_spm_workspace_t *ws, int need) {
if (need <= ws->ids_cap)
return 0;
int nc = ws->ids_cap ? ws->ids_cap : 64;
@@ -268,8 +268,7 @@ static char *memdup0(const unsigned char *data, size_t len) {
return s;
}
-static int
-parse_piece(sentencepiece_tokenizer_t *tok, int idx, const unsigned char *data, size_t len) {
+static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_t len) {
const unsigned char *p = data, *end = data + len;
char *piece = NULL;
float score = 0.0f;
@@ -316,7 +315,7 @@ fail:
return -1;
}
-static int parse_trainer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
+static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) {
const unsigned char *p = data, *end = data + len;
int model_type = SP_MODEL_UNIGRAM;
while (p < end) {
@@ -341,7 +340,7 @@ static int parse_trainer(sentencepiece_tokenizer_t *tok, const unsigned char *da
return 0;
}
-static int parse_normalizer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
+static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t len) {
const unsigned char *p = data, *end = data + len;
while (p < end) {
int field = 0, wire = 0;
@@ -407,7 +406,7 @@ static int count_pieces(const unsigned char *data, size_t len) {
return n;
}
-static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
+static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
int n_pieces = count_pieces(data, len);
if (n_pieces <= 0)
return -1;
@@ -441,8 +440,7 @@ static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data
} else if (wire == 2 && field == 3) {
const unsigned char *msg = NULL;
size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 ||
- parse_normalizer(tok, msg, msg_len) != 0)
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_normalizer(tok, msg, msg_len) != 0)
return -1;
} else if (pb_skip(&p, end, wire) != 0) {
return -1;
@@ -453,17 +451,16 @@ static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data
return 0;
}
-static int raw_piece_id(const sentencepiece_tokenizer_t *tok, const char *piece) {
+static int raw_piece_id(const tok_spm_t *tok, const char *piece) {
if (!tok || !piece)
return -1;
- return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece,
- (int)strlen(piece));
+ return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece));
}
-static int raw_to_model_id(const sentencepiece_tokenizer_t *tok, int raw_id) {
+static int raw_to_model_id(const tok_spm_t *tok, int raw_id) {
if (!tok || raw_id < 0 || raw_id >= tok->piece_count)
return -1;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
if (raw_id == 1)
return 0; /* <s> */
if (raw_id == 2)
@@ -475,10 +472,10 @@ static int raw_to_model_id(const sentencepiece_tokenizer_t *tok, int raw_id) {
return raw_id;
}
-static int model_to_raw_id(const sentencepiece_tokenizer_t *tok, int id) {
+static int model_to_raw_id(const tok_spm_t *tok, int id) {
if (!tok || id < 0)
return -1;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
if (id == 0)
return 1;
if (id == 2)
@@ -579,20 +576,18 @@ static int load_file(const char *path, unsigned char **out, size_t *out_len) {
return 0;
}
-static uint32_t xcda_base(const sentencepiece_tokenizer_t *tok, size_t idx) {
+static uint32_t xcda_base(const tok_spm_t *tok, size_t idx) {
uint32_t v = tok->xcda[idx];
return (v >> 10) << ((v & (1u << 9)) >> 6);
}
-static uint32_t xcda_lcheck(const sentencepiece_tokenizer_t *tok, size_t idx) {
+static uint32_t xcda_lcheck(const tok_spm_t *tok, size_t idx) {
return tok->xcda[idx] & ((1u << 31) | 0xffu);
}
-static int xcda_leaf(const sentencepiece_tokenizer_t *tok, size_t idx) {
- return (tok->xcda[idx] >> 8) & 1u;
-}
+static int xcda_leaf(const tok_spm_t *tok, size_t idx) { return (tok->xcda[idx] >> 8) & 1u; }
-static uint32_t xcda_value(const sentencepiece_tokenizer_t *tok, size_t idx) {
+static uint32_t xcda_value(const tok_spm_t *tok, size_t idx) {
return tok->xcda[idx] & ((1u << 31) - 1u);
}
@@ -602,10 +597,8 @@ typedef struct {
size_t consumed;
} norm_prefix_t;
-static norm_prefix_t normalize_prefix(const sentencepiece_tokenizer_t *tok,
- const char *input,
- size_t input_len,
- size_t off) {
+static norm_prefix_t
+normalize_prefix(const tok_spm_t *tok, const char *input, size_t input_len, size_t off) {
static const char repl[] = "\xef\xbf\xbd";
if (off >= input_len)
return (norm_prefix_t){input + off, 0, 0};
@@ -642,9 +635,7 @@ static norm_prefix_t normalize_prefix(const sentencepiece_tokenizer_t *tok,
return (norm_prefix_t){repl, 3, 1};
}
-static int normalize_into(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text) {
+static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text) {
ws->norm_len = 0;
if (!text)
return -1;
@@ -664,8 +655,8 @@ static int normalize_into(const sentencepiece_tokenizer_t *tok,
processing_non_ws = 1;
if ((tok->add_dummy_prefix && !space_prepended) ||
tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s,
- space_n) != 0)
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
+ 0)
return -1;
space_prepended = 1;
}
@@ -675,8 +666,7 @@ static int normalize_into(const sentencepiece_tokenizer_t *tok,
} else {
processing_non_ws = 0;
if (!tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
- 0)
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0)
return -1;
}
}
@@ -688,7 +678,7 @@ static int normalize_into(const sentencepiece_tokenizer_t *tok,
return 0;
}
-static double compute_unknown_score(const sentencepiece_tokenizer_t *tok) {
+static double compute_unknown_score(const tok_spm_t *tok) {
float min_score = FLT_MAX;
for (int i = 0; i < tok->piece_count; i++) {
if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score)
@@ -699,7 +689,7 @@ static double compute_unknown_score(const sentencepiece_tokenizer_t *tok) {
return (double)min_score - 10.0;
}
-static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace_t *ws) {
+static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
size_t n = ws->norm_len;
if (n == 0)
return 0;
@@ -722,8 +712,8 @@ static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace
int single_cp_found = 0;
for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) {
- int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap,
- ws->norm + off, (int)(end - off));
+ int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off,
+ (int)(end - off));
if (raw >= 0 && is_tokenizable_type(tok->types[raw])) {
if (end - off == cp_len)
single_cp_found = 1;
@@ -770,7 +760,7 @@ static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace
return out_n;
}
-sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) {
+tok_spm_t *tok_spm_load(const char *model_dir) {
if (!model_dir)
return NULL;
char path[1024];
@@ -785,8 +775,7 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) {
return NULL;
}
- sentencepiece_tokenizer_t *tok =
- (sentencepiece_tokenizer_t *)calloc(1, sizeof(sentencepiece_tokenizer_t));
+ tok_spm_t *tok = (tok_spm_t *)calloc(1, sizeof(tok_spm_t));
if (!tok) {
free(data);
return NULL;
@@ -794,12 +783,11 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) {
tok->add_dummy_prefix = 1;
tok->remove_extra_whitespaces = 1;
tok->escape_whitespaces = 1;
- tok->id_map =
- detect_xlm_roberta(model_dir) ? SENTENCEPIECE_XLM_ROBERTA : SENTENCEPIECE_IDENTITY;
+ tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY;
if (parse_model(tok, data, len) != 0) {
fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path);
free(data);
- sentencepiece_tokenizer_free(tok);
+ tok_spm_free(tok);
return NULL;
}
free(data);
@@ -807,14 +795,14 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) {
int cap = next_pow2(tok->piece_count * 2 + 1);
tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t));
if (!tok->piece_map) {
- sentencepiece_tokenizer_free(tok);
+ tok_spm_free(tok);
return NULL;
}
tok->piece_map_cap = cap;
for (int i = 0; i < tok->piece_count; i++) {
if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i],
tok->piece_lens[i], i) != 0) {
- sentencepiece_tokenizer_free(tok);
+ tok_spm_free(tok);
return NULL;
}
}
@@ -822,27 +810,25 @@ sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) {
tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>"));
tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>"));
tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>"));
- tok->pad_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA
- ? 1
- : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
- tok->mask_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA
+ tok->pad_id =
+ tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
+ tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA
? tok->piece_count + 1
: raw_to_model_id(tok, raw_piece_id(tok, "<mask>"));
- tok->vocab_size =
- tok->id_map == SENTENCEPIECE_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
+ tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
tok->unknown_score = compute_unknown_score(tok);
if (tok->unk_id < 0 || tok->max_piece_len <= 0) {
fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path);
- sentencepiece_tokenizer_free(tok);
+ tok_spm_free(tok);
return NULL;
}
return tok;
}
-const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id) {
+const char *tok_spm_decode(const tok_spm_t *tok, int id) {
if (!tok || id < 0)
return NULL;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
if (id == 1)
return "<pad>";
if (id == tok->piece_count + 1)
@@ -852,10 +838,10 @@ const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok,
return raw >= 0 ? tok->id_to_piece[raw] : NULL;
}
-int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, const char *token) {
+int tok_spm_token_id(const tok_spm_t *tok, const char *token) {
if (!tok || !token)
return -1;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
if (!strcmp(token, "<s>"))
return 0;
if (!strcmp(token, "<pad>"))
@@ -870,12 +856,12 @@ int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, const
return raw_to_model_id(tok, raw_piece_id(tok, token));
}
-int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
+int tok_spm_encode_into(const tok_spm_t *tok,
+ tok_spm_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens) {
if (out_n_tokens)
*out_n_tokens = 0;
if (!tok || !ws || !text)
@@ -895,16 +881,16 @@ int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
return 0;
}
-int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens) {
+int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
+ tok_spm_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens) {
if (out_n_tokens)
*out_n_tokens = 0;
if (!tok || !ws || !text)
return NULL;
int n = 0;
- int rc = sentencepiece_tokenizer_encode_into(tok, ws, text, NULL, 0, &n);
+ int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n);
if (rc < 0 && rc != -2)
return NULL;
if (out_n_tokens)
@@ -918,25 +904,23 @@ int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer
return ids;
}
-int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
- const char *text,
- int *out_n_tokens) {
- sentencepiece_workspace_t *ws = sentencepiece_workspace_new();
+int *tok_spm_encode(const tok_spm_t *tok, const char *text, int *out_n_tokens) {
+ tok_spm_workspace_t *ws = tok_spm_workspace_new();
if (!ws) {
if (out_n_tokens)
*out_n_tokens = 0;
return NULL;
}
- int *ids = sentencepiece_tokenizer_encode_with_workspace(tok, ws, text, out_n_tokens);
- sentencepiece_workspace_free(ws);
+ int *ids = tok_spm_encode_with_workspace(tok, ws, text, out_n_tokens);
+ tok_spm_workspace_free(ws);
return ids;
}
-sentencepiece_workspace_t *sentencepiece_workspace_new(void) {
- return (sentencepiece_workspace_t *)calloc(1, sizeof(sentencepiece_workspace_t));
+tok_spm_workspace_t *tok_spm_workspace_new(void) {
+ return (tok_spm_workspace_t *)calloc(1, sizeof(tok_spm_workspace_t));
}
-void sentencepiece_workspace_free(sentencepiece_workspace_t *ws) {
+void tok_spm_workspace_free(tok_spm_workspace_t *ws) {
if (!ws)
return;
free(ws->norm);
@@ -945,7 +929,7 @@ void sentencepiece_workspace_free(sentencepiece_workspace_t *ws) {
free(ws);
}
-void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok) {
+void tok_spm_free(tok_spm_t *tok) {
if (!tok)
return;
if (tok->id_to_piece) {
diff --git a/libffwd/sentencepiece.h b/libffwd/sentencepiece.h
new file mode 100644
index 0000000..7f5314a
--- /dev/null
+++ b/libffwd/sentencepiece.h
@@ -0,0 +1,88 @@
+/*
+ * sentencepiece.h - SentencePiece tokenizer for XLM-R style models.
+ *
+ * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`,
+ * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R
+ * and multilingual E5. It encodes text only; frontends add the model's special
+ * token layout, e.g. <s> ... </s> for XLM-R.
+ */
+
+#ifndef TOK_SPM_H
+#define TOK_SPM_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef enum {
+ TOK_SPM_IDENTITY = 0,
+ TOK_SPM_XLM_ROBERTA = 1,
+} tok_spm_id_map_t;
+
+typedef struct {
+ char **id_to_piece; /* [piece_count] raw SentencePiece ids */
+ uint16_t *piece_lens; /* byte length for id_to_piece entries */
+ float *scores; /* SentencePiece score per raw id */
+ uint8_t *types; /* SentencePiece piece type per raw id */
+ int piece_count; /* raw SentencePiece vocabulary size */
+ int vocab_size; /* model embedding vocab size after id mapping */
+ int unk_id; /* model-space unknown-token id */
+ int bos_id; /* model-space beginning-token id, or -1 */
+ int eos_id; /* model-space end-token id, or -1 */
+ int pad_id; /* model-space pad-token id, or -1 */
+ int mask_id; /* model-space mask-token id, or -1 */
+ int max_piece_len; /* max normal/user/unused piece byte length */
+ double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
+ int add_dummy_prefix;
+ int remove_extra_whitespaces;
+ int escape_whitespaces;
+ tok_spm_id_map_t id_map;
+
+ void *piece_map; /* piece bytes -> raw SentencePiece id */
+ int piece_map_cap;
+
+ uint32_t *xcda;
+ size_t xcda_count;
+ char *prefix_replacements;
+ size_t prefix_replacements_len;
+} tok_spm_t;
+
+typedef struct tok_spm_workspace tok_spm_workspace_t;
+
+/* Load the first supported SentencePiece model found in model_dir. Returns NULL
+ * on missing model file, unsupported model type, malformed protobuf, or OOM. */
+tok_spm_t *tok_spm_load(const char *model_dir);
+
+/* Decode one model-space id to an internal token string, or NULL when invalid. */
+const char *tok_spm_decode(const tok_spm_t *tok, int id);
+
+/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
+ * model-space id, or -1 when absent. */
+int tok_spm_token_id(const tok_spm_t *tok, const char *token);
+
+/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns
+ * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an
+ * error. */
+int *tok_spm_encode(const tok_spm_t *tok, const char *text, int *out_n_tokens);
+
+tok_spm_workspace_t *tok_spm_workspace_new(void);
+void tok_spm_workspace_free(tok_spm_workspace_t *ws);
+
+/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if
+ * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */
+int tok_spm_encode_into(const tok_spm_t *tok,
+ tok_spm_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
+
+/* Same tokenization as tok_spm_encode(), reusing ws internally
+ * and performing only the final returned int[] allocation. */
+int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
+ tok_spm_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
+void tok_spm_free(tok_spm_t *tok);
+
+#endif /* TOK_SPM_H */
diff --git a/src/threadpool.c b/libffwd/threadpool.c
index 40d22c7..2521a6d 100644
--- a/src/threadpool.c
+++ b/libffwd/threadpool.c
@@ -1,5 +1,9 @@
+/*
+ * tp.c - CPU thread pool
+ */
+
#if defined(__linux__) && !defined(_GNU_SOURCE)
-#define _GNU_SOURCE
+# define _GNU_SOURCE
#endif
#include "threadpool.h"
@@ -11,28 +15,23 @@
#include <string.h>
#ifdef __APPLE__
-#include <sys/sysctl.h>
+# include <sys/sysctl.h>
#else
-#ifdef __linux__
-#include <sched.h>
-#endif
-#include <unistd.h>
+# ifdef __linux__
+# include <sched.h>
+# endif
+# include <unistd.h>
#endif
#if defined(USE_OPENBLAS)
void openblas_set_num_threads(int num_threads);
#endif
-/* The global verbose flag (defined in model.c); used only for optional pool
- * diagnostics below. Declared here to keep the thread pool independent of the
- * kernel/model headers. */
-extern int embed_verbose;
-
-#define EMBED_MAX_THREADS 16
+#define TP_MAX_THREADS 16
static struct {
- pthread_t threads[EMBED_MAX_THREADS - 1];
- int tids[EMBED_MAX_THREADS - 1];
+ pthread_t threads[TP_MAX_THREADS - 1];
+ int tids[TP_MAX_THREADS - 1];
int n_threads;
int shutdown;
@@ -80,11 +79,11 @@ static void *worker_loop(void *arg) {
}
}
-void embed_set_threads(int n) {
+void tp_set_threads(int n) {
if (n < 1)
n = 1;
- if (n > EMBED_MAX_THREADS)
- n = EMBED_MAX_THREADS;
+ if (n > TP_MAX_THREADS)
+ n = TP_MAX_THREADS;
#if defined(USE_OPENBLAS)
/* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the
@@ -107,29 +106,13 @@ void embed_set_threads(int n) {
}
tp.n_threads = n;
- if (n <= 1) {
- if (embed_verbose >= 2) {
-#if defined(USE_OPENBLAS)
- fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads);
-#else
- fprintf(stderr, "Thread pool: %d threads\n", n);
-#endif
- }
+ if (n <= 1)
return;
- }
for (int i = 0; i < n - 1; i++) {
tp.tids[i] = i + 1;
pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]);
}
-
- if (embed_verbose >= 2) {
-#if defined(USE_OPENBLAS)
- fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads);
-#else
- fprintf(stderr, "Thread pool: %d threads\n", n);
-#endif
- }
}
#ifndef __APPLE__
@@ -177,7 +160,7 @@ static int cgroup_cpu_quota(void) {
}
#endif
-int embed_get_num_cpus(void) {
+int tp_get_num_cpus(void) {
#ifdef __APPLE__
int n = 0;
size_t len = sizeof(n);
@@ -187,14 +170,14 @@ int embed_get_num_cpus(void) {
int online = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (online < 1)
online = 1;
-#ifdef __linux__
+# ifdef __linux__
cpu_set_t set;
if (sched_getaffinity(0, sizeof(set), &set) == 0) {
int affinity = CPU_COUNT(&set);
if (affinity >= 1 && affinity < online)
online = affinity;
}
-#endif
+# endif
/* Honor a cgroup CPU quota when it is tighter than the visible core count,
* so the default thread pool does not oversubscribe a CPU-limited
* container and get throttled. */
@@ -205,7 +188,7 @@ int embed_get_num_cpus(void) {
#endif
}
-int embed_num_threads(void) { return tp.n_threads; }
+int tp_num_threads(void) { return tp.n_threads; }
/* Dispatch work to all threads; main thread is tid=0 */
void parallel_for(parallel_fn_t fn, void *arg) {
diff --git a/src/threadpool.h b/libffwd/threadpool.h
index 1813bb0..1dcc80a 100644
--- a/src/threadpool.h
+++ b/libffwd/threadpool.h
@@ -1,36 +1,28 @@
-#ifndef EMBED_THREADPOOL_H
-#define EMBED_THREADPOOL_H
+#ifndef TP_THREADPOOL_H
+#define TP_THREADPOOL_H
/* Persistent worker pool for the CPU math kernels: a fixed set of threads that
* sleep until parallel_for() hands them a callback, plus the CPU-count helper
* that sizes the pool (cgroup-quota aware). Separated from the math so the
- * kernels depend only on the parallel_for / embed_num_threads interface, not on
+ * kernels depend only on the parallel_for / tp_num_threads interface, not on
* the pool's internals. kernels.h includes this, so kernel callers need no
* extra include. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
/* Worker callback: (thread index, total threads, user arg). */
typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg);
/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call
* before inference. */
-EMBED_API void embed_set_threads(int n);
+void tp_set_threads(int n);
/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */
-EMBED_API int embed_get_num_cpus(void);
+int tp_get_num_cpus(void);
/* Current pool size (1 == serial). Kernels gate parallelism on this. */
-int embed_num_threads(void);
+int tp_num_threads(void);
/* Run fn across the pool; the calling thread participates as tid 0. Blocks
* until every worker has finished. */
void parallel_for(parallel_fn_t fn, void *arg);
-#endif /* EMBED_THREADPOOL_H */
+#endif /* TP_THREADPOOL_H */
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c
new file mode 100644
index 0000000..f7a50f7
--- /dev/null
+++ b/libffwd/tokenize.c
@@ -0,0 +1,350 @@
+/*
+ * tokenize.c - model-aware tokenization: text -> the token ids a model expects.
+ *
+ * Backend-independent, so this is its own handle (ffwd_tok_t), not part of the
+ * per-backend ffwd_t. ffwd_tok_open picks the tokenizer by the files present
+ * and resolves the model's special-token ids; ffwd_tokenize applies that
+ * model's layout (BERT/XLM-R [CLS]..[SEP], Qwen appended terminal token).
+ */
+
+#include "ffwd.h"
+
+#include "bpe.h"
+#include "sentencepiece.h"
+#include "wordpiece.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+typedef enum { TOK_KIND_BPE, TOK_KIND_WP, TOK_KIND_SPM } tok_kind_t;
+
+struct ffwd_tok {
+ tok_kind_t kind;
+ /* exactly one trio is populated, by kind */
+ tok_bpe_t *bpe;
+ tok_bpe_workspace_t *bpe_ws;
+ tok_wp_t *wp;
+ tok_wp_workspace_t *wp_ws;
+ tok_spm_t *spm;
+ tok_spm_workspace_t *spm_ws;
+
+ int cls_id; /* [CLS] / <s> — wp/spm wrap with these */
+ int sep_id; /* [SEP] / </s> */
+ int context_separator_id;
+ int terminal_id; /* appended on the bpe path when append_terminal */
+ int append_terminal; /* Qwen3-Embedding pools the last token; see note */
+
+ int is_late;
+ int late_mask_id;
+ int late_query_prefix_id;
+ int late_document_prefix_id;
+ int late_skip_ids[64];
+ int n_late_skip_ids;
+};
+
+static void set_err(char *err, size_t errlen, const char *msg) {
+ if (err && errlen)
+ snprintf(err, errlen, "%s", msg);
+}
+
+ffwd_tok_t *ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen) {
+ if (!model_dir) {
+ set_err(err, errlen, "model_dir is NULL");
+ return NULL;
+ }
+ ffwd_tok_t *t = calloc(1, sizeof(*t));
+ if (!t) {
+ set_err(err, errlen, "out of memory");
+ return NULL;
+ }
+ t->cls_id = -1;
+ t->sep_id = -1;
+ t->terminal_id = -1;
+ t->is_late = is_late;
+
+ char path[1024];
+
+ /* WordPiece (BERT family): vocab.txt. */
+ snprintf(path, sizeof(path), "%s/vocab.txt", model_dir);
+ if (access(path, R_OK) == 0) {
+ t->kind = TOK_KIND_WP;
+ t->wp = tok_wp_load(model_dir);
+ if (!t->wp) {
+ set_err(err, errlen, "failed to load WordPiece tokenizer");
+ goto fail;
+ }
+ t->wp_ws = tok_wp_workspace_new();
+ if (!t->wp_ws) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ t->cls_id = tok_wp_token_id(t->wp, "[CLS]");
+ t->sep_id = tok_wp_token_id(t->wp, "[SEP]");
+ if (t->cls_id < 0 || t->sep_id < 0) {
+ set_err(err, errlen, "WordPiece vocab missing [CLS]/[SEP]");
+ goto fail;
+ }
+ return t;
+ }
+
+ /* SentencePiece (XLM-R family): sentencepiece.bpe.model / tokenizer.model /
+ * spiece.model. */
+ snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", model_dir);
+ if (access(path, R_OK) != 0) {
+ snprintf(path, sizeof(path), "%s/tokenizer.model", model_dir);
+ if (access(path, R_OK) != 0)
+ snprintf(path, sizeof(path), "%s/spiece.model", model_dir);
+ }
+ if (access(path, R_OK) == 0) {
+ t->kind = TOK_KIND_SPM;
+ t->spm = tok_spm_load(model_dir);
+ if (!t->spm) {
+ set_err(err, errlen, "failed to load SentencePiece tokenizer");
+ goto fail;
+ }
+ t->spm_ws = tok_spm_workspace_new();
+ if (!t->spm_ws) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ t->cls_id = tok_spm_token_id(t->spm, "<s>");
+ t->sep_id = tok_spm_token_id(t->spm, "</s>");
+ if (t->cls_id < 0 || t->sep_id < 0) {
+ set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
+ goto fail;
+ }
+ return t;
+ }
+
+ /* Byte-level BPE (Qwen family): vocab.json. */
+ snprintf(path, sizeof(path), "%s/vocab.json", model_dir);
+ t->kind = TOK_KIND_BPE;
+ t->bpe = tok_bpe_load(path);
+ if (!t->bpe) {
+ set_err(err, errlen, "failed to load tokenizer");
+ goto fail;
+ }
+ t->bpe_ws = tok_bpe_workspace_new();
+ if (!t->bpe_ws) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ /* The contextual chunk separator is the tokenizer's <|endoftext|>; released
+ * models keep it out of vocab.json, so the family constant is the fallback.
+ * Qwen3-Embedding pools the last token (the same <|endoftext|>), so the bpe
+ * path appends it. A config that disables append_terminal_token would need
+ * to override this; the released models all enable it. */
+ int sep = tok_bpe_token_id(t->bpe, "<|endoftext|>");
+ t->context_separator_id = sep >= 0 ? sep : FFWD_CONTEXT_SEPARATOR_TOKEN_ID;
+ t->terminal_id = t->context_separator_id;
+ /* Off until the caller sets it from config->append_terminal_token; it is
+ * model-specific (pplx-embed does not append, for example). */
+ t->append_terminal = 0;
+
+ if (is_late) {
+ int id = tok_bpe_token_id(t->bpe, "[MASK]");
+ t->late_mask_id = id >= 0 ? id : FFWD_LATE_MASK_TOKEN_ID;
+ id = tok_bpe_token_id(t->bpe, "[Q]");
+ t->late_query_prefix_id = id >= 0 ? id : FFWD_LATE_QUERY_PREFIX_ID;
+ id = tok_bpe_token_id(t->bpe, "[D]");
+ t->late_document_prefix_id = id >= 0 ? id : FFWD_LATE_DOCUMENT_PREFIX_ID;
+
+ const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
+ for (const char *p = punct;
+ *p && t->n_late_skip_ids < (int)(sizeof(t->late_skip_ids) / sizeof(t->late_skip_ids[0]));
+ p++) {
+ char text[2] = {*p, '\0'};
+ int n_ids = 0;
+ int *ids = tok_bpe_encode(t->bpe, text, &n_ids);
+ if (ids && n_ids == 1)
+ t->late_skip_ids[t->n_late_skip_ids++] = ids[0];
+ free(ids);
+ }
+ }
+ return t;
+
+fail:
+ ffwd_tok_free(t);
+ return NULL;
+}
+
+void ffwd_tok_free(ffwd_tok_t *t) {
+ if (!t)
+ return;
+ if (t->bpe_ws)
+ tok_bpe_workspace_free(t->bpe_ws);
+ if (t->bpe)
+ tok_bpe_free(t->bpe);
+ if (t->wp_ws)
+ tok_wp_workspace_free(t->wp_ws);
+ if (t->wp)
+ tok_wp_free(t->wp);
+ if (t->spm_ws)
+ tok_spm_workspace_free(t->spm_ws);
+ if (t->spm)
+ tok_spm_free(t->spm);
+ free(t);
+}
+
+int ffwd_tok_context_separator_id(const ffwd_tok_t *t) {
+ return t ? t->context_separator_id : -1;
+}
+
+void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on) {
+ if (t)
+ t->append_terminal = on ? 1 : 0;
+}
+
+/* Encode text and apply the model's special-token layout. Returns malloc'd ids
+ * and sets *n_out; NULL on error. */
+static int *tokenize_core(ffwd_tok_t *t, const char *text, int *n_out) {
+ *n_out = 0;
+
+ if (t->kind == TOK_KIND_WP || t->kind == TOK_KIND_SPM) {
+ int n = 0;
+ int *core = (t->kind == TOK_KIND_WP)
+ ? tok_wp_encode_with_workspace(t->wp, t->wp_ws, text, &n)
+ : tok_spm_encode_with_workspace(t->spm, t->spm_ws, text, &n);
+ /* Wrap [CLS] core [SEP]. A whitespace-only input cleans to zero core
+ * tokens (core == NULL, n == 0); the bare [CLS][SEP] pair is what
+ * Hugging Face produces for it too. Grow in place so only the one
+ * allocation is touched. */
+ if (n < 0 || n > INT_MAX - 2) {
+ free(core);
+ return NULL;
+ }
+ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
+ return NULL;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = t->cls_id;
+ ids[n + 1] = t->sep_id;
+ *n_out = n + 2;
+ return ids;
+ }
+
+ /* BPE. */
+ int n = 0;
+ int *ids = tok_bpe_encode_with_workspace(t->bpe, t->bpe_ws, text, &n);
+ if (!ids || n <= 0) {
+ free(ids);
+ return NULL;
+ }
+ if (t->append_terminal) {
+ if (n == INT_MAX) {
+ free(ids);
+ return NULL;
+ }
+ int *grown = realloc(ids, (size_t)(n + 1) * sizeof(*ids));
+ if (!grown) {
+ free(ids);
+ return NULL;
+ }
+ ids = grown;
+ ids[n++] = t->terminal_id;
+ }
+ *n_out = n;
+ return ids;
+}
+
+int *ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out) {
+ if (n_out)
+ *n_out = 0;
+ if (!t || !text)
+ return NULL;
+
+ int n = 0;
+ int *ids;
+ if (!query_instruct) {
+ ids = tokenize_core(t, text, &n);
+ } else {
+ /* Tokenize the instruction and the input in one pass. */
+ size_t plen = strlen(query_instruct);
+ size_t tlen = strlen(text);
+ char *buf = malloc(plen + tlen + 1);
+ if (!buf)
+ return NULL;
+ memcpy(buf, query_instruct, plen);
+ memcpy(buf + plen, text, tlen + 1);
+ ids = tokenize_core(t, buf, &n);
+ free(buf);
+ }
+ if (!ids)
+ return NULL;
+ if (n_out)
+ *n_out = n;
+ return ids;
+}
+
+static int late_id_is_skipped(const ffwd_tok_t *t, int id) {
+ for (int i = 0; i < t->n_late_skip_ids; i++) {
+ if (t->late_skip_ids[i] == id)
+ return 1;
+ }
+ return 0;
+}
+
+int ffwd_tokenize_late(ffwd_tok_t *t,
+ const char *text,
+ int is_query,
+ ffwd_late_tokens_t *out) {
+ memset(out, 0, sizeof(*out));
+ if (!t || !text)
+ return -1;
+
+ int raw_n = 0;
+ int *raw = tokenize_core(t, text, &raw_n);
+ if (!raw)
+ return -1;
+
+ int target = raw_n + 1;
+ if (is_query && target < FFWD_LATE_QUERY_TOKENS)
+ target = FFWD_LATE_QUERY_TOKENS;
+
+ out->ids = malloc((size_t)target * sizeof(*out->ids));
+ if (!out->ids) {
+ free(raw);
+ return -1;
+ }
+ out->ids[0] = raw[0];
+ out->ids[1] = is_query ? t->late_query_prefix_id : t->late_document_prefix_id;
+ if (raw_n > 1)
+ memcpy(out->ids + 2, raw + 1, (size_t)(raw_n - 1) * sizeof(*out->ids));
+ out->n_tokens = raw_n + 1;
+ free(raw);
+
+ if (is_query) {
+ while (out->n_tokens < FFWD_LATE_QUERY_TOKENS)
+ out->ids[out->n_tokens++] = t->late_mask_id;
+ out->n_keep = out->n_tokens;
+ } else {
+ out->keep = malloc((size_t)out->n_tokens * sizeof(*out->keep));
+ if (!out->keep) {
+ ffwd_late_tokens_free(out);
+ return -1;
+ }
+ for (int i = 0; i < out->n_tokens; i++) {
+ if (!late_id_is_skipped(t, out->ids[i]))
+ out->keep[out->n_keep++] = i;
+ }
+ }
+
+ if (out->n_keep <= 0) {
+ ffwd_late_tokens_free(out);
+ return -1;
+ }
+ return 0;
+}
+
+void ffwd_late_tokens_free(ffwd_late_tokens_t *out) {
+ if (!out)
+ return;
+ free(out->ids);
+ free(out->keep);
+ memset(out, 0, sizeof(*out));
+}
diff --git a/src/tokenizer_unicode.h b/libffwd/unicode.h
index 8ca2458..bce8967 100644
--- a/src/tokenizer_unicode.h
+++ b/libffwd/unicode.h
@@ -31,54 +31,61 @@ typedef struct {
} wp_fold_t;
static const wp_range_t WP_PUNCT_RANGES[] = {
- {0x21, 0x2F}, {0x3A, 0x40}, {0x5B, 0x60}, {0x7B, 0x7E}, {0xA1, 0xA1}, {0xA7, 0xA7},
- {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387},
- {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3},
- {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D}, {0x61B, 0x61B},
- {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9},
- {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD},
- {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4},
- {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D},
- {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB},
- {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED},
- {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, {0x1944, 0x1945},
- {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF},
- {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027},
- {0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E},
- {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF},
- {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF},
- {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003},
- {0x3008, 0x3011}, {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0},
- {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E},
- {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC},
- {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F},
- {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19},
- {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B},
- {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20},
- {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65},
- {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F},
- {0x10857, 0x10857}, {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58},
- {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C},
- {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC},
- {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8},
- {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D},
- {0x112A9, 0x112A9}, {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D},
- {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C},
- {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2},
- {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45},
- {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474},
- {0x16A6E, 0x16A6F}, {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44},
- {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B},
- {0x1E95E, 0x1E95F},
+ {0x21, 0x2F}, {0x3A, 0x40}, {0x5B, 0x60}, {0x7B, 0x7E},
+ {0xA1, 0xA1}, {0xA7, 0xA7}, {0xAB, 0xAB}, {0xB6, 0xB7},
+ {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387},
+ {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0},
+ {0x5C3, 0x5C3}, {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A},
+ {0x60C, 0x60D}, {0x61B, 0x61B}, {0x61E, 0x61F}, {0x66A, 0x66D},
+ {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9}, {0x830, 0x83E},
+ {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD},
+ {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84},
+ {0xDF4, 0xDF4}, {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12},
+ {0xF14, 0xF14}, {0xF3A, 0xF3D}, {0xF85, 0xF85}, {0xFD0, 0xFD4},
+ {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB}, {0x1360, 0x1368},
+ {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED},
+ {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A},
+ {0x1944, 0x1945}, {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD},
+ {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF}, {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F},
+ {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027}, {0x2030, 0x2043},
+ {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E},
+ {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6},
+ {0x27E6, 0x27EF}, {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD},
+ {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF}, {0x2D70, 0x2D70}, {0x2E00, 0x2E2E},
+ {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003}, {0x3008, 0x3011},
+ {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0},
+ {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673},
+ {0xA67E, 0xA67E}, {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF},
+ {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC}, {0xA92E, 0xA92F}, {0xA95F, 0xA95F},
+ {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F}, {0xAADE, 0xAADF},
+ {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19},
+ {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68},
+ {0xFE6A, 0xFE6B}, {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F},
+ {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20}, {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F},
+ {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65}, {0x10100, 0x10102},
+ {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F}, {0x10857, 0x10857},
+ {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58}, {0x10A7F, 0x10A7F},
+ {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C}, {0x10EAD, 0x10EAD},
+ {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC}, {0x110BE, 0x110C1},
+ {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8}, {0x111CD, 0x111CD},
+ {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D}, {0x112A9, 0x112A9},
+ {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D}, {0x114C6, 0x114C6},
+ {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C}, {0x1173C, 0x1173E},
+ {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2}, {0x11A3F, 0x11A46},
+ {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45}, {0x11C70, 0x11C71},
+ {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474}, {0x16A6E, 0x16A6F},
+ {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44}, {0x16E97, 0x16E9A},
+ {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B}, {0x1E95E, 0x1E95F},
};
enum { WP_PUNCT_RANGES_N = (int)(sizeof(WP_PUNCT_RANGES) / sizeof(WP_PUNCT_RANGES[0])) };
static const wp_range_t WP_CONTROL_RANGES[] = {
- {0x0, 0x8}, {0xB, 0xC}, {0xE, 0x1F}, {0x7F, 0x9F}, {0xAD, 0xAD}, {0x600, 0x605},
- {0x61C, 0x61C}, {0x6DD, 0x6DD}, {0x70F, 0x70F}, {0x8E2, 0x8E2}, {0x180E, 0x180E},
- {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x2064}, {0x2066, 0x206F}, {0xE000, 0xF8FF},
- {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0x110BD, 0x110BD}, {0x110CD, 0x110CD},
- {0x13430, 0x13438}, {0x1BCA0, 0x1BCA3}, {0x1D173, 0x1D17A}, {0xE0001, 0xE0001},
+ {0x0, 0x8}, {0xB, 0xC}, {0xE, 0x1F}, {0x7F, 0x9F},
+ {0xAD, 0xAD}, {0x600, 0x605}, {0x61C, 0x61C}, {0x6DD, 0x6DD},
+ {0x70F, 0x70F}, {0x8E2, 0x8E2}, {0x180E, 0x180E}, {0x200B, 0x200F},
+ {0x202A, 0x202E}, {0x2060, 0x2064}, {0x2066, 0x206F}, {0xE000, 0xF8FF},
+ {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0x110BD, 0x110BD}, {0x110CD, 0x110CD},
+ {0x13430, 0x13438}, {0x1BCA0, 0x1BCA3}, {0x1D173, 0x1D17A}, {0xE0001, 0xE0001},
{0xE0020, 0xE007F}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD},
};
enum { WP_CONTROL_RANGES_N = (int)(sizeof(WP_CONTROL_RANGES) / sizeof(WP_CONTROL_RANGES[0])) };
diff --git a/src/vec.c b/libffwd/vec.c
index cea1576..f848be0 100644
--- a/src/vec.c
+++ b/libffwd/vec.c
@@ -1,4 +1,4 @@
-#include "embed.h"
+#include "ffwd.h"
#include <math.h>
@@ -10,7 +10,7 @@
* similarity. They touch no model or tensor internals, only the public API.
* ======================================================================== */
-int embed_l2_normalize(float *vec, int dim) {
+int ffwd_l2_normalize(float *vec, int dim) {
if (!vec || dim <= 0)
return -1;
@@ -27,7 +27,7 @@ int embed_l2_normalize(float *vec, int dim) {
return 0;
}
-float embed_cosine_similarity(const float *a, const float *b, int dim) {
+float ffwd_cosine_similarity(const float *a, const float *b, int dim) {
if (!a || !b || dim <= 0)
return 0.0f;
@@ -42,8 +42,8 @@ float embed_cosine_similarity(const float *a, const float *b, int dim) {
norm_b_sq += bv * bv;
}
- if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) ||
- !isfinite(norm_b_sq) || !isfinite(dot))
+ if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) || !isfinite(norm_b_sq) ||
+ !isfinite(dot))
return 0.0f;
double denom = sqrt((double)norm_a_sq) * sqrt((double)norm_b_sq);
diff --git a/src/tokenizer_wordpiece.c b/libffwd/wordpiece.c
index d8acbbe..d3f62dd 100644
--- a/src/tokenizer_wordpiece.c
+++ b/libffwd/wordpiece.c
@@ -1,5 +1,5 @@
/*
- * wordpiece_tokenizer.c - BERT-family WordPiece tokenizer.
+ * tok_wp_tokenizer.c - BERT-family WordPiece tokenizer.
*
* Pipeline (matching transformers' BertTokenizer):
* BasicTokenizer: clean control/whitespace -> space CJK ideographs ->
@@ -8,7 +8,7 @@
* pieces marked "##", whole token -> [UNK] when any piece fails to match.
*
* Unicode handling is full and exact against transformers, driven by generated
- * tables (wordpiece_unicode.h): punctuation and control by Unicode category,
+ * tables (tok_wp_unicode.h): punctuation and control by Unicode category,
* whitespace by the complete Zs set, the CJK split by the BERT ideograph ranges,
* and the do_lower_case fold by a lowercase+NFD+strip-marks table (with Hangul
* decomposed by formula). Cased and uncased multilingual encoders (e.g.
@@ -18,9 +18,9 @@
* Greek final sigma).
*/
-#include "tokenizer_wordpiece.h"
+#include "wordpiece.h"
-#include "tokenizer_unicode.h"
+#include "unicode.h"
#include <stdint.h>
#include <stdio.h>
@@ -237,7 +237,7 @@ static void wp_fold(uint32_t cp, uint32_t out[3], int *n_out) {
/* =============================== workspace ================================== */
-struct wordpiece_workspace {
+struct tok_wp_workspace {
uint32_t *cps; /* cleaned + CJK-spaced codepoints of the whole input */
int cps_cap;
uint32_t *norm; /* one word's normalized codepoints */
@@ -294,11 +294,11 @@ static int grow_char(char **buf, int *cap, int need) {
return 0;
}
-wordpiece_workspace_t *wordpiece_workspace_new(void) {
- return (wordpiece_workspace_t *)calloc(1, sizeof(wordpiece_workspace_t));
+tok_wp_workspace_t *tok_wp_workspace_new(void) {
+ return (tok_wp_workspace_t *)calloc(1, sizeof(tok_wp_workspace_t));
}
-void wordpiece_workspace_free(wordpiece_workspace_t *ws) {
+void tok_wp_workspace_free(tok_wp_workspace_t *ws) {
if (!ws)
return;
free(ws->cps);
@@ -315,11 +315,8 @@ void wordpiece_workspace_free(wordpiece_workspace_t *ws) {
/* Append the WordPiece ids of one BasicTokenizer sub-word (codepoints
* word[0..n)) to ws->ids[*n_out]. The whole sub-word maps to [UNK] when it is
* too long or any greedy piece misses the vocab. Returns 0 / -1 on OOM. */
-static int wordpiece_subword(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const uint32_t *word,
- int n,
- int *n_out) {
+static int
+tok_wp_subword(const tok_wp_t *tok, tok_wp_workspace_t *ws, const uint32_t *word, int n, int *n_out) {
if (n == 0)
return 0;
if (n > tok->max_chars_per_word) {
@@ -386,8 +383,7 @@ static int wordpiece_subword(const wordpiece_tokenizer_t *tok,
/* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. Returns the
* token count, or -1 on allocation failure. */
-static int
-wordpiece_run(const wordpiece_tokenizer_t *tok, wordpiece_workspace_t *ws, const char *text) {
+static int tok_wp_run(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text) {
/* clean_text + tokenize_chinese_chars, in one pass over the input. */
const unsigned char *p = (const unsigned char *)text;
const unsigned char *end = p + strlen(text);
@@ -449,16 +445,16 @@ wordpiece_run(const wordpiece_tokenizer_t *tok, wordpiece_workspace_t *ws, const
int run = 0;
for (int k = 0; k < m; k++) {
if (wp_is_punct(ws->norm[k])) {
- if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (k - run), run, &n_out) != 0)
+ if (run > 0 && tok_wp_subword(tok, ws, ws->norm + (k - run), run, &n_out) != 0)
return -1;
run = 0;
- if (wordpiece_subword(tok, ws, ws->norm + k, 1, &n_out) != 0)
+ if (tok_wp_subword(tok, ws, ws->norm + k, 1, &n_out) != 0)
return -1;
} else {
run++;
}
}
- if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (m - run), run, &n_out) != 0)
+ if (run > 0 && tok_wp_subword(tok, ws, ws->norm + (m - run), run, &n_out) != 0)
return -1;
}
return n_out;
@@ -496,7 +492,7 @@ static char *read_file(const char *path, long *out_len) {
return buf;
}
-/* do_lower_case from tokenizer_config.json: default true (BERT uncased). A
+/* do_lower_case from config.json: default true (BERT uncased). A
* lightweight scan is enough - we only need one boolean and avoid a JSON dep
* here, mirroring how the config layer reads small flags. */
static int read_do_lower_case(const char *model_dir) {
@@ -521,7 +517,7 @@ static int read_do_lower_case(const char *model_dir) {
return result;
}
-wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) {
+tok_wp_t *tok_wp_load(const char *model_dir) {
if (!model_dir)
return NULL;
char path[2048];
@@ -541,7 +537,7 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) {
if (len > 0 && buf[len - 1] != '\n')
count++;
- wordpiece_tokenizer_t *tok = (wordpiece_tokenizer_t *)calloc(1, sizeof(*tok));
+ tok_wp_t *tok = (tok_wp_t *)calloc(1, sizeof(*tok));
char **id_to_token = (char **)calloc((size_t)(count > 0 ? count : 1), sizeof(char *));
if (!tok || !id_to_token) {
free(buf);
@@ -582,7 +578,7 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) {
int cap = next_pow2(n * 2 < 16 ? 16 : n * 2);
wp_entry_t *map = (wp_entry_t *)calloc((size_t)cap, sizeof(wp_entry_t));
if (!map) {
- wordpiece_tokenizer_free(tok);
+ tok_wp_free(tok);
return NULL;
}
for (int i = 0; i < n; i++)
@@ -593,7 +589,7 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) {
tok->unk_id = map_get(map, cap, "[UNK]");
if (tok->unk_id < 0) {
fprintf(stderr, "wordpiece: vocab.txt has no [UNK] token\n");
- wordpiece_tokenizer_free(tok);
+ tok_wp_free(tok);
return NULL;
}
return tok;
@@ -601,29 +597,29 @@ wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) {
/* ============================== public API ================================= */
-const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id) {
+const char *tok_wp_decode(const tok_wp_t *tok, int id) {
if (!tok || id < 0 || id >= tok->vocab_size)
return NULL;
return tok->id_to_token[id];
}
-int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token) {
+int tok_wp_token_id(const tok_wp_t *tok, const char *token) {
if (!tok || !token)
return -1;
return map_get((const wp_entry_t *)tok->vocab_map, tok->vocab_map_cap, token);
}
-int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
+int tok_wp_encode_into(const tok_wp_t *tok,
+ tok_wp_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens) {
if (out_n_tokens)
*out_n_tokens = 0;
if (!tok || !ws || !text)
return -1;
- int n = wordpiece_run(tok, ws, text);
+ int n = tok_wp_run(tok, ws, text);
if (n < 0)
return -1;
if (out_n_tokens)
@@ -634,15 +630,15 @@ int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
return 0;
}
-int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens) {
+int *tok_wp_encode_with_workspace(const tok_wp_t *tok,
+ tok_wp_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens) {
if (out_n_tokens)
*out_n_tokens = 0;
if (!tok || !ws || !text)
return NULL;
- int n = wordpiece_run(tok, ws, text);
+ int n = tok_wp_run(tok, ws, text);
if (n < 0)
return NULL;
int *ids = (int *)malloc((size_t)(n > 0 ? n : 1) * sizeof(int));
@@ -654,18 +650,18 @@ int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
return ids;
}
-int *wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n) {
+int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n) {
if (out_n)
*out_n = 0;
- wordpiece_workspace_t *ws = wordpiece_workspace_new();
+ tok_wp_workspace_t *ws = tok_wp_workspace_new();
if (!ws)
return NULL;
- int *ids = wordpiece_tokenizer_encode_with_workspace(tok, ws, text, out_n);
- wordpiece_workspace_free(ws);
+ int *ids = tok_wp_encode_with_workspace(tok, ws, text, out_n);
+ tok_wp_workspace_free(ws);
return ids;
}
-void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok) {
+void tok_wp_free(tok_wp_t *tok) {
if (!tok)
return;
if (tok->id_to_token) {
diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h
new file mode 100644
index 0000000..66c62a0
--- /dev/null
+++ b/libffwd/wordpiece.h
@@ -0,0 +1,70 @@
+/*
+ * wordpiece.h - BERT-family WordPiece tokenizer.
+ *
+ * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional
+ * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting)
+ * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes
+ * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the
+ * byte-level BPE backend leaves terminal/separator tokens to the frontend.
+ *
+ * The public surface mirrors bpe.h (load / encode with a reusable
+ * workspace / decode / resolve a named special token / free) so both can sit
+ * behind one tokenizer interface later.
+ */
+
+#ifndef TOK_WP_H
+#define TOK_WP_H
+
+typedef struct {
+ char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */
+ int vocab_size;
+ int unk_id; /* [UNK] id, resolved at load */
+ int do_lower_case; /* lowercase + strip accents before WordPiece */
+ int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */
+
+ /* Opaque token-string -> id hash map. */
+ void *vocab_map;
+ int vocab_map_cap;
+} tok_wp_t;
+
+typedef struct tok_wp_workspace tok_wp_workspace_t;
+
+/* Load from <model_dir>/vocab.txt, reading do_lower_case from
+ * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on
+ * error (missing vocab.txt, missing [UNK], allocation failure). */
+tok_wp_t *tok_wp_load(const char *model_dir);
+
+/* Decode a single id to its vocab token string (internal pointer), or NULL. */
+const char *tok_wp_decode(const tok_wp_t *tok, int id);
+
+/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */
+int tok_wp_token_id(const tok_wp_t *tok, const char *token);
+
+/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on
+ * error (and *out_n_tokens = 0). */
+int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n_tokens);
+
+/* Reusable scratch for allocation-light repeated encoding. */
+tok_wp_workspace_t *tok_wp_workspace_new(void);
+void tok_wp_workspace_free(tok_wp_workspace_t *ws);
+
+/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1
+ * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to
+ * the number of tokens needed/emitted in all cases. */
+int tok_wp_encode_into(const tok_wp_t *tok,
+ tok_wp_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
+
+/* Same tokenization as tok_wp_encode(), reusing ws internally and
+ * performing only the final returned int[] allocation. */
+int *tok_wp_encode_with_workspace(const tok_wp_t *tok,
+ tok_wp_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
+void tok_wp_free(tok_wp_t *tok);
+
+#endif /* TOK_WP_H */
diff --git a/src/build.h b/src/build.h
deleted file mode 100644
index dc68405..0000000
--- a/src/build.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/* embed_build.h - build identity for the embed and embed-server front-ends.
- *
- * The real values come from the Makefile's VERSION_CFLAGS at compile time; the
- * fallbacks below apply when those flags are absent. Front-end only: the
- * inference library does not use these. */
-#ifndef EMBED_BUILD_H
-#define EMBED_BUILD_H
-
-#include <stdio.h>
-#include <string.h>
-
-#ifndef EMBED_VERSION
-#define EMBED_VERSION "development"
-#endif
-#ifndef EMBED_BUILD_COMMIT
-#define EMBED_BUILD_COMMIT "unknown"
-#endif
-#ifndef EMBED_BUILD_DATE
-#define EMBED_BUILD_DATE "unknown"
-#endif
-#ifndef EMBED_BUILD_OS
-#define EMBED_BUILD_OS "unknown"
-#endif
-#ifndef EMBED_BUILD_ARCH
-#define EMBED_BUILD_ARCH "unknown"
-#endif
-
-/* Backend label for --build-info, selected by the same macro that picks the
- * compiled inference path. */
-#if defined(USE_MLX)
-#define EMBED_BUILD_BACKENDS "accelerate,mlx"
-#elif defined(USE_CUDA)
-#define EMBED_BUILD_BACKENDS "openblas,cuda"
-#elif defined(__APPLE__)
-#define EMBED_BUILD_BACKENDS "accelerate"
-#else
-#define EMBED_BUILD_BACKENDS "openblas"
-#endif
-
-/* Program name with any leading directory removed. */
-static inline const char *embed_prog_name(const char *argv0)
-{
- const char *slash = strrchr(argv0, '/');
- return slash ? slash + 1 : argv0;
-}
-
-static inline void embed_print_version(const char *prog)
-{
- printf("%s %s (%s/%s)\n", prog, EMBED_VERSION,
- EMBED_BUILD_OS, EMBED_BUILD_ARCH);
-}
-
-static inline void embed_print_build_info(const char *prog)
-{
- embed_print_version(prog);
- printf("backend: %s\n"
- "compiler: %s\n"
- "built: %s\n"
- "commit: %s\n",
- EMBED_BUILD_BACKENDS, __VERSION__,
- EMBED_BUILD_DATE, EMBED_BUILD_COMMIT);
-}
-
-#endif /* EMBED_BUILD_H */
diff --git a/src/cli.c b/src/cli.c
deleted file mode 100644
index 0e3ed7d..0000000
--- a/src/cli.c
+++ /dev/null
@@ -1,911 +0,0 @@
-/* embed_cli.c - embed command-line tool */
-
-#include "embed.h"
-#include "build.h"
-#include "kernels.h"
-#include "tokenizer_bpe.h"
-#include "tokenizer_sentencepiece.h"
-#include "tokenizer_wordpiece.h"
-
-#ifdef USE_MLX
-#include "mlx.h"
-#endif
-#ifdef USE_CUDA
-#include "cuda.h"
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <limits.h>
-#include <unistd.h>
-
-static double now_ms(void) {
- struct timespec ts;
- clock_gettime(CLOCK_MONOTONIC, &ts);
- return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
-}
-
-static void print_usage(const char *prog) {
- fprintf(stderr,
- "Usage: %s -d <model_dir> [options] [text...]\n"
- "\n"
- "Options:\n"
- " -d <dir> Model directory (required)\n"
-#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
-#endif
-#ifdef USE_CUDA
- " --cuda Use CUDA/cuBLAS GPU backend\n"
-#endif
- " --stream Read lines from stdin, write one JSON embedding per line\n"
- " --json Emit JSON instead of plain text (embedding or matrix)\n"
-#ifdef USE_MLX
- " --mlx-quant-bits N\n"
- " Quantize MLX linear weights to 8 bits at load time\n"
- " --mlx-quant-group-size N\n"
- " MLX quantization group size (default: 64)\n"
-#endif
-#ifdef USE_CUDA
- " --cuda-gemm-mode MODE\n"
- " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
- " --cuda-weight-dtype DTYPE\n"
- " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
- " weight memory and uses BF16 tensor cores\n"
-#endif
- " -b, --batch-size N\n"
- " Max texts per engine batch (default: all; --stream: 1)\n"
- " -t, --threads N\n"
- " CPU threads (default: available cores, cgroup-aware)\n"
- " -e, --embeddings\n"
- " Print raw embeddings (with multiple texts)\n"
- " -v, --verbose\n"
- " Verbose (-vv for debug)\n"
- " -V, --version\n"
- " Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h Show this help\n"
- "\n"
- "Modes:\n"
- " 1 text arg Embedding as space-separated floats (JSON with --json)\n"
- " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n"
- " no args Batch: read all stdin lines, then similarity matrix\n"
- " --stream Streaming: read stdin lines, write JSON per line\n"
- "\n"
- "Examples:\n"
- " %s -d ./model \"what is AI?\"\n"
-#ifdef USE_MLX
- " %s -d ./model --mlx --stream < texts.txt\n",
- prog, prog, prog);
-#else
- ,
- prog, prog);
-#endif
-}
-
-/* ========================================================================
- * Embed one text: tokenize > forward > return float[dim]
- * ======================================================================== */
-
-typedef struct {
- embed_model_t *model;
- embed_workspace_t *workspace;
-#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
-#endif
-#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
-#endif
- embed_tokenizer_t *tok;
- embed_tokenizer_workspace_t *tok_ws;
- /* BERT-family WordPiece tokenizer, selected by a vocab.txt probe at load.
- * When wp_tok is set, tokenize_text wraps the ids with [CLS]/[SEP] and the
- * Qwen byte-level BPE fields above stay NULL. */
- wordpiece_tokenizer_t *wp_tok;
- wordpiece_workspace_t *wp_tok_ws;
- /* SentencePiece/XLM-R tokenizer. When sp_tok is set, tokenize_text wraps
- * core ids with <s> ... </s>. */
- sentencepiece_tokenizer_t *sp_tok;
- sentencepiece_workspace_t *sp_tok_ws;
- int cls_id;
- int sep_id;
- int dim;
- int terminal_token_id;
- int append_terminal_token;
-} engine_t;
-
-static void engine_free_tokenizers(engine_t *e) {
- embed_tokenizer_workspace_free(e->tok_ws);
- embed_tokenizer_free(e->tok);
- wordpiece_workspace_free(e->wp_tok_ws);
- wordpiece_tokenizer_free(e->wp_tok);
- sentencepiece_workspace_free(e->sp_tok_ws);
- sentencepiece_tokenizer_free(e->sp_tok);
-}
-
-typedef struct {
- int *ids;
- int n_tokens;
-} token_buf_t;
-
-static int
-engine_embed_batch(engine_t *e, const embed_input_t *inputs, int batch, float *out_embeddings) {
-#ifdef USE_MLX
- if (e->mlx_ctx)
- return embed_mlx_encode_batch(e->mlx_ctx, inputs, batch, out_embeddings);
-#endif
-#ifdef USE_CUDA
- if (e->cuda_ctx)
- return embed_cuda_encode_batch(e->cuda_ctx, inputs, batch, out_embeddings);
-#endif
- return embed_model_encode_batch(e->model, e->workspace, inputs, batch, out_embeddings);
-}
-
-static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) {
- memset(out, 0, sizeof(*out));
- if (e->wp_tok) {
- /* BERT: WordPiece encode, then wrap [CLS] ... [SEP] (the same contract
- * as the BPE terminal-token append below). An empty clean yields the
- * bare pair, matching Hugging Face. Grow the buffer in place. */
- int n = 0;
- int *core = wordpiece_tokenizer_encode_with_workspace(e->wp_tok, e->wp_tok_ws, text, &n);
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- return -1;
- }
- int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = e->cls_id;
- ids[n + 1] = e->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- } else if (e->sp_tok) {
- int n = 0;
- int *core =
- sentencepiece_tokenizer_encode_with_workspace(e->sp_tok, e->sp_tok_ws, text, &n);
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- return -1;
- }
- int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = e->cls_id;
- ids[n + 1] = e->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- } else {
- out->ids = embed_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens);
- if (!out->ids || out->n_tokens == 0) {
- fprintf(stderr, "tokenization failed: %s\n", text);
- free(out->ids);
- out->ids = NULL;
- out->n_tokens = 0;
- return -1;
- }
- if (e->append_terminal_token) {
- if (out->n_tokens == INT_MAX) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = (int *)realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
- if (!ids) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- out->ids = ids;
- out->ids[out->n_tokens++] = e->terminal_token_id;
- }
- }
-
- if (embed_verbose >= 1) {
- fprintf(stderr, "tokens (%d): ", out->n_tokens);
- for (int i = 0; i < out->n_tokens && i < 20; i++)
- fprintf(stderr, "%d ", out->ids[i]);
- if (out->n_tokens > 20)
- fprintf(stderr, "...");
- fprintf(stderr, "\n");
- }
- return 0;
-}
-
-static void free_tokens(token_buf_t *tokens, int n) {
- if (!tokens)
- return;
- for (int i = 0; i < n; i++)
- free(tokens[i].ids);
-}
-
-/* ========================================================================
- * Output helpers
- * ======================================================================== */
-
-static void print_embedding_raw(const float *emb, int dim) {
- for (int i = 0; i < dim; i++) {
- if (i > 0)
- putchar(' ');
- printf("%.8f", (double)emb[i]);
- }
- putchar('\n');
-}
-
-/* Single embedding as a JSON array of floats. */
-static void print_embedding_array(const float *emb, int dim) {
- putchar('[');
- for (int i = 0; i < dim; i++) {
- if (i)
- putchar(',');
- printf("%.8f", (double)emb[i]);
- }
- puts("]");
-}
-
-static size_t engine_workspace_nbytes(const engine_t *e) {
- return (e && e->workspace) ? embed_workspace_nbytes(e->workspace) : 0;
-}
-
-static void
-print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) {
- printf("{\"embedding\":[");
- for (int i = 0; i < dim; i++) {
- if (i > 0)
- putchar(',');
- printf("%.8f", (double)emb[i]);
- }
- printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms,
- workspace_bytes);
- fflush(stdout);
-}
-
-/* ========================================================================
- * Stdin mode: read lines from stdin, write JSON to stdout
- * ======================================================================== */
-
-static int process_stdin_batch(engine_t *e, char **lines, int n_lines) {
- if (n_lines <= 0)
- return 0;
-
- token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_lines * sizeof(embed_input_t));
- int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int));
- float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
- if (!tokens || !inputs || !input_to_line || !embs) {
- fprintf(stderr, "OOM\n");
- free(tokens);
- free(inputs);
- free(input_to_line);
- free(embs);
- for (int i = 0; i < n_lines; i++)
- printf("{\"error\":\"embedding failed\"}\n");
- fflush(stdout);
- return 1;
- }
-
- int n_inputs = 0;
- for (int i = 0; i < n_lines; i++) {
- if (tokenize_text(e, lines[i], &tokens[i]) == 0) {
- inputs[n_inputs].ids = tokens[i].ids;
- inputs[n_inputs].n_tokens = tokens[i].n_tokens;
- input_to_line[n_inputs] = i;
- n_inputs++;
- }
- }
-
- double ms = 0.0;
- int rc = 0;
- if (n_inputs > 0) {
- double t0 = now_ms();
- rc = engine_embed_batch(e, inputs, n_inputs, embs);
- ms = now_ms() - t0;
- if (embed_verbose >= 1)
- fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms);
- }
-
- int next_valid = 0;
- for (int i = 0; i < n_lines; i++) {
- if (!tokens[i].ids) {
- printf("{\"error\":\"tokenization failed\"}\n");
- continue;
- }
- if (rc != 0) {
- printf("{\"error\":\"embedding failed\"}\n");
- continue;
- }
- while (next_valid < n_inputs && input_to_line[next_valid] != i)
- next_valid++;
- if (next_valid >= n_inputs) {
- printf("{\"error\":\"embedding failed\"}\n");
- continue;
- }
- print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms,
- engine_workspace_nbytes(e));
- next_valid++;
- }
- fflush(stdout);
-
- free_tokens(tokens, n_lines);
- free(tokens);
- free(inputs);
- free(input_to_line);
- free(embs);
- return rc == 0 ? 0 : 1;
-}
-
-static int run_stdin(engine_t *e, int batch_size) {
- char line[65536];
- if (batch_size <= 0)
- batch_size = 1;
-
- if (embed_verbose >= 1)
- fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size);
-
- char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
- if (!batch_lines) {
- fprintf(stderr, "OOM\n");
- return 1;
- }
- int n_batch = 0;
- int rc = 0;
-
- while (fgets(line, sizeof(line), stdin)) {
- /* Strip newline */
- size_t len = strlen(line);
- while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
- line[--len] = '\0';
- if (len == 0)
- continue;
-
- if (embed_verbose >= 1)
- fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line,
- len > 60 ? "..." : "");
-
- batch_lines[n_batch] = strdup(line);
- if (!batch_lines[n_batch]) {
- fprintf(stderr, "OOM\n");
- rc = 1;
- break;
- }
- n_batch++;
-
- if (n_batch == batch_size) {
- if (process_stdin_batch(e, batch_lines, n_batch) != 0)
- rc = 1;
- for (int i = 0; i < n_batch; i++) {
- free(batch_lines[i]);
- batch_lines[i] = NULL;
- }
- n_batch = 0;
- }
- }
-
- if (n_batch > 0) {
- if (process_stdin_batch(e, batch_lines, n_batch) != 0)
- rc = 1;
- for (int i = 0; i < n_batch; i++)
- free(batch_lines[i]);
- }
- free(batch_lines);
-
- if (embed_verbose >= 1)
- fprintf(stderr, "stdin: EOF\n");
-
- return rc;
-}
-
-/* ========================================================================
- * Batch mode: embed args or stdin lines, then print similarity or vectors
- * ======================================================================== */
-
-static int append_text(char ***texts, int *n_texts, int *cap, const char *s) {
- if (*n_texts == *cap) {
- int new_cap = *cap ? *cap * 2 : 8;
- char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
- if (!new_texts)
- return -1;
- *texts = new_texts;
- *cap = new_cap;
- }
-
- (*texts)[*n_texts] = strdup(s);
- if (!(*texts)[*n_texts])
- return -1;
- (*n_texts)++;
- return 0;
-}
-
-/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one
- * row of space-separated values per line. With print_embs the raw embeddings
- * follow, one per line after a blank line (plain-text output only). */
-static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) {
- if (json) {
- putchar('[');
- for (int i = 0; i < n; i++) {
- if (i)
- putchar(',');
- putchar('[');
- for (int j = 0; j < n; j++) {
- if (j)
- putchar(',');
- printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
- embs + (size_t)j * dim, dim));
- }
- putchar(']');
- }
- puts("]");
- return;
- }
-
- for (int i = 0; i < n; i++) {
- for (int j = 0; j < n; j++) {
- if (j)
- putchar(' ');
- printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
- embs + (size_t)j * dim, dim));
- }
- putchar('\n');
- }
- if (print_embs) {
- putchar('\n');
- for (int i = 0; i < n; i++)
- print_embedding_raw(embs + (size_t)i * dim, dim);
- }
-}
-
-static int run_batch(
- engine_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) {
- int n_texts = 0, cap = 0;
- char **texts = NULL;
- int rc = 0;
-
- if (arg_start < argc) {
- for (int i = arg_start; i < argc; i++) {
- if (append_text(&texts, &n_texts, &cap, argv[i]) != 0)
- goto oom_texts;
- }
- } else {
- char line[65536];
- while (fgets(line, sizeof(line), stdin)) {
- size_t l = strlen(line);
- while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r'))
- line[--l] = '\0';
- if (l == 0)
- continue;
- if (append_text(&texts, &n_texts, &cap, line) != 0)
- goto oom_texts;
- }
- }
-
- if (n_texts == 0) {
- fprintf(stderr, "no input texts\n");
- free(texts);
- return 1;
- }
-
- int dim = e->dim;
- token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t));
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_texts * sizeof(embed_input_t));
- float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
- if (!tokens || !inputs || !embs) {
- fprintf(stderr, "OOM\n");
- free(tokens);
- free(inputs);
- free(embs);
- for (int i = 0; i < n_texts; i++)
- free(texts[i]);
- free(texts);
- return 1;
- }
-
- for (int i = 0; i < n_texts; i++) {
- if (embed_verbose >= 1)
- fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]);
- if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
- rc = 1;
- goto done;
- }
- inputs[i].ids = tokens[i].ids;
- inputs[i].n_tokens = tokens[i].n_tokens;
- }
-
- int max_batch = batch_size > 0 ? batch_size : n_texts;
- for (int start = 0; start < n_texts; start += max_batch) {
- int cur = n_texts - start;
- if (cur > max_batch)
- cur = max_batch;
-
- double t0 = now_ms();
- if (engine_embed_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) {
- fprintf(stderr, "forward pass failed\n");
- rc = 1;
- goto done;
- }
- if (embed_verbose >= 1)
- fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1,
- cur, now_ms() - t0);
- }
-
- if (n_texts == 1) {
- if (json)
- print_embedding_array(embs, dim);
- else
- print_embedding_raw(embs, dim);
- } else {
- print_matrix(embs, n_texts, dim, json, print_embs);
- }
-
-done:
- free_tokens(tokens, n_texts);
- for (int i = 0; i < n_texts; i++)
- free(texts[i]);
- free(texts);
- free(tokens);
- free(inputs);
- free(embs);
- return rc;
-
-oom_texts:
- fprintf(stderr, "OOM\n");
- for (int i = 0; i < n_texts; i++)
- free(texts[i]);
- free(texts);
- return 1;
-}
-
-/* ========================================================================
- * Main
- * ======================================================================== */
-
-int main(int argc, char *argv[]) {
- const char *model_dir = NULL;
- int n_threads = 0;
- int print_embs = 0;
- int verbose = 0;
- int stdin_mode = 0;
- int batch_size = 0;
- int json = 0;
- const char *prog = embed_prog_name(argv[0]);
-#ifdef USE_MLX
- int use_mlx = 0;
- int mlx_quantize_bits = 0;
- int mlx_quantize_group_size = 64;
-#endif
-#ifdef USE_CUDA
- int use_cuda = 0;
- const char *cuda_fast_gemm = NULL;
- const char *cuda_weights = NULL;
-#endif
-
- int arg_start = 1;
- while (arg_start < argc && argv[arg_start][0] == '-') {
- const char *f = argv[arg_start];
- if (!strcmp(f, "-d")) {
- model_dir = argv[++arg_start];
- } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
- embed_print_version(prog);
- return 0;
- } else if (!strcmp(f, "--build-info")) {
- embed_print_build_info(prog);
- return 0;
- } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
- n_threads = atoi(argv[++arg_start]);
- } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
- batch_size = atoi(argv[++arg_start]);
- } else if (!strcmp(f, "--json")) {
- json = 1;
- }
-#ifdef USE_MLX
- else if (!strcmp(f, "--mlx-quant-bits")) {
- mlx_quantize_bits = atoi(argv[++arg_start]);
- if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
- return 1;
- }
- } else if (!strcmp(f, "--mlx-quant-group-size")) {
- mlx_quantize_group_size = atoi(argv[++arg_start]);
- if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
- return 1;
- }
- }
-#endif
-#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda-gemm-mode")) {
- cuda_fast_gemm = argv[++arg_start];
- } else if (!strcmp(f, "--cuda-weight-dtype")) {
- cuda_weights = argv[++arg_start];
- }
-#endif
- else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
- print_embs = 1;
- } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
- verbose++;
- } else if (!strcmp(f, "-vv")) {
- verbose = 2;
- } else if (!strcmp(f, "--stream")) {
- stdin_mode = 1;
- }
-#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) {
- use_mlx = 1;
- }
-#endif
-#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) {
- use_cuda = 1;
- }
-#endif
- else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
- print_usage(prog);
- return 0;
- } else if (!strcmp(f, "--")) {
- /* End of options: everything after is positional text, even if it
- * starts with a dash. */
- arg_start++;
- break;
- } else if (f[0] == '-' && f[1] == '-') {
- /* An unrecognized --option is a hard error, not silently-embedded
- * text. Catches typos and stale flags (e.g. the removed --backend)
- * loudly instead of running the default path or embedding the flag. */
- fprintf(stderr, "unknown option: %s\n", f);
- print_usage(prog);
- return 1;
- } else
- break;
- arg_start++;
- }
-
- embed_verbose = verbose;
-
-#ifdef USE_CUDA
- if (cuda_fast_gemm) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-gemm-mode requires --cuda\n");
- return 1;
- }
- if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
- fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
- return 1;
- }
- }
- if (cuda_weights) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-weight-dtype requires --cuda\n");
- return 1;
- }
- if (!strcmp(cuda_weights, "bf16")) {
- embed_cuda_set_weights_bf16(1);
- } else if (!strcmp(cuda_weights, "f32")) {
- embed_cuda_set_weights_bf16(0);
- } else {
- fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
- return 1;
- }
- }
-#endif
-#ifdef USE_MLX
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quant-bits requires --mlx\n");
- return 1;
- }
-#endif
-
- if (!model_dir || !model_dir[0]) {
- fprintf(stderr, "model directory required (-d <dir>)\n");
- print_usage(prog);
- return 1;
- }
-
-#ifdef USE_MLX
- if (use_mlx) {
- if (verbose >= 1)
- fprintf(stderr, "Using MLX GPU backend\n");
- } else
-#endif
-#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1)
- fprintf(stderr, "Using CUDA GPU backend\n");
- } else
-#endif
- {
- if (n_threads <= 0)
- n_threads = embed_get_num_cpus();
- embed_set_threads(n_threads);
- if (verbose >= 1)
- fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- }
-
- /* Tokenizer selection by files present: WordPiece vocab.txt for BERT,
- * SentencePiece .model for XLM-R/RoBERTa, otherwise Qwen byte-level BPE. */
- engine_t e = {0};
- e.cls_id = -1;
- e.sep_id = -1;
- double t0 = now_ms();
- char wp_path[1024];
- snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", model_dir);
- if (access(wp_path, R_OK) == 0) {
- e.wp_tok = wordpiece_tokenizer_load(model_dir);
- if (e.wp_tok) {
- e.cls_id = wordpiece_tokenizer_token_id(e.wp_tok, "[CLS]");
- e.sep_id = wordpiece_tokenizer_token_id(e.wp_tok, "[SEP]");
- }
- if (!e.wp_tok || e.cls_id < 0 || e.sep_id < 0) {
- fprintf(stderr, "failed to load WordPiece tokenizer: %s\n", wp_path);
- wordpiece_tokenizer_free(e.wp_tok);
- return 1;
- }
- } else {
- char sp_path[1024];
- snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", model_dir);
- if (access(sp_path, R_OK) != 0) {
- snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", model_dir);
- if (access(sp_path, R_OK) != 0)
- snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", model_dir);
- }
- if (access(sp_path, R_OK) == 0) {
- e.sp_tok = sentencepiece_tokenizer_load(model_dir);
- if (e.sp_tok) {
- e.cls_id = sentencepiece_tokenizer_token_id(e.sp_tok, "<s>");
- e.sep_id = sentencepiece_tokenizer_token_id(e.sp_tok, "</s>");
- }
- if (!e.sp_tok || e.cls_id < 0 || e.sep_id < 0) {
- fprintf(stderr, "failed to load SentencePiece tokenizer: %s\n", sp_path);
- sentencepiece_tokenizer_free(e.sp_tok);
- return 1;
- }
- }
- }
- if (!e.wp_tok && !e.sp_tok) {
- char vocab_path[1024];
- snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
- e.tok = embed_tokenizer_load(vocab_path);
- if (!e.tok) {
- fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path);
- return 1;
- }
- }
- if (verbose >= 1)
- fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
-
- /* Model */
- t0 = now_ms();
- const embed_config_t *config = NULL;
-
-#ifdef USE_MLX
- if (use_mlx) {
- embed_mlx_options_t mlx_opts = {
- .quantize_bits = mlx_quantize_bits,
- .quantize_group_size = mlx_quantize_group_size,
- };
- e.mlx_ctx = embed_mlx_load_with_options(model_dir, &mlx_opts);
- if (!e.mlx_ctx) {
- fprintf(stderr, "failed to load model\n");
- engine_free_tokenizers(&e);
- return 1;
- }
- config = embed_mlx_config(e.mlx_ctx);
- e.dim = config->hidden_size;
- } else
-#endif
-#ifdef USE_CUDA
- if (use_cuda) {
- e.cuda_ctx = embed_cuda_load(model_dir);
- if (!e.cuda_ctx) {
- fprintf(stderr, "failed to load CUDA model\n");
- engine_free_tokenizers(&e);
- return 1;
- }
- config = embed_cuda_config(e.cuda_ctx);
- e.dim = config->hidden_size;
- } else
-#endif
- {
- e.model = embed_model_load(model_dir);
- if (!e.model) {
- fprintf(stderr, "failed to load model\n");
- engine_free_tokenizers(&e);
- return 1;
- }
- e.workspace = embed_workspace_new(e.model);
- if (!e.workspace) {
- fprintf(stderr, "failed to allocate workspace\n");
- embed_model_free(e.model);
- engine_free_tokenizers(&e);
- return 1;
- }
- config = embed_model_config(e.model);
- e.dim = config->hidden_size;
- }
- e.append_terminal_token = config->append_terminal_token;
- /* The appended terminal token is the tokenizer's <|endoftext|>, resolved
- * like the contextual separator: a vocab that defines it (test models)
- * wins; otherwise the released-model family constant applies. */
- if (e.append_terminal_token) {
- if (!e.tok) {
- fprintf(stderr, "terminal-token appending requires the BPE tokenizer\n");
- if (e.workspace)
- embed_workspace_free(e.workspace);
- if (e.model)
- embed_model_free(e.model);
-#ifdef USE_MLX
- if (e.mlx_ctx)
- embed_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx)
- embed_cuda_free(e.cuda_ctx);
-#endif
- engine_free_tokenizers(&e);
- return 1;
- }
- int eot = embed_tokenizer_token_id(e.tok, "<|endoftext|>");
- e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
- }
- if (verbose >= 1)
- fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0,
-#ifdef USE_MLX
- use_mlx ? " (MLX)" :
-#endif
-#ifdef USE_CUDA
- use_cuda ? " (CUDA)"
- :
-#endif
- "");
-
- if (e.wp_tok)
- e.wp_tok_ws = wordpiece_workspace_new();
- else if (e.sp_tok)
- e.sp_tok_ws = sentencepiece_workspace_new();
- else
- e.tok_ws = embed_tokenizer_workspace_new();
- if (!e.tok_ws && !e.wp_tok_ws && !e.sp_tok_ws) {
- fprintf(stderr, "failed to allocate tokenizer workspace\n");
- if (e.workspace)
- embed_workspace_free(e.workspace);
- if (e.model)
- embed_model_free(e.model);
-#ifdef USE_MLX
- if (e.mlx_ctx)
- embed_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx)
- embed_cuda_free(e.cuda_ctx);
-#endif
- engine_free_tokenizers(&e);
- return 1;
- }
-
- /* Run */
- int rc;
- if (stdin_mode)
- rc = run_stdin(&e, batch_size > 0 ? batch_size : 1);
- else
- rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json);
-
- /* Cleanup */
- if (e.workspace)
- embed_workspace_free(e.workspace);
- if (e.model)
- embed_model_free(e.model);
-#ifdef USE_MLX
- if (e.mlx_ctx)
- embed_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx)
- embed_cuda_free(e.cuda_ctx);
-#endif
- engine_free_tokenizers(&e);
- return rc;
-}
diff --git a/src/config.h b/src/config.h
deleted file mode 100644
index e8353d3..0000000
--- a/src/config.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef EMBED_CONFIG_H
-#define EMBED_CONFIG_H
-
-#include "embed.h"
-
-int embed_config_parse(embed_config_t *cfg, const char *model_dir);
-
-#endif /* EMBED_CONFIG_H */
diff --git a/src/embed.h b/src/embed.h
deleted file mode 100644
index 5c4f2f3..0000000
--- a/src/embed.h
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
- * embed.h - embed API
- */
-
-#ifndef EMBED_H
-#define EMBED_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-#include <stddef.h>
-#include <stdint.h>
-
-/* ========================================================================
- * Shared constants and family defaults
- * ======================================================================== */
-
-#define EMBED_VOCAB_SIZE 151936
-#define EMBED_HEAD_DIM 128
-#define EMBED_MAX_LAYERS 64 /* upper bound for stack arrays */
-#define EMBED_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */
-
-/* ========================================================================
- * Model Configuration (populated from config.json at load time)
- * ======================================================================== */
-
-typedef enum {
- EMBED_ATTENTION_BIDIRECTIONAL = 0,
- EMBED_ATTENTION_CAUSAL = 1,
-} embed_attention_mode_t;
-
-typedef enum {
- EMBED_POOL_MEAN = 0,
- EMBED_POOL_LAST_TOKEN = 1,
- EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */
-} embed_pooling_mode_t;
-
-typedef enum {
- EMBED_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (pplx, Qwen2/3) */
- EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */
-} embed_family_t;
-
-/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf
- * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or
- * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */
-typedef enum {
- EMBED_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */
- EMBED_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */
-} embed_act_t;
-
-typedef struct {
- embed_family_t family;
- embed_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */
- int hidden_size;
- int n_layers;
- int n_heads;
- int n_kv_heads;
- int head_dim;
- int q_dim; /* n_heads * head_dim */
- int kv_dim; /* n_kv_heads * head_dim */
- int intermediate_size;
- int vocab_size;
- int max_position_embeddings; /* BERT learned position table size */
- int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */
- int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */
- float rms_norm_eps;
- float layer_norm_eps; /* BERT LayerNorm epsilon */
- float rope_theta;
- int qk_norm;
- int qkv_bias;
- embed_attention_mode_t attention_mode;
- embed_pooling_mode_t pooling_mode;
- int normalize_embeddings;
- /* Qwen3-Embedding pools the last token, which is the tokenizer's
- * <|endoftext|> suffix added by its post-processor - not the model's chat
- * eos_token_id (<|im_end|>). Text frontends append it and resolve the id
- * from the tokenizer exactly like the contextual separator
- * (EMBED_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The
- * model forward consumes token ids exactly and never appends. */
- int append_terminal_token;
-} embed_config_t;
-
-/* ========================================================================
- * Model + workspace ownership
- * ======================================================================== */
-
-typedef struct embed_model embed_model_t;
-typedef struct embed_workspace embed_workspace_t;
-typedef struct embed_late_model embed_late_model_t;
-typedef struct embed_late_workspace embed_late_workspace_t;
-
-typedef struct {
- const int *ids;
- int n_tokens;
-} embed_input_t;
-
-typedef struct {
- int start;
- int n_tokens;
-} embed_span_t;
-
-typedef struct {
- embed_input_t input;
- const embed_span_t *spans;
- int n_spans;
-} embed_context_input_t;
-
-/* ========================================================================
- * API
- * ======================================================================== */
-
-/*
- * Load immutable model data from a directory containing:
- * config.json - model hyperparameters
- * model.safetensors (or sharded) - F32 or BF16 weights
- * vocab.json + merges.txt - BPE tokenizer
- *
- * Reads config.json to determine model size (0.6B vs 4B).
- * Returns NULL on error.
- */
-EMBED_API embed_model_t *embed_model_load(const char *model_dir);
-
-EMBED_API void embed_model_free(embed_model_t *model);
-
-/*
- * Allocate/free mutable scratch buffers for a model. The model must outlive
- * workspaces created from it.
- */
-EMBED_API embed_workspace_t *embed_workspace_new(const embed_model_t *model);
-EMBED_API void embed_workspace_free(embed_workspace_t *ws);
-
-/* Config accessor for opaque model ownership. */
-EMBED_API const embed_config_t *embed_model_config(const embed_model_t *model);
-
-/*
- * Current mutable workspace memory owned by this workspace, in bytes.
- *
- * This includes the workspace struct, scratch buffers, and RoPE cache capacity.
- * It does not include immutable mmap'd model weights.
- */
-EMBED_API size_t embed_workspace_nbytes(const embed_workspace_t *ws);
-
-/*
- * Compute embedding for a token sequence.
- *
- * 1. Token embedding lookup
- * 2. N transformer layers (GQA + RoPE + SwiGLU)
- * 3. Final RMSNorm
- * 4. Model-defined pooling and optional L2 normalization
- *
- * pplx-embed-v1 uses bidirectional attention and unnormalized mean pooling.
- * Qwen3-Embedding uses causal attention and normalized last-token pooling.
- * Token IDs are consumed exactly as provided; text frontends append any
- * model-required terminal token before calling this API.
- *
- * Returns malloc'd float[hidden_size] (caller frees). NULL on error.
- */
-EMBED_API float *embed_model_encode(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens);
-
-/*
- * Compute one embedding into caller-provided out[hidden_size].
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_encode_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
-
-/*
- * Compute embeddings for a true packed/ragged batch.
- *
- * out_embeddings is caller-provided [batch, hidden_size].
- * Each sequence attends only to tokens from the same input, and RoPE
- * positions restart at zero for every input.
- *
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_encode_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings);
-
-/*
- * Apply the config's pooling mode to packed final hidden states.
- *
- * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive
- * token count per sequence. The states must already include final RMSNorm.
- */
-EMBED_API int embed_pool_batch(const embed_config_t *cfg,
- const float *states,
- const int *seq_lengths,
- int batch,
- float *out_embeddings);
-
-/*
- * Run the transformer forward pass WITHOUT pooling.
- * Returns the full [n_tokens * hidden_size] output after final RMSNorm.
- * Caller frees the returned buffer. NULL on error.
- *
- * Useful for contextual (late-chunking) models where you need
- * per-token embeddings before splitting by separator positions.
- */
-EMBED_API float *embed_model_forward(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens);
-
-/*
- * Run the transformer forward pass into caller-provided
- * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_forward_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_states);
-
-/*
- * Pool embeddings from final hidden states.
- *
- * states is [n_tokens, hidden_size], normally produced by
- * embed_model_forward_into(). Each span is mean pooled into
- * out_embeddings[n_spans, hidden_size] without L2 normalization.
- */
-EMBED_API int embed_pool_spans(const embed_config_t *cfg,
- const float *states,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings);
-
-/*
- * Run one contextual sequence and pool selected token spans.
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_encode_spans(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings);
-
-/*
- * Run a packed/ragged contextual document batch and pool every selected span.
- *
- * Documents attend independently, RoPE positions restart for each document,
- * and out_embeddings contains spans in document order.
- */
-EMBED_API int embed_model_encode_spans_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings);
-
-/*
- * L2-normalize vec[dim] in place.
- *
- * Returns 0 on success. Returns -1 for invalid arguments or a zero-length
- * vector, which cannot be normalized.
- */
-EMBED_API int embed_l2_normalize(float *vec, int dim);
-
-/*
- * Cosine similarity between two vectors.
- * Returns 0 for invalid arguments or if either vector has zero length.
- */
-EMBED_API float embed_cosine_similarity(const float *a, const float *b, int dim);
-
-/*
- * Late-interaction model API.
- *
- * A late model produces one 128D vector per retained token. The vectors are
- * projected by the snapshot's 1_Dense head and L2-normalized per token by
- * default, matching ColBERT/MaxSim scoring semantics.
- */
-EMBED_API embed_late_model_t *embed_late_model_load(const char *model_dir);
-EMBED_API void embed_late_model_free(embed_late_model_t *model);
-
-EMBED_API embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model);
-EMBED_API void embed_late_workspace_free(embed_late_workspace_t *ws);
-
-EMBED_API const embed_config_t *embed_late_model_config(const embed_late_model_t *model);
-EMBED_API int embed_late_model_token_dim(const embed_late_model_t *model);
-
-/*
- * Encode token_ids into out_vectors[n_tokens, token_dim].
- *
- * If normalize is non-zero, every non-zero token vector is L2-normalized.
- */
-EMBED_API int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors);
-
-/*
- * Encode n_docs documents in one block-diagonal forward and pack each
- * document's kept token vectors back-to-back into out_vectors[total_keep,
- * token_dim] in document order, where total_keep is the sum of n_keep.
- * out_offsets[n_docs + 1] receives the prefix sum of kept counts, the layout
- * embed_late_maxsim_batch consumes. keep[d] lists the n_keep[d] token indices to
- * retain from document d. Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_late_model_encode_docs(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets);
-
-/*
- * Sum over query tokens of max dot product against document tokens.
- */
-EMBED_API float embed_late_maxsim(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- int doc_tokens,
- int dim);
-
-/*
- * Score one query against a packed/ragged batch of documents.
- *
- * doc_offsets has docs + 1 entries. Document i is stored in
- * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim).
- * scores is caller-provided [docs].
- */
-EMBED_API int embed_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores);
-
-/* Verbose level: 0=quiet, 1=info, 2=debug */
-EMBED_API extern int embed_verbose;
-
-#endif /* EMBED_H */
diff --git a/src/kernels_attn.c b/src/kernels_attn.c
deleted file mode 100644
index 66f5ca5..0000000
--- a/src/kernels_attn.c
+++ /dev/null
@@ -1,469 +0,0 @@
-#include "kernels.h"
-#include "kernels_impl.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-#if (defined(__AVX512F__) || defined(__AVX2__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
-#endif
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
-#endif
-
-#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
-#endif
-
-#ifndef M_PI
-#define M_PI 3.14159265358979323846
-#endif
-
-/* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE)
- * GQA, each with an online-softmax and a BLAS-GEMM path. */
-
-/* ========================================================================
- * Attention Operations
- * ======================================================================== */
-
-float embed_dot_f32(const float *a, const float *b, int n) { return embed_dot_f32_impl(a, b, n); }
-
-static inline float embed_dot_f32_fast(const float *a, const float *b, int n) {
- return embed_dot_f32_impl(a, b, n);
-}
-
-/* dst = dst * scale */
-static inline void embed_vec_scale_inplace(float *dst, float scale, int n) {
- embed_vec_scale_inplace_impl(dst, scale, n);
-}
-
-/* dst += alpha * src */
-static inline void embed_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
- embed_vec_axpy_inplace_impl(dst, src, alpha, n);
-}
-
-/* dst = dst * correction + src */
-static inline void embed_vec_scale_add(float *dst, const float *src, float correction, int n) {
- embed_vec_scale_add_impl(dst, src, correction, n);
-}
-
-#define EMBED_PACKED_ATTN_QUERY_TILE 32
-#define EMBED_PACKED_ATTN_BLAS_MIN_SEQ 128
-
-static void embed_bidirectional_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
-
- for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
-
- float max_score = -1e30f;
- float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++)
- o_row[d] = 0.0f;
-
- for (int j = start; j < end; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
-
- float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale;
-
- if (score > max_score) {
- float correction = expf(max_score - score);
- sum_exp = sum_exp * correction + 1.0f;
- embed_vec_scale_add(o_row, v_row, correction, head_dim);
- max_score = score;
- } else {
- float wt = expf(score - max_score);
- sum_exp += wt;
- embed_vec_axpy_inplace(o_row, v_row, wt, head_dim);
- }
- }
-
- if (sum_exp > 0.0f) {
- float inv_sum = 1.0f / sum_exp;
- embed_vec_scale_inplace(o_row, inv_sum, head_dim);
- }
- }
-}
-
-#ifdef USE_BLAS
-static void embed_bidirectional_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
- int rows = query_end - query_start;
- int keys = end - start;
-
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
-
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden,
- k, kv_hidden, 0.0f, scores, keys);
- embed_softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
-}
-#endif
-
-typedef struct {
- float *out;
- const float *Q;
- const float *K;
- const float *V;
- const int *offsets;
- int batch;
- int n_heads, n_kv_heads;
- int head_dim;
- float scale;
- float *scores;
- int max_seq;
-} packed_gqa_attn_task_t;
-
-static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
- int task_id = 0;
-#ifdef USE_BLAS
- float *scores =
- t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
-#endif
-
- for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
-#ifdef USE_BLAS
- if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) {
- embed_bidirectional_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads,
- t->head_dim, t->scale, h, q0, q1, scores);
- continue;
- }
-#endif
- embed_bidirectional_gqa_attention_packed_online_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1);
- }
- }
- }
-}
-
-size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
-#ifdef USE_BLAS
- if (!offsets || batch <= 0)
- return 0;
-
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- if (len > max_seq)
- max_seq = len;
- }
- if (max_seq < EMBED_PACKED_ATTN_BLAS_MIN_SEQ)
- return 0;
- if ((size_t)max_seq >
- SIZE_MAX / (sizeof(float) * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)embed_num_threads()))
- return 0;
- return (size_t)embed_num_threads() * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
-#else
- (void)offsets;
- (void)batch;
- return 0;
-#endif
-}
-
-void embed_bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * len * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
- float *scores = NULL;
-#ifdef USE_BLAS
- size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required)
- scores = scratch;
-#else
- (void)scratch;
- (void)scratch_bytes;
-#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
- if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) {
- parallel_for(packed_gqa_attn_worker, &task);
- } else {
- packed_gqa_attn_worker(0, 1, &task);
- }
-}
-
-void embed_bidirectional_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale) {
- size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
- embed_bidirectional_gqa_attention_packed_with_scratch(
- out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes);
- free(scores);
-}
-
-static void embed_causal_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
-
- for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
-
- float max_score = -1e30f;
- float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++)
- o_row[d] = 0.0f;
-
- for (int j = start; j <= i; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
- float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale;
-
- if (score > max_score) {
- float correction = expf(max_score - score);
- sum_exp = sum_exp * correction + 1.0f;
- embed_vec_scale_add(o_row, v_row, correction, head_dim);
- max_score = score;
- } else {
- float wt = expf(score - max_score);
- sum_exp += wt;
- embed_vec_axpy_inplace(o_row, v_row, wt, head_dim);
- }
- }
-
- if (sum_exp > 0.0f)
- embed_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim);
- }
-}
-
-#ifdef USE_BLAS
-static void embed_causal_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
- int rows = query_end - query_start;
- int keys = query_end - start;
-
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
-
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden,
- k, kv_hidden, 0.0f, scores, keys);
- for (int r = 0; r < rows; r++) {
- int allowed = query_start + r - start + 1;
- float *row = scores + (size_t)r * keys;
- for (int j = allowed; j < keys; j++)
- row[j] = -INFINITY;
- }
- embed_softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
-}
-#endif
-
-static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
- int task_id = 0;
-#ifdef USE_BLAS
- float *scores =
- t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
-#endif
-
- for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
-#ifdef USE_BLAS
- if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) {
- embed_causal_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1, scores);
- continue;
- }
-#endif
- embed_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start,
- t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1);
- }
- }
- }
-}
-
-void embed_causal_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * (len + 1) / 2 * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
- float *scores = NULL;
-#ifdef USE_BLAS
- size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required)
- scores = scratch;
-#else
- (void)scratch;
- (void)scratch_bytes;
-#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
- if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096)
- parallel_for(packed_causal_gqa_attn_worker, &task);
- else
- packed_causal_gqa_attn_worker(0, 1, &task);
-}
-
-void embed_causal_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale) {
- size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
- embed_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
- head_dim, scale, scores, scratch_bytes);
- free(scores);
-}
diff --git a/src/kernels_impl.h b/src/kernels_impl.h
deleted file mode 100644
index 4ee7d74..0000000
--- a/src/kernels_impl.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * embed_kernels_impl.h - internal architecture dispatch for hot kernels
- */
-
-#ifndef EMBED_KERNELS_IMPL_H
-#define EMBED_KERNELS_IMPL_H
-
-#include <stdint.h>
-
-void embed_bf16_matvec_fused_generic(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
-float embed_dot_f32_generic(const float *a, const float *b, int n);
-void embed_vec_scale_inplace_generic(float *dst, float scale, int n);
-void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
-void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n);
-
-#ifdef __ARM_NEON
-void embed_bf16_matvec_fused_neon(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
-float embed_dot_f32_neon(const float *a, const float *b, int n);
-void embed_vec_scale_inplace_neon(float *dst, float scale, int n);
-void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n);
-void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n);
-
-#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_neon
-#define embed_dot_f32_impl embed_dot_f32_neon
-#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_neon
-#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_neon
-#define embed_vec_scale_add_impl embed_vec_scale_add_neon
-
-#elif defined(__AVX2__) && defined(__FMA__)
-void embed_bf16_matvec_fused_avx(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
-float embed_dot_f32_avx(const float *a, const float *b, int n);
-void embed_vec_scale_inplace_avx(float *dst, float scale, int n);
-void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n);
-void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n);
-
-#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_avx
-#define embed_dot_f32_impl embed_dot_f32_avx
-#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_avx
-#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_avx
-#define embed_vec_scale_add_impl embed_vec_scale_add_avx
-
-#else
-#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_generic
-#define embed_dot_f32_impl embed_dot_f32_generic
-#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_generic
-#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_generic
-#define embed_vec_scale_add_impl embed_vec_scale_add_generic
-#endif
-
-#endif /* EMBED_KERNELS_IMPL_H */
diff --git a/src/kernels_norm.c b/src/kernels_norm.c
deleted file mode 100644
index 6bae71d..0000000
--- a/src/kernels_norm.c
+++ /dev/null
@@ -1,245 +0,0 @@
-#include "kernels.h"
-#include "kernels_impl.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-#if (defined(__AVX512F__) || defined(__AVX2__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
-#endif
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
-#endif
-
-#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
-#endif
-
-#ifndef M_PI
-#define M_PI 3.14159265358979323846
-#endif
-
-/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the
- * per-head RMSNorm used for QK-norm. */
-
-/* ========================================================================
- * Normalization
- * ======================================================================== */
-
-/* Rows split across the pool only when the tensor is big enough to pay
- * for the dispatch. */
-#define EMBED_RMS_NORM_PARALLEL_ELEMS 262144
-
-static void embed_rms_norm_range(
- float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) {
- for (int s = start; s < end; s++) {
- const float *x_row = x + (size_t)s * hidden;
- float *out_row = out + (size_t)s * hidden;
-
-#if defined(__AVX512F__) && defined(__FMA__)
- __m512 accv = _mm512_setzero_ps();
- int i = 0;
- for (; i + 16 <= hidden; i += 16) {
- __m512 v = _mm512_loadu_ps(x_row + i);
- accv = _mm512_fmadd_ps(v, v, accv);
- }
- float sum_sq = _mm512_reduce_add_ps(accv);
- for (; i < hidden; i++)
- sum_sq += x_row[i] * x_row[i];
-#elif defined(__AVX2__) && defined(__FMA__)
- __m256 accv = _mm256_setzero_ps();
- int i = 0;
- for (; i + 8 <= hidden; i += 8) {
- __m256 v = _mm256_loadu_ps(x_row + i);
- accv = _mm256_fmadd_ps(v, v, accv);
- }
- __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
- acc128 = _mm_hadd_ps(acc128, acc128);
- acc128 = _mm_hadd_ps(acc128, acc128);
- float sum_sq = _mm_cvtss_f32(acc128);
- for (; i < hidden; i++)
- sum_sq += x_row[i] * x_row[i];
-#else
- float sum_sq = 0.0f;
- for (int i = 0; i < hidden; i++) {
- sum_sq += x_row[i] * x_row[i];
- }
-#endif
- float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps);
-
-#if defined(__AVX512F__)
- __m512 scale = _mm512_set1_ps(rms_inv);
- int j = 0;
- for (; j + 16 <= hidden; j += 16) {
- __m512 vx = _mm512_loadu_ps(x_row + j);
- __m512 vw = _mm512_loadu_ps(weight + j);
- _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale));
- }
- for (; j < hidden; j++)
- out_row[j] = x_row[j] * rms_inv * weight[j];
-#elif defined(__AVX2__)
- __m256 scale = _mm256_set1_ps(rms_inv);
- int j = 0;
- for (; j + 8 <= hidden; j += 8) {
- __m256 vx = _mm256_loadu_ps(x_row + j);
- __m256 vw = _mm256_loadu_ps(weight + j);
- _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale));
- }
- for (; j < hidden; j++)
- out_row[j] = x_row[j] * rms_inv * weight[j];
-#else
- for (int i = 0; i < hidden; i++) {
- out_row[i] = x_row[i] * rms_inv * weight[i];
- }
-#endif
- }
-}
-
-typedef struct {
- float *out;
- const float *x;
- const float *weight;
- int seq_len;
- int hidden;
- float eps;
-} rms_norm_task_t;
-
-static void rms_norm_worker(int tid, int n_threads, void *arg) {
- rms_norm_task_t *t = (rms_norm_task_t *)arg;
- int chunk = (t->seq_len + n_threads - 1) / n_threads;
- int start = tid * chunk;
- int end = start + chunk;
- if (end > t->seq_len)
- end = t->seq_len;
- if (start >= end)
- return;
- embed_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps);
-}
-
-void embed_rms_norm(
- float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) {
- if (seq_len <= 0 || hidden <= 0)
- return;
-
- long long elems = (long long)seq_len * hidden;
- if (embed_num_threads() > 1 && elems >= EMBED_RMS_NORM_PARALLEL_ELEMS) {
- rms_norm_task_t task = {
- .out = out,
- .x = x,
- .weight = weight,
- .seq_len = seq_len,
- .hidden = hidden,
- .eps = eps,
- };
- parallel_for(rms_norm_worker, &task);
- } else {
- embed_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps);
- }
-}
-
-void embed_rms_norm_per_head(
- float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) {
- /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */
- int hidden = n_heads * head_dim;
- for (int s = 0; s < seq_len; s++) {
- for (int h = 0; h < n_heads; h++) {
- float *vec = x + s * hidden + h * head_dim;
-
-#if defined(__AVX512F__) && defined(__FMA__)
- __m512 accv = _mm512_setzero_ps();
- int d = 0;
- for (; d + 16 <= head_dim; d += 16) {
- __m512 v = _mm512_loadu_ps(vec + d);
- accv = _mm512_fmadd_ps(v, v, accv);
- }
- float sum_sq = _mm512_reduce_add_ps(accv);
- for (; d < head_dim; d++)
- sum_sq += vec[d] * vec[d];
-#elif defined(__AVX2__) && defined(__FMA__)
- __m256 accv = _mm256_setzero_ps();
- int d = 0;
- for (; d + 8 <= head_dim; d += 8) {
- __m256 v = _mm256_loadu_ps(vec + d);
- accv = _mm256_fmadd_ps(v, v, accv);
- }
- __m128 acc128 =
- _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
- acc128 = _mm_hadd_ps(acc128, acc128);
- acc128 = _mm_hadd_ps(acc128, acc128);
- float sum_sq = _mm_cvtss_f32(acc128);
- for (; d < head_dim; d++)
- sum_sq += vec[d] * vec[d];
-#else
- float sum_sq = 0.0f;
- for (int d = 0; d < head_dim; d++) {
- sum_sq += vec[d] * vec[d];
- }
-#endif
- float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps);
-
-#if defined(__AVX512F__)
- __m512 scale = _mm512_set1_ps(rms_inv);
- int j = 0;
- for (; j + 16 <= head_dim; j += 16) {
- __m512 v = _mm512_loadu_ps(vec + j);
- __m512 w = _mm512_loadu_ps(weight + j);
- _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale));
- }
- for (; j < head_dim; j++)
- vec[j] = vec[j] * rms_inv * weight[j];
-#elif defined(__AVX2__)
- __m256 scale = _mm256_set1_ps(rms_inv);
- int j = 0;
- for (; j + 8 <= head_dim; j += 8) {
- __m256 v = _mm256_loadu_ps(vec + j);
- __m256 w = _mm256_loadu_ps(weight + j);
- _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale));
- }
- for (; j < head_dim; j++)
- vec[j] = vec[j] * rms_inv * weight[j];
-#else
- for (int d = 0; d < head_dim; d++) {
- vec[d] = vec[d] * rms_inv * weight[d];
- }
-#endif
- }
- }
-}
-
-/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each
- * row's mean and variance are reduced before the row is rewritten. Scalar for
- * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */
-void embed_layer_norm(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int seq_len,
- int hidden,
- float eps) {
- if (seq_len <= 0 || hidden <= 0)
- return;
- float inv_h = 1.0f / (float)hidden;
- for (int s = 0; s < seq_len; s++) {
- const float *row = x + (size_t)s * hidden;
- float *dst = out + (size_t)s * hidden;
- float mean = 0.0f;
- for (int d = 0; d < hidden; d++)
- mean += row[d];
- mean *= inv_h;
- float var = 0.0f;
- for (int d = 0; d < hidden; d++) {
- float diff = row[d] - mean;
- var += diff * diff;
- }
- var *= inv_h;
- float inv_std = 1.0f / sqrtf(var + eps);
- for (int d = 0; d < hidden; d++)
- dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d];
- }
-}
diff --git a/src/late.c b/src/late.c
deleted file mode 100644
index 6ba52cc..0000000
--- a/src/late.c
+++ /dev/null
@@ -1,383 +0,0 @@
-#include "embed.h"
-#include "model_internal.h"
-#include "kernels.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <limits.h>
-#include <float.h>
-
-/* ========================================================================
- * Late-interaction model
- *
- * A late-interaction model is a pooled base model plus a 1_Dense linear
- * projection from hidden states to token vectors. Instead of pooling to one
- * vector, it keeps one projected vector per kept token and scores query/doc
- * pairs with MaxSim. It reuses the base model's packed forward pass and
- * workspace; only the projection, the token-vector buffer, and the scoring
- * live here.
- * ======================================================================== */
-
-struct embed_late_model {
- embed_model_t *base;
- safetensors_file_t *dense_sf;
- embed_weight_ref_t projection; /* [token_dim, hidden] */
- int token_dim;
-};
-
-struct embed_late_workspace {
- const embed_late_model_t *model;
- embed_workspace_t *base_ws;
- float *states;
- int states_seq_cap;
-};
-
-static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) {
- if (!ws || !ws->model || n_tokens <= 0)
- return -1;
- if (n_tokens <= ws->states_seq_cap)
- return 0;
- int hidden = ws->model->base->config.hidden_size;
- if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
- return -1;
- ws->states_seq_cap = n_tokens;
- return 0;
-}
-
-static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
- if (!sf || !name)
- return NULL;
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0)
- return &sf->tensors[i];
- }
- return NULL;
-}
-
-embed_late_model_t *embed_late_model_load(const char *model_dir) {
- if (!model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n");
- return NULL;
- }
-
- embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late));
- if (!late)
- return NULL;
-
- late->base = model_load_range_ex(model_dir, 0, -1, 1);
- if (!late->base)
- goto fail;
-
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_late_model_load: model path too long\n");
- goto fail;
- }
-
- late->dense_sf = safetensors_open(path);
- if (!late->dense_sf) {
- fprintf(stderr, "embed_late_model_load: cannot open %s\n", path);
- goto fail;
- }
-
- const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
- if (!t) {
- fprintf(stderr, "embed_late_model_load: missing linear.weight\n");
- goto fail;
- }
- if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) {
- fprintf(stderr, "embed_late_model_load: bad projection shape\n");
- goto fail;
- }
-
- int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
- if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
- goto fail;
-
- /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */
- late->projection = weight_ref_from_tensor(late->dense_sf, t);
- if (!late->projection.data)
- goto fail;
- late->token_dim = (int)t->shape[0];
- return late;
-
-fail:
- embed_late_model_free(late);
- return NULL;
-}
-
-void embed_late_model_free(embed_late_model_t *model) {
- if (!model)
- return;
- embed_model_free(model->base);
- /* Free the projection before the mmap: only an F16-widened ref owns its
- * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */
- if (model->projection.owned)
- free((void *)model->projection.data);
- safetensors_close(model->dense_sf);
- free(model);
-}
-
-embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) {
- if (!model || !model->base)
- return NULL;
- embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws));
- if (!ws)
- return NULL;
- ws->model = model;
- ws->base_ws = embed_workspace_new(model->base);
- if (!ws->base_ws) {
- free(ws);
- return NULL;
- }
- return ws;
-}
-
-void embed_late_workspace_free(embed_late_workspace_t *ws) {
- if (!ws)
- return;
- embed_workspace_free(ws->base_ws);
- free(ws->states);
- free(ws);
-}
-
-const embed_config_t *embed_late_model_config(const embed_late_model_t *model) {
- return model && model->base ? &model->base->config : NULL;
-}
-
-int embed_late_model_token_dim(const embed_late_model_t *model) {
- return model ? model->token_dim : 0;
-}
-
-/* Internal (embed_internal.h): expose the base model and projection so GPU
- * backends can upload them to device without seeing struct embed_late_model. */
-embed_model_t *embed_late_model_base(const embed_late_model_t *model) {
- return model ? model->base : NULL;
-}
-
-const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) {
- return model ? &model->projection : NULL;
-}
-
-int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors ||
- model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
- if (ensure_late_state_buffer(ws, n_tokens) != 0)
- return -1;
- if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
- return -1;
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden,
- dim);
- if (normalize) {
- for (int i = 0; i < n_tokens; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
-
-float embed_late_maxsim(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- int doc_tokens,
- int dim) {
- if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
- return 0.0f;
-
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *q = query_vectors + (size_t)qi * dim;
- float best = -FLT_MAX;
- for (int di = 0; di < doc_tokens; di++) {
- const float *d = doc_vectors + (size_t)di * dim;
- float dot = embed_dot_f32(q, d, dim);
- if (dot > best)
- best = dot;
- }
- score += best;
- }
- return score;
-}
-
-int embed_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores) {
- if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 ||
- docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
- return -1;
-
- for (int i = 0; i < docs; i++) {
- if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i])
- return -1;
- }
-
-#ifdef USE_BLAS
- /* One similarity GEMM per group of whole candidates:
- * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document
- * row-max plus sum. The group token budget bounds the scratch matrix;
- * one oversized document still runs alone in a group of its own. */
- enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
- int total = doc_offsets[docs];
- int budget = LATE_GROUP_TOKEN_BUDGET;
- size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
- int largest = 0;
- for (int i = 0; i < docs; i++) {
- int len = doc_offsets[i + 1] - doc_offsets[i];
- if (len > largest)
- largest = len;
- }
- if ((size_t)query_tokens * (size_t)largest > cap)
- cap = (size_t)query_tokens * (size_t)largest;
- float *sim = (float *)malloc(cap * sizeof(float));
- if (sim) {
- int doc = 0;
- while (doc < docs) {
- int first = doc;
- int start = doc_offsets[first];
- doc++;
- while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
- doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
- doc++;
- int group_tokens = doc_offsets[doc] - start;
-
- embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
- group_tokens);
-
- for (int i = first; i < doc; i++) {
- int s0 = doc_offsets[i] - start;
- int s1 = doc_offsets[i + 1] - start;
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *row = sim + (size_t)qi * group_tokens;
- float best = row[s0];
- for (int t = s0 + 1; t < s1; t++)
- if (row[t] > best)
- best = row[t];
- score += best;
- }
- scores[i] = score;
- }
- }
- free(sim);
- return 0;
- }
- /* Allocation failure: fall through to the scalar path. */
-#endif
-
- for (int i = 0; i < docs; i++) {
- int start = doc_offsets[i];
- int end = doc_offsets[i + 1];
- scores[i] = embed_late_maxsim(query_vectors, query_tokens,
- doc_vectors + (size_t)start * dim, end - start, dim);
- }
- return 0;
-}
-
-/* Late-interaction batch encoder. Lives here, beside the per-document encoder,
- * and reuses the base model's packed/ragged batch forward pass. */
-int embed_late_model_encode_docs(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets) {
- if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep ||
- n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
-
- /* Pack all candidates for one block-diagonal forward. inputs/offsets index
- * the full token stream; out_offsets is the prefix sum of kept tokens. */
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
- int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets));
- if (!inputs || !offsets) {
- free(inputs);
- free(offsets);
- return -1;
- }
-
- int total_keep = 0;
- out_offsets[0] = 0;
- for (int d = 0; d < n_docs; d++) {
- if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
- n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- for (int k = 0; k < n_keep[d]; k++) {
- if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- }
- inputs[d].ids = doc_ids[d];
- inputs[d].n_tokens = n_tokens[d];
- total_keep += n_keep[d];
- out_offsets[d + 1] = total_keep;
- }
-
- int total_seq = 0, max_seq = 0;
- int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq);
- if (rc == 0)
- rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq,
- max_seq, 1);
- free(inputs);
- if (rc != 0) {
- free(offsets);
- return -1;
- }
-
- /* Gather each document's kept hidden rows into ws->states (its natural
- * [seq, hidden] shape), then project the packed [total_keep, hidden] block
- * to [total_keep, dim] in one GEMM, matching the per-document path. */
- if (ensure_late_state_buffer(ws, total_keep) != 0) {
- free(offsets);
- return -1;
- }
- const float *states = ws->base_ws->x;
- int pos = 0;
- for (int d = 0; d < n_docs; d++) {
- int start = offsets[d];
- for (int k = 0; k < n_keep[d]; k++) {
- memcpy(ws->states + (size_t)pos * hidden,
- states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float));
- pos++;
- }
- }
- free(offsets);
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep,
- hidden, dim);
- if (normalize) {
- for (int i = 0; i < total_keep; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
diff --git a/src/server.c b/src/server.c
deleted file mode 100644
index 808c6f8..0000000
--- a/src/server.c
+++ /dev/null
@@ -1,1042 +0,0 @@
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include "build.h"
-#include "server.h"
-#include "server_internal.h"
-#include "server_util.h"
-#include "sbuf.h"
-#include "embed.h"
-#include "safetensors.h"
-#include "kernels.h"
-#include "tokenizer_bpe.h"
-#include "tokenizer_wordpiece.h"
-
-#ifdef USE_MLX
-#include "mlx.h"
-#endif
-#ifdef USE_CUDA
-#include "cuda.h"
-#endif
-
-#include "deps/ae/ae.h"
-#include "deps/ae/anet.h"
-
-#include <arpa/inet.h>
-#include <ctype.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <netinet/in.h>
-#include <pthread.h>
-#include <signal.h>
-#include <stdbool.h>
-#include <stdint.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-#include <sys/socket.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <time.h>
-#include <unistd.h>
-
-#ifdef __APPLE__
-#include <sys/sysctl.h>
-#endif
-
-#include <cjson/cJSON.h>
-
-static volatile sig_atomic_t g_stop_requested = 0;
-static volatile sig_atomic_t g_signal_wfd = -1;
-
-static void stop_signal_handler(int sig) {
- (void)sig;
- if (g_stop_requested)
- _exit(130);
- int saved = errno;
- g_stop_requested = 1;
- int fd = (int)g_signal_wfd;
- if (fd >= 0) {
- const unsigned char byte = 1;
- if (write(fd, &byte, 1) < 0) {
- /* best-effort stop wakeup; a failed self-pipe write is ignored */
- }
- }
- errno = saved;
-}
-
-static bool auth_ok(http_server *s, const char *auth) {
- if (!s->api_key || !s->api_key[0])
- return true;
- if (!auth)
- return false;
- const char prefix[] = "Bearer ";
- return !strncmp(auth, prefix, sizeof(prefix) - 1) &&
- !strcmp(auth + sizeof(prefix) - 1, s->api_key);
-}
-
-void dispatch_request(client *c) {
- aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
-
- if (!strcmp(c->req.method, "OPTIONS")) {
- append_http_response(c, 204, NULL, "", 0);
- if (queue_write(c) != AE_OK)
- close_client(c);
- return;
- }
-
- if (!route_is_inference(c->req.method, c->req.path)) {
- size_t len;
- char *body = json_error_body("unknown endpoint", "invalid_request_error", &len);
- append_http_response(c, 404, "application/json", body, len);
- free(body);
- if (queue_write(c) != AE_OK)
- close_client(c);
- return;
- }
-
- if (!auth_ok(c->srv, c->req.auth)) {
- size_t len;
- char *body =
- json_error_body("missing or invalid bearer token", "authentication_error", &len);
- append_http_response(c, 401, "application/json", body, len);
- free(body);
- if (queue_write(c) != AE_OK)
- close_client(c);
- return;
- }
-
- job *j = xcalloc(1, sizeof(*j));
- j->srv = c->srv;
- j->c = c;
- snprintf(j->method, sizeof(j->method), "%s", c->req.method);
- snprintf(j->path, sizeof(j->path), "%s", c->req.path);
- j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len);
- j->body_len = c->req.body_len;
- j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
- j->created_ns = nstime();
- client_incref(c);
- enqueue_job(j);
-}
-
-static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop;
- (void)mask;
- http_server *s = clientData;
- char cip[64];
- int cport = 0;
- for (;;) {
- int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport);
- if (cfd == ANET_ERR) {
- if (errno == EAGAIN || errno == EWOULDBLOCK)
- return;
- if (errno == EINTR)
- continue;
- return;
- }
- set_nonblock(cfd);
- client *c = xcalloc(1, sizeof(*c));
- c->srv = s;
- c->fd = cfd;
- c->refcount = 1;
- c->last_active_ms = mstime();
- c->linked = 1;
- c->next = s->clients;
- if (s->clients)
- s->clients->prev = c;
- s->clients = c;
- s->n_clients++;
- if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK)
- close_client(c);
- }
-}
-
-static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
- (void)loop;
- (void)id;
- http_server *s = clientData;
- uint64_t now = mstime();
- client *c = s->clients;
- while (c) {
- client *next = c->next;
- /* refcount > 1 means a request from this client is queued or being
- * processed by the worker; long batch queues (e.g. 4B long-document
- * batches) must not be reaped as idle. */
- if (!c->cancelled && c->refcount <= 1 &&
- now - c->last_active_ms > EMBED_HTTP_CLIENT_TIMEOUT_MS)
- close_client(c);
- c = next;
- }
- return EMBED_HTTP_SWEEP_MS;
-}
-
-static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)fd;
- (void)mask;
- http_server *s = clientData;
- unsigned char tmp[128];
- while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {
- }
- aeStop(loop);
-}
-
-static void *worker_main(void *arg) {
- http_server *s = arg;
-#ifdef USE_MLX
- if (s->use_mlx) {
- int rc = 0;
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- embed_mlx_options_t mlx_opts = {
- .quantize_bits = s->mlx_quantize_bits,
- .quantize_group_size = s->mlx_quantize_group_size,
- };
- if (m->info->kind == MODEL_KIND_LATE)
- m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts);
- else
- m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts);
- if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) ||
- (!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) {
- server_log("embed-server: failed to load MLX model on worker: %s", m->path);
- rc = 1;
- break;
- }
- const embed_config_t *config = m->info->kind == MODEL_KIND_LATE
- ? embed_mlx_late_config(m->mlx_late_ctx)
- : embed_mlx_config(m->mlx_ctx);
- int token_dim =
- m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0;
- if (configure_loaded_model(m, config, token_dim) != 0 ||
- (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size ||
- m->late_query_prefix_id >= config->vocab_size ||
- m->late_document_prefix_id >= config->vocab_size))) {
- rc = 1;
- break;
- }
- }
- pthread_mutex_lock(&s->mu);
- s->worker_init_rc = rc;
- s->worker_ready = 1;
- if (rc)
- s->stopping = 1;
- pthread_cond_broadcast(&s->cv);
- pthread_mutex_unlock(&s->mu);
- if (rc) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->mlx_ctx) {
- embed_mlx_free(m->mlx_ctx);
- m->mlx_ctx = NULL;
- }
- if (m->mlx_late_ctx) {
- embed_mlx_late_free(m->mlx_late_ctx);
- m->mlx_late_ctx = NULL;
- }
- }
- return NULL;
- }
- } else
-#endif
-#ifdef USE_CUDA
- if (s->use_cuda) {
- int rc = 0;
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->info->kind == MODEL_KIND_LATE)
- m->cuda_late_ctx = embed_cuda_late_load(m->path);
- else
- m->cuda_ctx = embed_cuda_load(m->path);
- if ((!m->cuda_ctx && m->info->kind != MODEL_KIND_LATE) ||
- (!m->cuda_late_ctx && m->info->kind == MODEL_KIND_LATE)) {
- server_log("embed-server: failed to load CUDA model on worker: %s", m->path);
- rc = 1;
- break;
- }
- const embed_config_t *config = m->info->kind == MODEL_KIND_LATE
- ? embed_cuda_late_config(m->cuda_late_ctx)
- : embed_cuda_config(m->cuda_ctx);
- int token_dim =
- m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_token_dim(m->cuda_late_ctx) : 0;
- if (configure_loaded_model(m, config, token_dim) != 0 ||
- (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size ||
- m->late_query_prefix_id >= config->vocab_size ||
- m->late_document_prefix_id >= config->vocab_size))) {
- rc = 1;
- break;
- }
- }
- pthread_mutex_lock(&s->mu);
- s->worker_init_rc = rc;
- s->worker_ready = 1;
- if (rc)
- s->stopping = 1;
- pthread_cond_broadcast(&s->cv);
- pthread_mutex_unlock(&s->mu);
- if (rc) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->cuda_ctx) {
- embed_cuda_free(m->cuda_ctx);
- m->cuda_ctx = NULL;
- }
- if (m->cuda_late_ctx) {
- embed_cuda_late_free(m->cuda_late_ctx);
- m->cuda_late_ctx = NULL;
- }
- }
- return NULL;
- }
- } else
-#endif
- {
- pthread_mutex_lock(&s->mu);
- s->worker_ready = 1;
- pthread_cond_broadcast(&s->cv);
- pthread_mutex_unlock(&s->mu);
- }
-
- for (;;) {
- job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS];
- int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS);
- if (n_jobs == 0)
- break;
- process_job_group(s, jobs, n_jobs);
- }
-#ifdef USE_MLX
- if (s->use_mlx) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->mlx_ctx) {
- embed_mlx_free(m->mlx_ctx);
- m->mlx_ctx = NULL;
- }
- if (m->mlx_late_ctx) {
- embed_mlx_late_free(m->mlx_late_ctx);
- m->mlx_late_ctx = NULL;
- }
- }
- }
-#endif
-#ifdef USE_CUDA
- if (s->use_cuda) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->cuda_ctx) {
- embed_cuda_free(m->cuda_ctx);
- m->cuda_ctx = NULL;
- }
- if (m->cuda_late_ctx) {
- embed_cuda_late_free(m->cuda_late_ctx);
- m->cuda_late_ctx = NULL;
- }
- }
- }
-#endif
- return NULL;
-}
-
-static int listen_on(const char *host, int port) {
- char portbuf[32];
- snprintf(portbuf, sizeof(portbuf), "%d", port);
- char err[ANET_ERR_LEN];
- int fd = anetTcpServer(err, port, (char *)host, EMBED_HTTP_BACKLOG);
- (void)portbuf;
- if (fd == ANET_ERR) {
- server_log("embed-server: listen failed on %s:%d: %s", host, port, err);
- return -1;
- }
- set_nonblock(fd);
- return fd;
-}
-
-static uint64_t physical_memory_nbytes(void) {
-#ifdef __APPLE__
- uint64_t bytes = 0;
- size_t len = sizeof(bytes);
- if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0)
- return bytes;
-#endif
- long pages = sysconf(_SC_PHYS_PAGES);
- long page_size = sysconf(_SC_PAGESIZE);
- if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
- return 0;
- return (uint64_t)pages * (uint64_t)page_size;
-}
-
-static int validate_model_specs(const embed_server_config_t *cfg) {
- for (int i = 0; i < cfg->n_models; i++) {
- const embed_server_model_spec_t *spec = &cfg->models[i];
- if (!spec->id || !spec->id[0]) {
- fprintf(stderr, "embed-server: model label must not be empty\n");
- return -1;
- }
- if (!spec->path || !spec->path[0]) {
- fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id);
- return -1;
- }
- if (spec->kind < EMBED_SERVER_MODEL_STANDARD || spec->kind > EMBED_SERVER_MODEL_LATE) {
- fprintf(stderr, "embed-server: model %s has an invalid kind\n", spec->id);
- return -1;
- }
- if (spec->api < EMBED_SERVER_API_DEFAULT || spec->api > EMBED_SERVER_API_PERPLEXITY) {
- fprintf(stderr, "embed-server: model %s has an invalid api\n", spec->id);
- return -1;
- }
- if (spec->min_dim < 0) {
- fprintf(stderr, "embed-server: model %s has an invalid min_dim\n", spec->id);
- return -1;
- }
- for (int j = 0; j < i; j++) {
- if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) {
- fprintf(stderr, "embed-server: duplicate model label: %s\n", spec->id);
- return -1;
- }
- }
- }
- return 0;
-}
-
-static int mlx_memory_preflight(const embed_server_config_t *cfg) {
- if (!cfg->use_mlx)
- return 0;
-
- uint64_t payload = 0;
- for (int i = 0; i < cfg->n_models; i++) {
- multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path);
- size_t model_payload = 0;
- if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) {
- multi_safetensors_close(ms);
- fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path);
- return -1;
- }
- multi_safetensors_close(ms);
- if ((uint64_t)model_payload > UINT64_MAX - payload) {
- fprintf(stderr, "embed-server: MLX model payload size overflow\n");
- return -1;
- }
- payload += (uint64_t)model_payload;
- }
-
- if (payload > UINT64_MAX / EMBED_MLX_RESIDENT_MULTIPLIER) {
- fprintf(stderr, "embed-server: MLX resident-memory estimate overflow\n");
- return -1;
- }
- uint64_t estimated = payload * EMBED_MLX_RESIDENT_MULTIPLIER;
- uint64_t physical = physical_memory_nbytes();
- if (physical == 0) {
- server_log("embed-server: warning: could not determine physical memory; "
- "skipping MLX memory preflight");
- return 0;
- }
- double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization
- : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0;
- uint64_t budget = (uint64_t)((double)physical * utilization);
- const double gib = 1024.0 * 1024.0 * 1024.0;
- server_log("embed-server: MLX memory preflight: %.1f GiB tensors, "
- "%.1f GiB estimated resident, %.1f GiB budget "
- "(%.2f of physical memory)",
- (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization);
- if (cfg->mlx_quantize_bits) {
- server_log("embed-server: MLX %d-bit quantization enabled; preflight "
- "uses source payload as a conservative peak estimate",
- cfg->mlx_quantize_bits);
- }
- if (estimated <= budget)
- return 0;
- server_log("embed-server: refusing MLX model set above the host-memory "
- "budget");
- server_log("embed-server: use BF16 artifacts, load fewer models, or raise "
- "--memory-utilization (values above 1.0 overcommit)");
- return -1;
-}
-
-int embed_run_server(const embed_server_config_t *cfg) {
- if (!cfg || !cfg->models || cfg->n_models <= 0) {
- fprintf(stderr, "embed-server: at least one model label and path is required\n");
- return 1;
- }
- if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0)
- return 1;
-
- http_server s;
- memset(&s, 0, sizeof(s));
- s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1";
- s.port = cfg->port > 0 ? cfg->port : 8000;
- s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE;
- s.max_batch_tokens =
- cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS;
- s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us
- : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US
- : EMBED_SERVER_BATCH_WAIT_US);
- s.use_mlx = cfg->use_mlx;
- s.use_cuda = cfg->use_cuda;
- s.mlx_quantize_bits = cfg->mlx_quantize_bits;
- s.mlx_quantize_group_size =
- cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64;
- if (cfg->api_key && cfg->api_key[0])
- s.api_key = xstrdup(cfg->api_key);
- else {
- const char *env_key = getenv("EMBED_API_KEY");
- if (env_key && env_key[0])
- s.api_key = xstrdup(env_key);
- }
-
- pthread_mutex_init(&s.mu, NULL);
- pthread_cond_init(&s.cv, NULL);
-
- for (int i = 0; i < cfg->n_models; i++) {
- if (load_one_model(&s, &cfg->models[i]) != 0) {
- free_models(&s);
- return 1;
- }
- }
-
- if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) {
- perror("pipe");
- free_models(&s);
- return 1;
- }
- set_nonblock(s.completion_pipe[0]);
- set_nonblock(s.completion_pipe[1]);
- set_nonblock(s.signal_pipe[0]);
- set_nonblock(s.signal_pipe[1]);
-
- struct sigaction sa;
- memset(&sa, 0, sizeof(sa));
- sa.sa_handler = stop_signal_handler;
- sigemptyset(&sa.sa_mask);
- sigaction(SIGINT, &sa, NULL);
- sigaction(SIGTERM, &sa, NULL);
- signal(SIGPIPE, SIG_IGN);
- g_signal_wfd = s.signal_pipe[1];
-
- s.loop = aeCreateEventLoop(EMBED_HTTP_SETSIZE);
- if (!s.loop) {
- fprintf(stderr, "embed-server: failed to create event loop\n");
- free_models(&s);
- return 1;
- }
-
- if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) {
- fprintf(stderr, "embed-server: failed to start worker\n");
- aeDeleteEventLoop(s.loop);
- free_models(&s);
- return 1;
- }
- pthread_mutex_lock(&s.mu);
- while (!s.worker_ready)
- pthread_cond_wait(&s.cv, &s.mu);
- int worker_init_rc = s.worker_init_rc;
- pthread_mutex_unlock(&s.mu);
- if (worker_init_rc) {
- pthread_join(s.worker, NULL);
- aeDeleteEventLoop(s.loop);
- free_models(&s);
- free(s.api_key);
- pthread_cond_destroy(&s.cv);
- pthread_mutex_destroy(&s.mu);
- return 1;
- }
-
- s.listen_fd = listen_on(s.host, s.port);
- if (s.listen_fd < 0) {
- pthread_mutex_lock(&s.mu);
- s.stopping = 1;
- pthread_cond_signal(&s.cv);
- pthread_mutex_unlock(&s.mu);
- pthread_join(s.worker, NULL);
- aeDeleteEventLoop(s.loop);
- free_models(&s);
- return 1;
- }
-
- aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s);
- aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s);
- aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s);
- aeCreateTimeEvent(s.loop, EMBED_HTTP_SWEEP_MS, timeout_cb, &s, NULL);
-
- server_log("embed-server: listening on http://%s:%d", s.host, s.port);
- aeMain(s.loop);
-
- server_log("embed-server: shutdown requested");
- if (s.listen_fd >= 0)
- close(s.listen_fd);
- client *c = s.clients;
- while (c) {
- client *next = c->next;
- close_client(c);
- c = next;
- }
-
- pthread_mutex_lock(&s.mu);
- s.stopping = 1;
- pthread_cond_signal(&s.cv);
- pthread_mutex_unlock(&s.mu);
- pthread_join(s.worker, NULL);
-
- aeDeleteEventLoop(s.loop);
- close(s.completion_pipe[0]);
- close(s.completion_pipe[1]);
- close(s.signal_pipe[0]);
- close(s.signal_pipe[1]);
- free_models(&s);
- free(s.api_key);
- pthread_cond_destroy(&s.cv);
- pthread_mutex_destroy(&s.mu);
- return 0;
-}
-
-/* ====================================================================
- * embed-server entry point
- * ==================================================================== */
-#ifndef EMBED_SERVER_TEST
-
-static void print_usage(const char *prog) {
- fprintf(stderr,
- "Usage: %s --model LABEL=DIR [options]\n"
- "\n"
- "Options:\n"
- " --model LABEL=DIR[:key=val]\n"
- " Standard embedding model (repeatable)\n"
- " --contextual-model LABEL=DIR[:key=val]\n"
- " Contextual embedding model (repeatable)\n"
- " --late-model LABEL=DIR[:key=val]\n"
- " Late-interaction rerank model (repeatable)\n"
- " keys: api=openai|perplexity, min_dim=N,\n"
- " query=TEXT (last key; \\n escapes newline)\n"
-#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
-#endif
-#ifdef USE_CUDA
- " --cuda Use NVIDIA CUDA GPU backend\n"
-#endif
- " --host HOST Bind host (default: 127.0.0.1)\n"
- " --port N Bind port (default: 8000)\n"
- " --api-key K Require Authorization: Bearer K\n"
- " -b, --batch-size N Max texts or documents per inference batch\n"
- " (default: 32)\n"
- " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
- " --batch-wait-us N First-arrival micro-batch deadline in us\n"
-#ifdef USE_CUDA
- " (default: CUDA 1000; CPU 0)\n"
-#else
- " (default: 0)\n"
-#endif
-#ifdef USE_MLX
- " --memory-utilization F Fraction of physical memory the MLX model-set\n"
- " preflight may plan for (default: 0.90; values\n"
- " above 1.0 overcommit)\n"
- " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
- " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
-#endif
-#ifdef USE_CUDA
- " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
- " (default: f32, exact)\n"
- " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
- " f32); bf16 halves weight memory\n"
-#endif
- " -t, --threads N CPU threads (default: available cores, cgroup-aware)\n"
- " -v, --verbose Verbose (-vv for debug)\n"
- " -V, --version Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h, --help Show this help\n"
- "\n"
- "Examples:\n"
- " %s --model docs=./model --port 8000\n"
- " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n"
-#ifdef USE_MLX
- " %s --mlx --mlx-quant-bits 8 \\\n"
- " --model pplx=./model-4b-bf16:api=perplexity:min_dim=128\n",
- prog, prog, prog, prog);
-#else
- ,
- prog, prog, prog);
-#endif
-}
-
-typedef struct {
- embed_server_model_spec_t *v;
- int n;
- int cap;
-} model_specs_t;
-
-static char *dup_range(const char *p, size_t n) {
- char *s = (char *)malloc(n + 1);
- if (!s)
- return NULL;
- memcpy(s, p, n);
- s[n] = '\0';
- return s;
-}
-
-static char *unescape_model_option(const char *s) {
- size_t n = strlen(s);
- char *out = (char *)malloc(n + 1);
- if (!out)
- return NULL;
- size_t w = 0;
- for (size_t r = 0; r < n; r++) {
- if (s[r] == '\\' && r + 1 < n) {
- r++;
- if (s[r] == 'n')
- out[w++] = '\n';
- else if (s[r] == 't')
- out[w++] = '\t';
- else if (s[r] == 'r')
- out[w++] = '\r';
- else
- out[w++] = s[r];
- } else {
- out[w++] = s[r];
- }
- }
- out[w] = '\0';
- return out;
-}
-
-static int parse_model_option(embed_server_model_spec_t *spec, const char *opt, const char *flag) {
- const char *eq = strchr(opt, '=');
- if (!eq || eq == opt || !eq[1]) {
- fprintf(stderr, "%s option expects key=value: %s\n", flag, opt);
- return -1;
- }
- size_t key_len = (size_t)(eq - opt);
- const char *val = eq + 1;
- if (key_len == 3 && strncmp(opt, "api", key_len) == 0) {
- if (!strcmp(val, "openai")) {
- spec->api = EMBED_SERVER_API_OPENAI;
- } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) {
- spec->api = EMBED_SERVER_API_PERPLEXITY;
- } else if (!strcmp(val, "default")) {
- spec->api = EMBED_SERVER_API_DEFAULT;
- } else {
- fprintf(stderr, "%s api must be openai or perplexity\n", flag);
- return -1;
- }
- return 0;
- }
- if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) {
- char *end = NULL;
- long v = strtol(val, &end, 10);
- if (!end || *end || v <= 0 || v > INT_MAX) {
- fprintf(stderr, "%s min_dim must be a positive integer\n", flag);
- return -1;
- }
- spec->min_dim = (int)v;
- return 0;
- }
- fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt);
- return -1;
-}
-
-static int append_model_spec(model_specs_t *specs,
- const char *arg,
- embed_server_model_kind_t kind,
- const char *flag) {
- const char *eq = strchr(arg, '=');
- if (!eq || eq == arg || !eq[1]) {
- fprintf(stderr, "%s expects LABEL=DIR\n", flag);
- return -1;
- }
- if (specs->n == specs->cap) {
- int new_cap = specs->cap ? specs->cap * 2 : 4;
- void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
- if (!p)
- return -1;
- specs->v = p;
- specs->cap = new_cap;
- }
- size_t id_len = (size_t)(eq - arg);
- const char *path_start = eq + 1;
- const char *opts = strchr(path_start, ':');
- size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start);
- if (path_len == 0) {
- fprintf(stderr, "%s expects a non-empty DIR\n", flag);
- return -1;
- }
- embed_server_model_spec_t spec;
- memset(&spec, 0, sizeof(spec));
- spec.id = dup_range(arg, id_len);
- spec.path = dup_range(path_start, path_len);
- spec.kind = kind;
- if (!spec.id || !spec.path) {
- free((char *)spec.id);
- free((char *)spec.path);
- return -1;
- }
- const char *p = opts ? opts + 1 : NULL;
- while (p && *p) {
- if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) {
- const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15);
- spec.query_instruct = unescape_model_option(val);
- if (!spec.query_instruct)
- goto fail;
- break;
- }
- const char *next = strchr(p, ':');
- size_t opt_len = next ? (size_t)(next - p) : strlen(p);
- char *opt = dup_range(p, opt_len);
- if (!opt)
- goto fail;
- int rc = parse_model_option(&spec, opt, flag);
- free(opt);
- if (rc != 0)
- goto fail;
- p = next ? next + 1 : NULL;
- }
- specs->v[specs->n] = spec;
- specs->n++;
- return 0;
-
-fail:
- free((char *)spec.id);
- free((char *)spec.path);
- free((char *)spec.query_instruct);
- return -1;
-}
-
-static void free_model_specs(model_specs_t *specs) {
- if (!specs)
- return;
- for (int i = 0; i < specs->n; i++) {
- free((char *)specs->v[i].id);
- free((char *)specs->v[i].path);
- free((char *)specs->v[i].query_instruct);
- }
- free(specs->v);
-}
-
-int main(int argc, char *argv[]) {
- int n_threads = 0;
- int verbose = 0;
- int batch_size = 0;
- int max_batch_tokens = 0;
- int batch_wait_us = -1;
-#ifdef USE_MLX
- int use_mlx = 0;
- int mlx_quantize_bits = 0;
- int mlx_quantize_group_size = 64;
- double memory_utilization = 0.0;
-#endif
-#ifdef USE_CUDA
- int use_cuda = 0;
- const char *cuda_fast_gemm = NULL;
- const char *cuda_weights = NULL;
-#endif
- const char *host = "127.0.0.1";
- const char *api_key = NULL;
- int port = 8000;
- model_specs_t model_specs = {0};
-
- const char *prog = embed_prog_name(argv[0]);
- int arg = 1;
- while (arg < argc && argv[arg][0] == '-') {
- const char *f = argv[arg];
- if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
- embed_print_version(prog);
- free_model_specs(&model_specs);
- return 0;
- } else if (!strcmp(f, "--build-info")) {
- embed_print_build_info(prog);
- free_model_specs(&model_specs);
- return 0;
- } else if (!strcmp(f, "--model")) {
- if (arg + 1 >= argc ||
- append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_STANDARD, f) !=
- 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--contextual-model")) {
- if (arg + 1 >= argc ||
- append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_CONTEXTUAL, f) !=
- 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--late-model")) {
- if (arg + 1 >= argc ||
- append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_LATE, f) != 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- }
-#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) {
- use_mlx = 1;
- } else if (!strcmp(f, "--mlx-quant-bits")) {
- mlx_quantize_bits = atoi(argv[++arg]);
- if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--mlx-quant-group-size")) {
- mlx_quantize_group_size = atoi(argv[++arg]);
- if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--memory-utilization")) {
- memory_utilization = atof(argv[++arg]);
- if (memory_utilization <= 0.0) {
- fprintf(stderr, "--memory-utilization must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
-#endif
-#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) {
- use_cuda = 1;
- } else if (!strcmp(f, "--cuda-gemm-mode")) {
- cuda_fast_gemm = argv[++arg];
- } else if (!strcmp(f, "--cuda-weight-dtype")) {
- cuda_weights = argv[++arg];
- }
-#endif
- else if (!strcmp(f, "--host")) {
- host = argv[++arg];
- } else if (!strcmp(f, "--port")) {
- port = atoi(argv[++arg]);
- } else if (!strcmp(f, "--api-key")) {
- api_key = argv[++arg];
- } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
- batch_size = atoi(argv[++arg]);
- } else if (!strcmp(f, "--max-batch-tokens")) {
- max_batch_tokens = atoi(argv[++arg]);
- if (max_batch_tokens <= 0) {
- fprintf(stderr, "--max-batch-tokens must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--batch-wait-us")) {
- batch_wait_us = atoi(argv[++arg]);
- if (batch_wait_us < 0) {
- fprintf(stderr, "--batch-wait-us must be >= 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
- n_threads = atoi(argv[++arg]);
- } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
- verbose++;
- } else if (!strcmp(f, "-vv")) {
- verbose = 2;
- } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
- print_usage(prog);
- free_model_specs(&model_specs);
- return 0;
- } else {
- fprintf(stderr, "unknown option: %s\n", f);
- print_usage(prog);
- free_model_specs(&model_specs);
- return 1;
- }
- arg++;
- }
-
- embed_verbose = verbose;
-
- if (arg < argc) {
- fprintf(stderr, "unexpected argument: %s\n", argv[arg]);
- print_usage(prog);
- free_model_specs(&model_specs);
- return 1;
- }
- if (model_specs.n == 0) {
- fprintf(stderr, "at least one --model LABEL=DIR is required\n");
- print_usage(prog);
- free_model_specs(&model_specs);
- return 1;
- }
-#ifdef USE_MLX
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quant-bits requires --mlx\n");
- free_model_specs(&model_specs);
- return 1;
- }
-#endif
-#ifdef USE_CUDA
- if (cuda_fast_gemm) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-gemm-mode requires --cuda\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
- fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- if (cuda_weights) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-weight-dtype requires --cuda\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (!strcmp(cuda_weights, "bf16")) {
- embed_cuda_set_weights_bf16(1);
- } else if (!strcmp(cuda_weights, "f32")) {
- embed_cuda_set_weights_bf16(0);
- } else {
- fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
-#endif
-
-#ifdef USE_MLX
- if (use_mlx) {
- if (verbose >= 1)
- fprintf(stderr, "Using MLX GPU backend\n");
- } else
-#endif
-#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1)
- fprintf(stderr, "Using CUDA GPU backend\n");
- } else
-#endif
- {
- if (n_threads <= 0)
- n_threads = embed_get_num_cpus();
- embed_set_threads(n_threads);
- if (verbose >= 1)
- fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- }
-
- embed_server_config_t scfg = {
- .models = model_specs.v,
- .n_models = model_specs.n,
- .host = host,
- .port = port,
- .batch_size = batch_size > 0 ? batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE,
- .max_batch_tokens = max_batch_tokens,
- .batch_wait_us = batch_wait_us,
-#ifdef USE_MLX
- .use_mlx = use_mlx,
- .mlx_quantize_bits = mlx_quantize_bits,
- .mlx_quantize_group_size = mlx_quantize_group_size,
- .memory_utilization = memory_utilization,
-#endif
-#ifdef USE_CUDA
- .use_cuda = use_cuda,
-#endif
- .api_key = api_key,
- };
- int rc = embed_run_server(&scfg);
- free_model_specs(&model_specs);
- return rc;
-}
-
-#endif /* EMBED_SERVER_TEST */
diff --git a/src/server.h b/src/server.h
deleted file mode 100644
index 5ad2b12..0000000
--- a/src/server.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#ifndef EMBED_SERVER_H
-#define EMBED_SERVER_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-#include <stdbool.h>
-
-typedef enum {
- EMBED_SERVER_MODEL_STANDARD = 0,
- EMBED_SERVER_MODEL_CONTEXTUAL = 1,
- EMBED_SERVER_MODEL_LATE = 2,
-} embed_server_model_kind_t;
-
-typedef enum {
- EMBED_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */
- EMBED_SERVER_API_OPENAI = 1,
- EMBED_SERVER_API_PERPLEXITY = 2,
-} embed_server_embedding_api_t;
-
-typedef struct {
- const char *id; /* served label; requests match this string exactly */
- const char *path;
- embed_server_model_kind_t kind;
- embed_server_embedding_api_t api;
- int min_dim; /* 0 means the loaded model's full hidden size */
- const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */
-} embed_server_model_spec_t;
-
-typedef struct {
- const embed_server_model_spec_t *models;
- int n_models;
- const char *host;
- int port;
- int batch_size;
- int max_batch_tokens;
- int batch_wait_us;
- int use_mlx;
- int use_cuda;
- int mlx_quantize_bits;
- int mlx_quantize_group_size;
- double memory_utilization; /* fraction of physical memory the MLX
- preflight may plan for; 0 = default 0.90,
- values above 1.0 allow overcommit */
- const char *api_key;
-} embed_server_config_t;
-
-EMBED_API int embed_run_server(const embed_server_config_t *cfg);
-
-#endif
diff --git a/src/server_models.c b/src/server_models.c
deleted file mode 100644
index 320840b..0000000
--- a/src/server_models.c
+++ /dev/null
@@ -1,627 +0,0 @@
-#include "server_internal.h"
-#include "server_util.h"
-
-#include <errno.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-/* ========================================================================
- * Model load/lifecycle, and tokenize/inference dispatch
- *
- * Each loaded model owns resolved serving metadata for the operator-chosen
- * label. Transformer behavior comes from the model files through embed_config;
- * server-only facts absent from those files, such as output API family and
- * Matryoshka floor, come from the load spec. The tokenize_* and model_embed_*
- * helpers are the single dispatch point the request handlers call, hiding the
- * CPU/MLX/CUDA backend choice behind one signature.
- * ======================================================================== */
-
-loaded_model *loaded_model_for_label(http_server *s, const char *label) {
- if (!label)
- return NULL;
- for (int i = 0; i < s->n_models; i++) {
- if (s->models[i].info && !strcmp(label, s->models[i].info->id))
- return &s->models[i];
- }
- return NULL;
-}
-
-static model_kind model_kind_from_public(embed_server_model_kind_t kind) {
- switch (kind) {
- case EMBED_SERVER_MODEL_CONTEXTUAL:
- return MODEL_KIND_CONTEXTUAL;
- case EMBED_SERVER_MODEL_LATE:
- return MODEL_KIND_LATE;
- case EMBED_SERVER_MODEL_STANDARD:
- default:
- return MODEL_KIND_STANDARD;
- }
-}
-
-static embedding_api_t embedding_api_from_public(embed_server_embedding_api_t api) {
- return api == EMBED_SERVER_API_PERPLEXITY ? EMBED_API_PERPLEXITY : EMBED_API_OPENAI;
-}
-
-static int read_optional_text_file(const char *model_dir, const char *relative, char **out) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- server_log("embed-server: model path too long");
- return -1;
- }
- FILE *f = fopen(path, "rb");
- if (!f && errno == ENOENT)
- return 0;
- if (!f) {
- server_log("embed-server: failed to open %s", path);
- return -1;
- }
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return -1;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return -1;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf) {
- fclose(f);
- return -1;
- }
- if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- free(buf);
- fclose(f);
- return -1;
- }
- fclose(f);
- buf[sz] = '\0';
- *out = buf;
- return 1;
-}
-
-static int read_query_instruct_from_files(const char *model_dir, char **out) {
- char *buf = NULL;
- int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf);
- if (rc <= 0)
- return rc;
- cJSON *root = cJSON_Parse(buf);
- free(buf);
- if (!root) {
- server_log("embed-server: invalid config_sentence_transformers.json: %s", model_dir);
- return -1;
- }
- cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts");
- cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL;
- if (query && !cJSON_IsString(query)) {
- cJSON_Delete(root);
- server_log("embed-server: prompts.query must be a string: %s", model_dir);
- return -1;
- }
- if (query && query->valuestring && query->valuestring[0]) {
- *out = xstrdup(query->valuestring);
- cJSON_Delete(root);
- return *out ? 1 : -1;
- }
- cJSON_Delete(root);
- return 0;
-}
-
-static int init_model_info(model_info **out, const embed_server_model_spec_t *spec) {
- model_info *info = xcalloc(1, sizeof(*info));
- info->id = xstrdup(spec->id);
- info->kind = model_kind_from_public(spec->kind);
- info->api = embedding_api_from_public(spec->api);
- info->min_dim = spec->min_dim;
- if (spec->query_instruct) {
- info->query_instruct = xstrdup(spec->query_instruct);
- } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) {
- free(info->id);
- free(info);
- return -1;
- }
- if (!info->id || (spec->query_instruct && !info->query_instruct)) {
- free(info->query_instruct);
- free(info->id);
- free(info);
- return -1;
- }
- *out = info;
- return 0;
-}
-
-static void free_model_info(model_info *info) {
- if (!info)
- return;
- free(info->id);
- free(info->query_instruct);
- free(info);
-}
-
-void free_token_bufs(token_buf *t, int n) {
- if (!t)
- return;
- for (int i = 0; i < n; i++)
- free(t[i].ids);
-}
-
-void embedding_request_free(embedding_request *r) {
- if (!r)
- return;
- free_token_bufs(r->tokens, r->n_inputs);
- free(r->tokens);
- free(r->inputs);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-void contextual_request_free(contextual_request *r) {
- if (!r)
- return;
- for (int i = 0; i < r->n_docs; i++) {
- free(r->docs[i].ids);
- free(r->docs[i].spans);
- }
- free(r->docs);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-void late_tokens_free(late_tokens *t) {
- if (!t)
- return;
- free(t->ids);
- free(t->keep);
- memset(t, 0, sizeof(*t));
-}
-
-void rerank_request_free(rerank_request *r) {
- if (!r)
- return;
- late_tokens_free(&r->query);
- for (int i = 0; i < r->n_documents; i++)
- late_tokens_free(&r->documents[i]);
- free(r->documents);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
- memset(out, 0, sizeof(*out));
- uint64_t t0 = nstime();
- if (m->wp_tok) {
- int n = 0;
- int *core = wordpiece_tokenizer_encode_with_workspace(m->wp_tok, m->wp_tok_ws, text, &n);
- if (j)
- j->tokenize_ns += nstime() - t0;
- /* BERT wraps the WordPiece ids with [CLS] ... [SEP], exactly as the BPE
- * branch below appends its terminal token. A whitespace-only input
- * cleans to zero core tokens (core == NULL, n == 0); the bare
- * [CLS][SEP] pair is what Hugging Face produces for it too. Grow the
- * returned buffer in place so only one allocation is touched. */
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = m->cls_id;
- ids[n + 1] = m->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- return 0;
- }
- if (m->sp_tok) {
- int n = 0;
- int *core =
- sentencepiece_tokenizer_encode_with_workspace(m->sp_tok, m->sp_tok_ws, text, &n);
- if (j)
- j->tokenize_ns += nstime() - t0;
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = m->cls_id;
- ids[n + 1] = m->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- return 0;
- }
- out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens);
- if (j)
- j->tokenize_ns += nstime() - t0;
- if (!out->ids || out->n_tokens <= 0) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- if (m->append_terminal_token) {
- if (out->n_tokens == INT_MAX) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
- if (!ids) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- out->ids = ids;
- out->ids[out->n_tokens++] = m->terminal_token_id;
- }
- return 0;
-}
-
-/* Tokenize one embedding input, optionally prepending the model's query
- * instruction. The instruction and input are tokenized in one pass. */
-int tokenize_input(
- loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
- if (!query_instruct)
- return tokenize_one(m, j, text, out);
- size_t plen = strlen(query_instruct);
- size_t tlen = strlen(text);
- char *buf = (char *)malloc(plen + tlen + 1);
- if (!buf) {
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memcpy(buf, query_instruct, plen);
- memcpy(buf + plen, text, tlen + 1);
- int rc = tokenize_one(m, j, buf, out);
- free(buf);
- return rc;
-}
-
-static int late_id_is_skipped(const loaded_model *m, int id) {
- for (int i = 0; i < m->n_late_skip_ids; i++) {
- if (m->late_skip_ids[i] == id)
- return 1;
- }
- return 0;
-}
-
-int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
- token_buf raw = {0};
- if (tokenize_one(m, j, text, &raw) != 0)
- return -1;
-
- int raw_tokens = raw.n_tokens;
- int target = raw_tokens + 1;
- if (is_query && target < EMBED_LATE_QUERY_TOKENS)
- target = EMBED_LATE_QUERY_TOKENS;
-
- out->ids = xmalloc((size_t)target * sizeof(*out->ids));
- out->ids[0] = raw.ids[0];
- out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id;
- if (raw_tokens > 1) {
- memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids));
- }
- out->n_tokens = raw_tokens + 1;
- if (is_query) {
- while (out->n_tokens < EMBED_LATE_QUERY_TOKENS)
- out->ids[out->n_tokens++] = m->late_mask_id;
- out->n_keep = out->n_tokens;
- } else {
- out->keep = xmalloc((size_t)out->n_tokens * sizeof(*out->keep));
- for (int i = 0; i < out->n_tokens; i++) {
- if (!late_id_is_skipped(m, out->ids[i]))
- out->keep[out->n_keep++] = i;
- }
- }
- free(raw.ids);
- return out->n_keep > 0 ? 0 : -1;
-}
-
-int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) {
-#ifdef USE_MLX
- if (m->mlx_ctx)
- return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- return embed_cuda_encode_batch(m->cuda_ctx, inputs, batch, out);
-#endif
- return embed_model_encode_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
-}
-
-int model_embed_spans_batch(loaded_model *m,
- const embed_context_input_t *inputs,
- int batch,
- float *out) {
-#ifdef USE_MLX
- if (m->mlx_ctx)
- return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out);
-#endif
- return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
-}
-
-static int model_uses_dense_batches(const loaded_model *m) {
-#ifdef USE_MLX
- return m->mlx_ctx != NULL;
-#else
- (void)m;
- return 0;
-#endif
-}
-
-/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
- * pads each dense row to the longest input in the chunk. */
-int inference_batch_accepts_input(
- const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
- if (batch == 0)
- return 1;
- if (model_uses_dense_batches(m))
- return next_tokens <= max_batch_tokens / (batch + 1);
- return next_tokens <= max_batch_tokens - packed_tokens;
-}
-
-int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) {
- if (!m || !m->info || !config || config->hidden_size <= 0) {
- if (m && m->info)
- server_log("embed-server: model %s has invalid config", m->info->id);
- return -1;
- }
- if (m->info->kind == MODEL_KIND_LATE) {
- if (token_dim <= 0) {
- server_log("embed-server: late model %s is missing token projection metadata",
- m->info->id);
- return -1;
- }
- } else if (token_dim != 0) {
- server_log("embed-server: non-late model %s unexpectedly has token projection metadata",
- m->info->id);
- return -1;
- }
-
- int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size;
- if (min_dim <= 0 || min_dim > config->hidden_size) {
- server_log("embed-server: model %s has invalid min_dim=%d for dim=%d", m->info->id,
- min_dim, config->hidden_size);
- return -1;
- }
- m->info->dim = config->hidden_size;
- m->info->min_dim = min_dim;
- m->info->token_dim = token_dim;
- m->info->attention_mode = config->attention_mode;
- m->info->pooling_mode = config->pooling_mode;
- m->info->normalize_embeddings = config->normalize_embeddings;
- m->append_terminal_token = config->append_terminal_token;
- /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
- m->renormalize_truncated =
- config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN;
- return 0;
-}
-
-int load_one_model(http_server *s, const embed_server_model_spec_t *spec) {
- /* Grow the model list by one entry. */
- loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models));
- if (!nm)
- return -1;
- s->models = nm;
- loaded_model *m = &s->models[s->n_models];
- memset(m, 0, sizeof(*m));
- s->n_models++;
-
- if (init_model_info(&m->info, spec) != 0)
- return -1;
- const char *path = spec->path;
- m->path = xstrdup(path);
- m->terminal_token_id = -1;
- m->cls_id = -1;
- m->sep_id = -1;
- /* Pick the tokenizer by the files present: WordPiece vocab.txt, then a
- * SentencePiece .model, otherwise the Qwen byte-level BPE vocab.json. */
- char wp_path[1024];
- snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path);
- if (access(wp_path, R_OK) == 0) {
- m->wp_tok = wordpiece_tokenizer_load(path);
- if (!m->wp_tok) {
- server_log("embed-server: failed to load WordPiece tokenizer: %s", wp_path);
- return -1;
- }
- m->wp_tok_ws = wordpiece_workspace_new();
- if (!m->wp_tok_ws) {
- server_log("embed-server: failed to allocate WordPiece workspace: %s", path);
- return -1;
- }
- /* [CLS]/[SEP] wrap every input; resolve their ids once from the vocab. */
- m->cls_id = wordpiece_tokenizer_token_id(m->wp_tok, "[CLS]");
- m->sep_id = wordpiece_tokenizer_token_id(m->wp_tok, "[SEP]");
- if (m->cls_id < 0 || m->sep_id < 0) {
- server_log("embed-server: WordPiece vocab missing [CLS]/[SEP]: %s", path);
- return -1;
- }
- } else {
- char sp_path[1024];
- snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", path);
- if (access(sp_path, R_OK) != 0) {
- snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", path);
- if (access(sp_path, R_OK) != 0)
- snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", path);
- }
- if (access(sp_path, R_OK) == 0) {
- m->sp_tok = sentencepiece_tokenizer_load(path);
- if (!m->sp_tok) {
- server_log("embed-server: failed to load SentencePiece tokenizer: %s", sp_path);
- return -1;
- }
- m->sp_tok_ws = sentencepiece_workspace_new();
- if (!m->sp_tok_ws) {
- server_log("embed-server: failed to allocate SentencePiece workspace: %s", path);
- return -1;
- }
- m->cls_id = sentencepiece_tokenizer_token_id(m->sp_tok, "<s>");
- m->sep_id = sentencepiece_tokenizer_token_id(m->sp_tok, "</s>");
- if (m->cls_id < 0 || m->sep_id < 0) {
- server_log("embed-server: SentencePiece vocab missing <s>/</s>: %s", path);
- return -1;
- }
- }
- }
- if (!m->wp_tok && !m->sp_tok) {
- char vocab_path[1024];
- snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path);
- m->tok = embed_tokenizer_load(vocab_path);
- if (!m->tok) {
- server_log("embed-server: failed to load tokenizer: %s", vocab_path);
- return -1;
- }
- m->tok_ws = embed_tokenizer_workspace_new();
- if (!m->tok_ws) {
- server_log("embed-server: failed to allocate tokenizer workspace: %s", path);
- return -1;
- }
- /* The contextual chunk separator is the tokenizer's <|endoftext|>. The
- * released models keep added special tokens out of vocab.json, so the
- * family constant is the normal case; a vocab that does define the
- * token (e.g. small test models) takes precedence so the id stays
- * inside that model's embedding table. */
- int sep_id = embed_tokenizer_token_id(m->tok, "<|endoftext|>");
- m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
- /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|>
- * suffix - the same token as the separator, not the model's chat
- * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */
- m->terminal_token_id = m->context_separator_id;
- if (m->info->kind == MODEL_KIND_LATE) {
- int id = embed_tokenizer_token_id(m->tok, "[MASK]");
- m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID;
- id = embed_tokenizer_token_id(m->tok, "[Q]");
- m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
- id = embed_tokenizer_token_id(m->tok, "[D]");
- m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
-
- const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
- for (const char *p = punct;
- *p &&
- m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / sizeof(m->late_skip_ids[0]));
- p++) {
- char text[2] = {*p, '\0'};
- int n_ids = 0;
- int *ids = embed_tokenizer_encode(m->tok, text, &n_ids);
- if (ids && n_ids == 1)
- m->late_skip_ids[m->n_late_skip_ids++] = ids[0];
- free(ids);
- }
- }
- }
-#ifdef USE_MLX
- if (s->use_mlx) {
- /* MLX streams are thread-local. The inference worker loads the MLX
- * model so all MLX arrays/streams are created, used, and freed by the
- * same thread. */
- return 0;
- }
-#endif
-#ifdef USE_CUDA
- if (s->use_cuda) {
- return 0;
- }
-#endif
- const embed_config_t *config = NULL;
- int token_dim = 0;
- if (m->info->kind == MODEL_KIND_LATE) {
- m->cpu_late_model = embed_late_model_load(path);
- if (!m->cpu_late_model) {
- server_log("embed-server: failed to load late model: %s", path);
- return -1;
- }
- m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model);
- if (!m->cpu_late_ws) {
- server_log("embed-server: failed to allocate late workspace: %s", path);
- return -1;
- }
- config = embed_late_model_config(m->cpu_late_model);
- token_dim = embed_late_model_token_dim(m->cpu_late_model);
- } else {
- m->cpu_model = embed_model_load(path);
- if (!m->cpu_model) {
- server_log("embed-server: failed to load model: %s", path);
- return -1;
- }
- m->cpu_ws = embed_workspace_new(m->cpu_model);
- if (!m->cpu_ws) {
- server_log("embed-server: failed to allocate workspace: %s", path);
- return -1;
- }
- config = embed_model_config(m->cpu_model);
- }
- if (configure_loaded_model(m, config, token_dim) != 0) {
- return -1;
- }
- if (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size ||
- m->late_document_prefix_id >= config->vocab_size)) {
- server_log("embed-server: late special-token ids exceed model vocab");
- return -1;
- }
- return 0;
-}
-
-void free_models(http_server *s) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- free_model_info(m->info);
- free(m->path);
- if (m->tok_ws)
- embed_tokenizer_workspace_free(m->tok_ws);
- if (m->tok)
- embed_tokenizer_free(m->tok);
- if (m->wp_tok_ws)
- wordpiece_workspace_free(m->wp_tok_ws);
- if (m->wp_tok)
- wordpiece_tokenizer_free(m->wp_tok);
- if (m->sp_tok_ws)
- sentencepiece_workspace_free(m->sp_tok_ws);
- if (m->sp_tok)
- sentencepiece_tokenizer_free(m->sp_tok);
- if (m->cpu_ws)
- embed_workspace_free(m->cpu_ws);
- if (m->cpu_model)
- embed_model_free(m->cpu_model);
- if (m->cpu_late_ws)
- embed_late_workspace_free(m->cpu_late_ws);
- if (m->cpu_late_model)
- embed_late_model_free(m->cpu_late_model);
-#ifdef USE_MLX
- if (m->mlx_ctx)
- embed_mlx_free(m->mlx_ctx);
- if (m->mlx_late_ctx)
- embed_mlx_late_free(m->mlx_late_ctx);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- embed_cuda_free(m->cuda_ctx);
- if (m->cuda_late_ctx)
- embed_cuda_late_free(m->cuda_late_ctx);
-#endif
- }
- free(s->models);
- s->models = NULL;
- s->n_models = 0;
-}
diff --git a/src/tokenizer_bpe.h b/src/tokenizer_bpe.h
deleted file mode 100644
index 8d13d47..0000000
--- a/src/tokenizer_bpe.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * embed_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level)
- *
- * Supports:
- * - decode token ID -> UTF-8 text
- * - encode UTF-8 text -> token IDs (vocab.json + merges.txt)
- */
-
-#ifndef EMBED_TOKENIZER_H
-#define EMBED_TOKENIZER_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef struct {
- char **id_to_text; /* [vocab_size] decoded text strings */
- char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
- int vocab_size;
-
- /* Internal hash maps (opaque to callers) */
- void *vocab_map;
- int vocab_map_cap;
- void *merge_map;
- int merge_map_cap;
- /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id).
- * NULL when the vocab could not resolve every merge; encoding then falls
- * back to the string-based merge loop. */
- void *int_merges;
- int int_merges_cap;
- /* Byte-level unicode codepoint -> single-character token id (512 slots),
- * or NULL when any byte token is missing from the vocab. */
- int *cp_to_id;
-} embed_tokenizer_t;
-
-typedef struct embed_tokenizer_workspace embed_tokenizer_workspace_t;
-
-/* Load tokenizer from vocab.json in model directory */
-EMBED_API embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path);
-
-/* Decode a single token ID to text. Returns pointer to internal string. */
-EMBED_API const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id);
-
-/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>").
- * Returns -1 when the token is not in the vocab. */
-EMBED_API int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token);
-
-/* Encode UTF-8 text into token IDs using BPE.
- * Returns malloc'd array of token IDs and sets *out_n_tokens.
- * Returns NULL on error (and sets *out_n_tokens to 0). */
-EMBED_API int *
-embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens);
-
-/* Reusable scratch for allocation-light repeated encoding. */
-EMBED_API embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void);
-EMBED_API void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws);
-
-/* Encode into caller-provided out_ids[0..out_cap-1].
- * Returns 0 on success, -1 on error, -2 if out_cap is too small.
- * In all cases *out_n_tokens is set to the number of tokens needed/emitted
- * when out_n_tokens is non-NULL. */
-EMBED_API int embed_tokenizer_encode_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as embed_tokenizer_encode(), but reuses ws internally and
- * performs only the final returned int[] allocation. */
-EMBED_API int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-/* Free tokenizer */
-EMBED_API void embed_tokenizer_free(embed_tokenizer_t *tok);
-
-#endif /* EMBED_TOKENIZER_H */
diff --git a/src/tokenizer_sentencepiece.h b/src/tokenizer_sentencepiece.h
deleted file mode 100644
index b4ce48c..0000000
--- a/src/tokenizer_sentencepiece.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * tokenizer_sentencepiece.h - SentencePiece tokenizer for XLM-R style models.
- *
- * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`,
- * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R
- * and multilingual E5. It encodes text only; frontends add the model's special
- * token layout, e.g. <s> ... </s> for XLM-R.
- */
-
-#ifndef SENTENCEPIECE_TOKENIZER_H
-#define SENTENCEPIECE_TOKENIZER_H
-
-#include <stddef.h>
-#include <stdint.h>
-
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef enum {
- SENTENCEPIECE_IDENTITY = 0,
- SENTENCEPIECE_XLM_ROBERTA = 1,
-} sentencepiece_id_map_t;
-
-typedef struct {
- char **id_to_piece; /* [piece_count] raw SentencePiece ids */
- uint16_t *piece_lens; /* byte length for id_to_piece entries */
- float *scores; /* SentencePiece score per raw id */
- uint8_t *types; /* SentencePiece piece type per raw id */
- int piece_count; /* raw SentencePiece vocabulary size */
- int vocab_size; /* model embedding vocab size after id mapping */
- int unk_id; /* model-space unknown-token id */
- int bos_id; /* model-space beginning-token id, or -1 */
- int eos_id; /* model-space end-token id, or -1 */
- int pad_id; /* model-space pad-token id, or -1 */
- int mask_id; /* model-space mask-token id, or -1 */
- int max_piece_len; /* max normal/user/unused piece byte length */
- double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
- int add_dummy_prefix;
- int remove_extra_whitespaces;
- int escape_whitespaces;
- sentencepiece_id_map_t id_map;
-
- void *piece_map; /* piece bytes -> raw SentencePiece id */
- int piece_map_cap;
-
- uint32_t *xcda;
- size_t xcda_count;
- char *prefix_replacements;
- size_t prefix_replacements_len;
-} sentencepiece_tokenizer_t;
-
-typedef struct sentencepiece_workspace sentencepiece_workspace_t;
-
-/* Load the first supported SentencePiece model found in model_dir. Returns NULL
- * on missing model file, unsupported model type, malformed protobuf, or OOM. */
-EMBED_API sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir);
-
-/* Decode one model-space id to an internal token string, or NULL when invalid. */
-EMBED_API const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id);
-
-/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
- * model-space id, or -1 when absent. */
-EMBED_API int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok,
- const char *token);
-
-/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns
- * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an
- * error. */
-EMBED_API int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
- const char *text,
- int *out_n_tokens);
-
-EMBED_API sentencepiece_workspace_t *sentencepiece_workspace_new(void);
-EMBED_API void sentencepiece_workspace_free(sentencepiece_workspace_t *ws);
-
-/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if
- * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */
-EMBED_API int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as sentencepiece_tokenizer_encode(), reusing ws internally
- * and performing only the final returned int[] allocation. */
-EMBED_API int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-EMBED_API void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok);
-
-#endif /* SENTENCEPIECE_TOKENIZER_H */
diff --git a/src/tokenizer_wordpiece.h b/src/tokenizer_wordpiece.h
deleted file mode 100644
index 59cb2bb..0000000
--- a/src/tokenizer_wordpiece.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * wordpiece_tokenizer.h - BERT-family WordPiece tokenizer.
- *
- * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional
- * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting)
- * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes
- * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the
- * byte-level BPE backend leaves terminal/separator tokens to the frontend.
- *
- * The public surface mirrors embed_tokenizer.h (load / encode with a reusable
- * workspace / decode / resolve a named special token / free) so both can sit
- * behind one tokenizer interface later.
- */
-
-#ifndef WORDPIECE_TOKENIZER_H
-#define WORDPIECE_TOKENIZER_H
-
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef struct {
- char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */
- int vocab_size;
- int unk_id; /* [UNK] id, resolved at load */
- int do_lower_case; /* lowercase + strip accents before WordPiece */
- int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */
-
- /* Opaque token-string -> id hash map. */
- void *vocab_map;
- int vocab_map_cap;
-} wordpiece_tokenizer_t;
-
-typedef struct wordpiece_workspace wordpiece_workspace_t;
-
-/* Load from <model_dir>/vocab.txt, reading do_lower_case from
- * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on
- * error (missing vocab.txt, missing [UNK], allocation failure). */
-EMBED_API wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir);
-
-/* Decode a single id to its vocab token string (internal pointer), or NULL. */
-EMBED_API const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id);
-
-/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */
-EMBED_API int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token);
-
-/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on
- * error (and *out_n_tokens = 0). */
-EMBED_API int *
-wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n_tokens);
-
-/* Reusable scratch for allocation-light repeated encoding. */
-EMBED_API wordpiece_workspace_t *wordpiece_workspace_new(void);
-EMBED_API void wordpiece_workspace_free(wordpiece_workspace_t *ws);
-
-/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1
- * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to
- * the number of tokens needed/emitted in all cases. */
-EMBED_API int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as wordpiece_tokenizer_encode(), reusing ws internally and
- * performing only the final returned int[] allocation. */
-EMBED_API int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-EMBED_API void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok);
-
-#endif /* WORDPIECE_TOKENIZER_H */
diff --git a/tools/cli/Makefile b/tools/cli/Makefile
new file mode 100644
index 0000000..cc076f1
--- /dev/null
+++ b/tools/cli/Makefile
@@ -0,0 +1,22 @@
+ROOT := ../..
+COMPONENT := cli
+include $(ROOT)/build.mk
+
+.DEFAULT_GOAL := all
+
+LIBFFWD := $(ROOT)/libffwd
+
+BIN := $(OUTDIR)/ffwd-cli
+SRCS := main.c
+LIBS := $(OUTDIR)/libffwd.a
+LDLIBS := $(BACKEND_LDFLAGS) -lm
+
+CFLAGS += -std=gnu11 -pthread
+CPPFLAGS += -I. -I$(LIBFFWD)
+
+$(OBJDIR)/main.o: CFLAGS += $(VERSION_CFLAGS)
+
+$(OUTDIR)/libffwd.a:
+ $(MAKE) -C $(LIBFFWD)
+
+$(eval $(program))
diff --git a/tools/cli/main.c b/tools/cli/main.c
new file mode 100644
index 0000000..d9516b8
--- /dev/null
+++ b/tools/cli/main.c
@@ -0,0 +1,556 @@
+/* main.c - ffwd-cli command-line tool */
+
+#include "ffwd.h"
+#include "internal.h"
+#include "build.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <limits.h>
+#include <unistd.h>
+
+static double now_ms(void) {
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
+}
+
+static void print_usage(const char *prog) {
+ fprintf(stderr,
+ "Usage: %s -d <model_dir> [options] [text...]\n"
+ "\n"
+ "Options:\n"
+ " -d <dir> Model directory (required)\n"
+ " --stream Read lines from stdin, write one JSON embedding per line\n"
+ " --json Emit JSON instead of plain text (embedding or matrix)\n"
+ " -b, --batch-size N\n"
+ " Max texts per inference batch (default: all; --stream: 1)\n",
+ prog);
+ ffwd_cli_help(stderr);
+ fprintf(stderr,
+ " -e, --embeddings\n"
+ " Print raw embeddings (with multiple texts)\n"
+ " -v, --verbose\n"
+ " Verbose (-vv for debug)\n"
+ " -V, --version\n"
+ " Print version and exit\n"
+ " --build-info Print build details and exit\n"
+ " -h Show this help\n"
+ "\n"
+ "Modes:\n"
+ " 1 text arg Embedding as space-separated floats (JSON with --json)\n"
+ " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n"
+ " no args Batch: read all stdin lines, then similarity matrix\n"
+ " --stream Streaming: read stdin lines, write JSON per line\n"
+ "\n"
+ "Examples:\n"
+ " %s -d ./model \"what is AI?\"\n"
+ " %s -d ./model --stream < texts.txt\n",
+ prog, prog);
+}
+
+/* ========================================================================
+ * Embed one text: tokenize > forward > return float[dim]
+ * ======================================================================== */
+
+typedef struct {
+ ffwd_t *backend;
+ ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */
+ int dim;
+} runner_t;
+
+static void runner_free_tokenizers(runner_t *e) { ffwd_tok_free(e->tok); }
+
+typedef struct {
+ int *ids;
+ int n_tokens;
+} token_buf_t;
+
+static int
+runner_ffwd_batch(runner_t *e, const ffwd_input_t *inputs, int batch, float *out_embeddings) {
+ return ffwd_encode_batch(e->backend, inputs, batch, out_embeddings);
+}
+
+static int tokenize_text(runner_t *e, const char *text, token_buf_t *out) {
+ memset(out, 0, sizeof(*out));
+ out->ids = ffwd_tokenize(e->tok, text, NULL, &out->n_tokens);
+ if (!out->ids) {
+ fprintf(stderr, "tokenization failed: %s\n", text);
+ return -1;
+ }
+
+ if (ffwd_verbose >= 1) {
+ fprintf(stderr, "tokens (%d): ", out->n_tokens);
+ for (int i = 0; i < out->n_tokens && i < 20; i++)
+ fprintf(stderr, "%d ", out->ids[i]);
+ if (out->n_tokens > 20)
+ fprintf(stderr, "...");
+ fprintf(stderr, "\n");
+ }
+ return 0;
+}
+
+static void free_tokens(token_buf_t *tokens, int n) {
+ if (!tokens)
+ return;
+ for (int i = 0; i < n; i++)
+ free(tokens[i].ids);
+}
+
+/* ========================================================================
+ * Output helpers
+ * ======================================================================== */
+
+static void print_embedding_raw(const float *emb, int dim) {
+ for (int i = 0; i < dim; i++) {
+ if (i > 0)
+ putchar(' ');
+ printf("%.8f", (double)emb[i]);
+ }
+ putchar('\n');
+}
+
+/* Single embedding as a JSON array of floats. */
+static void print_embedding_array(const float *emb, int dim) {
+ putchar('[');
+ for (int i = 0; i < dim; i++) {
+ if (i)
+ putchar(',');
+ printf("%.8f", (double)emb[i]);
+ }
+ puts("]");
+}
+
+static size_t runner_workspace_nbytes(const runner_t *e) {
+ return e ? ffwd_scratch_nbytes(e->backend) : 0;
+}
+
+static void
+print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) {
+ printf("{\"embedding\":[");
+ for (int i = 0; i < dim; i++) {
+ if (i > 0)
+ putchar(',');
+ printf("%.8f", (double)emb[i]);
+ }
+ printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms,
+ workspace_bytes);
+ fflush(stdout);
+}
+
+/* ========================================================================
+ * Stdin mode: read lines from stdin, write JSON to stdout
+ * ======================================================================== */
+
+static int process_stdin_batch(runner_t *e, char **lines, int n_lines) {
+ if (n_lines <= 0)
+ return 0;
+
+ token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
+ ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_lines * sizeof(ffwd_input_t));
+ int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int));
+ float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
+ if (!tokens || !inputs || !input_to_line || !embs) {
+ fprintf(stderr, "OOM\n");
+ free(tokens);
+ free(inputs);
+ free(input_to_line);
+ free(embs);
+ for (int i = 0; i < n_lines; i++)
+ printf("{\"error\":\"embedding failed\"}\n");
+ fflush(stdout);
+ return 1;
+ }
+
+ int n_inputs = 0;
+ for (int i = 0; i < n_lines; i++) {
+ if (tokenize_text(e, lines[i], &tokens[i]) == 0) {
+ inputs[n_inputs].ids = tokens[i].ids;
+ inputs[n_inputs].n_tokens = tokens[i].n_tokens;
+ input_to_line[n_inputs] = i;
+ n_inputs++;
+ }
+ }
+
+ double ms = 0.0;
+ int rc = 0;
+ if (n_inputs > 0) {
+ double t0 = now_ms();
+ rc = runner_ffwd_batch(e, inputs, n_inputs, embs);
+ ms = now_ms() - t0;
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms);
+ }
+
+ int next_valid = 0;
+ for (int i = 0; i < n_lines; i++) {
+ if (!tokens[i].ids) {
+ printf("{\"error\":\"tokenization failed\"}\n");
+ continue;
+ }
+ if (rc != 0) {
+ printf("{\"error\":\"embedding failed\"}\n");
+ continue;
+ }
+ while (next_valid < n_inputs && input_to_line[next_valid] != i)
+ next_valid++;
+ if (next_valid >= n_inputs) {
+ printf("{\"error\":\"embedding failed\"}\n");
+ continue;
+ }
+ print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms,
+ runner_workspace_nbytes(e));
+ next_valid++;
+ }
+ fflush(stdout);
+
+ free_tokens(tokens, n_lines);
+ free(tokens);
+ free(inputs);
+ free(input_to_line);
+ free(embs);
+ return rc == 0 ? 0 : 1;
+}
+
+static int run_stdin(runner_t *e, int batch_size) {
+ char line[65536];
+ if (batch_size <= 0)
+ batch_size = 1;
+
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size);
+
+ char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
+ if (!batch_lines) {
+ fprintf(stderr, "OOM\n");
+ return 1;
+ }
+ int n_batch = 0;
+ int rc = 0;
+
+ while (fgets(line, sizeof(line), stdin)) {
+ /* Strip newline */
+ size_t len = strlen(line);
+ while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
+ line[--len] = '\0';
+ if (len == 0)
+ continue;
+
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line,
+ len > 60 ? "..." : "");
+
+ batch_lines[n_batch] = strdup(line);
+ if (!batch_lines[n_batch]) {
+ fprintf(stderr, "OOM\n");
+ rc = 1;
+ break;
+ }
+ n_batch++;
+
+ if (n_batch == batch_size) {
+ if (process_stdin_batch(e, batch_lines, n_batch) != 0)
+ rc = 1;
+ for (int i = 0; i < n_batch; i++) {
+ free(batch_lines[i]);
+ batch_lines[i] = NULL;
+ }
+ n_batch = 0;
+ }
+ }
+
+ if (n_batch > 0) {
+ if (process_stdin_batch(e, batch_lines, n_batch) != 0)
+ rc = 1;
+ for (int i = 0; i < n_batch; i++)
+ free(batch_lines[i]);
+ }
+ free(batch_lines);
+
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "stdin: EOF\n");
+
+ return rc;
+}
+
+/* ========================================================================
+ * Batch mode: embed args or stdin lines, then print similarity or vectors
+ * ======================================================================== */
+
+static int append_text(char ***texts, int *n_texts, int *cap, const char *s) {
+ if (*n_texts == *cap) {
+ int new_cap = *cap ? *cap * 2 : 8;
+ char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
+ if (!new_texts)
+ return -1;
+ *texts = new_texts;
+ *cap = new_cap;
+ }
+
+ (*texts)[*n_texts] = strdup(s);
+ if (!(*texts)[*n_texts])
+ return -1;
+ (*n_texts)++;
+ return 0;
+}
+
+/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one
+ * row of space-separated values per line. With print_embs the raw embeddings
+ * follow, one per line after a blank line (plain-text output only). */
+static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) {
+ if (json) {
+ putchar('[');
+ for (int i = 0; i < n; i++) {
+ if (i)
+ putchar(',');
+ putchar('[');
+ for (int j = 0; j < n; j++) {
+ if (j)
+ putchar(',');
+ printf("%.6f", (double)ffwd_cosine_similarity(embs + (size_t)i * dim,
+ embs + (size_t)j * dim, dim));
+ }
+ putchar(']');
+ }
+ puts("]");
+ return;
+ }
+
+ for (int i = 0; i < n; i++) {
+ for (int j = 0; j < n; j++) {
+ if (j)
+ putchar(' ');
+ printf("%.6f", (double)ffwd_cosine_similarity(embs + (size_t)i * dim,
+ embs + (size_t)j * dim, dim));
+ }
+ putchar('\n');
+ }
+ if (print_embs) {
+ putchar('\n');
+ for (int i = 0; i < n; i++)
+ print_embedding_raw(embs + (size_t)i * dim, dim);
+ }
+}
+
+static int run_batch(
+ runner_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) {
+ int n_texts = 0, cap = 0;
+ char **texts = NULL;
+ int rc = 0;
+
+ if (arg_start < argc) {
+ for (int i = arg_start; i < argc; i++) {
+ if (append_text(&texts, &n_texts, &cap, argv[i]) != 0)
+ goto oom_texts;
+ }
+ } else {
+ char line[65536];
+ while (fgets(line, sizeof(line), stdin)) {
+ size_t l = strlen(line);
+ while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r'))
+ line[--l] = '\0';
+ if (l == 0)
+ continue;
+ if (append_text(&texts, &n_texts, &cap, line) != 0)
+ goto oom_texts;
+ }
+ }
+
+ if (n_texts == 0) {
+ fprintf(stderr, "no input texts\n");
+ free(texts);
+ return 1;
+ }
+
+ int dim = e->dim;
+ token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t));
+ ffwd_input_t *inputs = (ffwd_input_t *)malloc((size_t)n_texts * sizeof(ffwd_input_t));
+ float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
+ if (!tokens || !inputs || !embs) {
+ fprintf(stderr, "OOM\n");
+ free(tokens);
+ free(inputs);
+ free(embs);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
+ free(texts);
+ return 1;
+ }
+
+ for (int i = 0; i < n_texts; i++) {
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]);
+ if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
+ rc = 1;
+ goto done;
+ }
+ inputs[i].ids = tokens[i].ids;
+ inputs[i].n_tokens = tokens[i].n_tokens;
+ }
+
+ int max_batch = batch_size > 0 ? batch_size : n_texts;
+ for (int start = 0; start < n_texts; start += max_batch) {
+ int cur = n_texts - start;
+ if (cur > max_batch)
+ cur = max_batch;
+
+ double t0 = now_ms();
+ if (runner_ffwd_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) {
+ fprintf(stderr, "forward pass failed\n");
+ rc = 1;
+ goto done;
+ }
+ if (ffwd_verbose >= 1)
+ fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1,
+ cur, now_ms() - t0);
+ }
+
+ if (n_texts == 1) {
+ if (json)
+ print_embedding_array(embs, dim);
+ else
+ print_embedding_raw(embs, dim);
+ } else {
+ print_matrix(embs, n_texts, dim, json, print_embs);
+ }
+
+done:
+ free_tokens(tokens, n_texts);
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
+ free(texts);
+ free(tokens);
+ free(inputs);
+ free(embs);
+ return rc;
+
+oom_texts:
+ fprintf(stderr, "OOM\n");
+ for (int i = 0; i < n_texts; i++)
+ free(texts[i]);
+ free(texts);
+ return 1;
+}
+
+/* ========================================================================
+ * Main
+ * ======================================================================== */
+
+int main(int argc, char *argv[]) {
+ const char *model_dir = NULL;
+ int print_embs = 0;
+ int verbose = 0;
+ int stdin_mode = 0;
+ int batch_size = 0;
+ int json = 0;
+ const char *prog = ffwd_prog_name(argv[0]);
+ /* All backend tuning flags parse unconditionally into opts; each backend
+ * uses the fields it understands and ignores the rest. */
+ ffwd_options_t opts = {.gpu_quant_group_size = 64};
+
+ int arg_start = 1;
+ while (arg_start < argc && argv[arg_start][0] == '-') {
+ const char *f = argv[arg_start];
+ if (!strcmp(f, "-d")) {
+ model_dir = argv[++arg_start];
+ } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
+ ffwd_print_version(prog);
+ return 0;
+ } else if (!strcmp(f, "--build-info")) {
+ ffwd_print_build_info(prog);
+ return 0;
+ } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ opts.n_threads = atoi(argv[++arg_start]);
+ } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ batch_size = atoi(argv[++arg_start]);
+ } else if (!strcmp(f, "--json")) {
+ json = 1;
+ } else if (!strcmp(f, "--gpu-quant-bits")) {
+ opts.gpu_quant_bits = atoi(argv[++arg_start]);
+ } else if (!strcmp(f, "--gpu-quant-group-size")) {
+ opts.gpu_quant_group_size = atoi(argv[++arg_start]);
+ } else if (!strcmp(f, "--gpu-gemm-mode")) {
+ opts.gpu_gemm_mode = argv[++arg_start];
+ } else if (!strcmp(f, "--gpu-weight-dtype")) {
+ opts.gpu_weight_dtype = argv[++arg_start];
+ } else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
+ print_embs = 1;
+ } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
+ verbose++;
+ } else if (!strcmp(f, "-vv")) {
+ verbose = 2;
+ } else if (!strcmp(f, "--stream")) {
+ stdin_mode = 1;
+ } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ print_usage(prog);
+ return 0;
+ } else if (!strcmp(f, "--")) {
+ /* End of options: everything after is positional text, even if it
+ * starts with a dash. */
+ arg_start++;
+ break;
+ } else if (f[0] == '-' && f[1] == '-') {
+ /* An unrecognized --option is a hard error, not silently-embedded
+ * text. Catches typos and stale flags (e.g. the removed --backend)
+ * loudly instead of running the default path or embedding the flag. */
+ fprintf(stderr, "unknown option: %s\n", f);
+ print_usage(prog);
+ return 1;
+ } else
+ break;
+ arg_start++;
+ }
+
+ ffwd_verbose = verbose;
+
+ if (!model_dir || !model_dir[0]) {
+ fprintf(stderr, "model directory required (-d <dir>)\n");
+ print_usage(prog);
+ return 1;
+ }
+
+ char err[256];
+ if (ffwd_init(&opts, err, sizeof(err)) != 0) {
+ fprintf(stderr, "%s\n", err);
+ return 1;
+ }
+
+ /* Tokenizer (file probe + special-token layout) is owned by libffwd */
+ runner_t e = {0};
+ double t0 = now_ms();
+ e.tok = ffwd_tok_open(model_dir, 0, err, sizeof(err));
+ if (!e.tok) {
+ fprintf(stderr, "%s\n", err);
+ return 1;
+ }
+ if (verbose >= 1)
+ fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
+
+ /* Model */
+ t0 = now_ms();
+ e.backend = ffwd_open(model_dir, 0, &opts, err, sizeof(err));
+ if (!e.backend || ffwd_activate(e.backend, err, sizeof(err)) != 0) {
+ fprintf(stderr, "%s\n", err);
+ ffwd_free(e.backend);
+ runner_free_tokenizers(&e);
+ return 1;
+ }
+ e.dim = ffwd_config(e.backend)->hidden_size;
+ ffwd_tok_set_append_terminal(e.tok, ffwd_config(e.backend)->append_terminal_token);
+ if (verbose >= 1)
+ fprintf(stderr, "Model: %d-dim, %.0f ms\n", e.dim, now_ms() - t0);
+
+ /* Run */
+ int rc;
+ if (stdin_mode)
+ rc = run_stdin(&e, batch_size > 0 ? batch_size : 1);
+ else
+ rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json);
+
+ /* Cleanup */
+ ffwd_free(e.backend);
+ runner_free_tokenizers(&e);
+ return rc;
+}
diff --git a/tools/server/Makefile b/tools/server/Makefile
new file mode 100644
index 0000000..4a11b0d
--- /dev/null
+++ b/tools/server/Makefile
@@ -0,0 +1,49 @@
+ROOT := ../..
+COMPONENT := server
+include $(ROOT)/build.mk
+
+LIBFFWD := $(ROOT)/libffwd
+AE := $(ROOT)/deps/ae
+
+BIN := $(OUTDIR)/ffwd-server
+SRCS := base64.c encode.c handlers.c http.c json.c main.c models.c sbuf.c \
+ schedule.c server.c util.c
+OBJS := $(call objs,$(SRCS))
+DEPS := $(call deps,$(OBJS))
+
+TEST_BINS := test_base64 test_sbuf test_encode test_json test_schedule test_models
+TEST_OBJS := $(filter-out $(OBJDIR)/main.o,$(OBJS))
+
+LIBS := $(OUTDIR)/libffwd.a $(OUTDIR)/libae.a
+LDLIBS := $(CJSON_LDFLAGS) $(BACKEND_LDFLAGS) -lm
+
+CFLAGS += -std=gnu11 -pthread
+CPPFLAGS += -I. -I$(LIBFFWD) -I$(AE) $(CJSON_CFLAGS)
+
+$(OBJDIR)/main.o: CFLAGS += $(VERSION_CFLAGS)
+
+.PHONY: all clean test
+
+all: $(BIN)
+
+$(eval $(call link_program,$(BIN),$(OBJS),$(LIBS),$(LDLIBS)))
+
+test: $(TEST_BINS)
+ @rc=0; for t in $(TEST_BINS); do ./$$t || rc=1; done; \
+ if [ $$rc -eq 0 ]; then echo "server ok"; else echo "server not ok"; exit 1; fi
+
+$(TEST_BINS): %: %.c test_util.h $(TEST_OBJS) $(LIBS)
+ $(CC) $(CFLAGS) $(CPPFLAGS) $< $(TEST_OBJS) $(LIBS) $(LDLIBS) $(LDFLAGS_OPT) -o $@
+
+$(OUTDIR)/libffwd.a:
+ $(MAKE) -C $(LIBFFWD)
+
+$(OUTDIR)/libae.a:
+ $(MAKE) -C $(AE)
+
+clean:
+ $(RM) -r $(OBJDIR) $(BIN) $(BIN).dSYM \
+ $(TEST_BINS) $(addsuffix .dSYM,$(TEST_BINS))
+ $(MAKE) -C $(AE) clean
+
+-include $(DEPS)
diff --git a/src/base64.c b/tools/server/base64.c
index 082101f..87351d2 100644
--- a/src/base64.c
+++ b/tools/server/base64.c
@@ -1,5 +1,5 @@
#include "base64.h"
-#include "server_util.h"
+#include "util.h"
#include <stdint.h>
diff --git a/src/base64.h b/tools/server/base64.h
index 17e2036..88d88b3 100644
--- a/src/base64.h
+++ b/tools/server/base64.h
@@ -1,5 +1,5 @@
-#ifndef EMBED_BASE64_H
-#define EMBED_BASE64_H
+#ifndef FFWD_BASE64_H
+#define FFWD_BASE64_H
#include <stddef.h>
@@ -7,4 +7,4 @@
* string the caller frees; die-on-OOM, so it never returns NULL. */
char *base64_encode(const unsigned char *src, size_t n);
-#endif /* EMBED_BASE64_H */
+#endif /* FFWD_BASE64_H */
diff --git a/src/server_encode.c b/tools/server/encode.c
index ef4e3ec..faa58a0 100644
--- a/src/server_encode.c
+++ b/tools/server/encode.c
@@ -1,8 +1,8 @@
#include "server_internal.h"
-#include "server_util.h"
+#include "util.h"
#include "sbuf.h"
#include "base64.h"
-#include "embed.h"
+#include "ffwd.h"
#include <math.h>
#include <stdlib.h>
@@ -73,7 +73,7 @@ void append_embedding_value(
if (i)
sbuf_putc(b, ',');
float v =
- api == EMBED_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i];
+ api == FFWD_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i];
sbuf_printf(b, "%.9g", (double)v);
}
sbuf_puts(b, "]}");
@@ -101,7 +101,7 @@ int render_embedding_response(embedding_request *r, const float *embs) {
for (int i = 0; i < r->n_inputs; i++) {
float *dst = truncated + (size_t)i * r->dims;
memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst));
- if (embed_l2_normalize(dst, r->dims) != 0) {
+ if (ffwd_l2_normalize(dst, r->dims) != 0) {
free(truncated);
return -1;
}
diff --git a/src/server_handlers.c b/tools/server/handlers.c
index b3476d9..6720784 100644
--- a/src/server_handlers.c
+++ b/tools/server/handlers.c
@@ -1,6 +1,6 @@
#include "server_internal.h"
-#include "server_util.h"
-#include "embed.h"
+#include "util.h"
+#include "ffwd.h"
#include <limits.h>
#include <stdio.h>
@@ -11,10 +11,10 @@
* Endpoint handlers: prepare, batch, execute, render
*
* One prepare_* per endpoint validates the request JSON and tokenizes its
- * inputs into a *_request; the worker (server_schedule.c) groups compatible
+ * inputs into a *_request; the worker (schedule.c) groups compatible
* ready requests and calls the matching execute_*_request_list, which runs the
- * batched forward pass through model_embed_* and renders each response via
- * server_encode.c. Rerank runs MaxSim over late-interaction token vectors.
+ * batched forward pass through model_ffwd_* and renders each response via
+ * encode.c. Rerank runs MaxSim over late-interaction token vectors.
* ======================================================================== */
int embedding_request_compatible(const embedding_request *a, const embedding_request *b) {
@@ -23,7 +23,7 @@ int embedding_request_compatible(const embedding_request *a, const embedding_req
}
typedef struct {
- embed_input_t input;
+ ffwd_input_t input;
int output_index;
} embedding_batch_item;
@@ -74,7 +74,7 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
if (max_batch > total_inputs)
max_batch = total_inputs;
int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
- embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
+ ffwd_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float));
int failed = 0;
@@ -91,9 +91,9 @@ void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
cur++;
}
uint64_t t0 = nstime();
- int embed_rc = model_embed_batch(m, inputs, cur, batch_embs);
+ int ffwd_rc = model_ffwd_batch(m, inputs, cur, batch_embs);
infer_ns += nstime() - t0;
- if (embed_rc != 0) {
+ if (ffwd_rc != 0) {
failed = 1;
break;
}
@@ -135,12 +135,11 @@ int contextual_request_fits_group(const contextual_request *r,
int max_batch_tokens) {
if (group_docs == 0)
return 1;
- return r->n_docs <= max_batch - group_docs &&
- r->total_tokens <= max_batch_tokens - group_tokens;
+ return r->n_docs <= max_batch - group_docs && r->total_tokens <= max_batch_tokens - group_tokens;
}
typedef struct {
- embed_context_input_t input;
+ ffwd_context_input_t input;
int output_span_index;
int order;
} contextual_batch_item;
@@ -201,7 +200,7 @@ void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
if (max_batch > total_docs)
max_batch = total_docs;
int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
- embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
+ ffwd_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
float *batch_embs = xmalloc(sizeof(*batch_embs));
size_t batch_emb_cap = 0;
@@ -212,7 +211,7 @@ void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
int tokens = 0;
int chunks = 0;
while (start + cur < total_docs && cur < max_batch) {
- const embed_context_input_t *input = &items[start + cur].input;
+ const ffwd_context_input_t *input = &items[start + cur].input;
if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens,
max_batch_tokens))
break;
@@ -227,9 +226,9 @@ void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
batch_emb_cap = needed;
}
uint64_t t0 = nstime();
- int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs);
+ int ffwd_rc = model_ffwd_spans_batch(m, inputs, cur, batch_embs);
infer_ns += nstime() - t0;
- if (embed_rc != 0) {
+ if (ffwd_rc != 0) {
failed = 1;
break;
}
@@ -283,13 +282,12 @@ static int validate_common(cJSON *root,
ve_add(detail, "[\"body\",\"model\"]",
"value is not a valid enum member for this endpoint", "enum");
} else if (m->info->kind != expected_kind) {
- ve_add(detail, "[\"body\",\"model\"]",
- "model is not available on this endpoint", "enum");
+ ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum");
m = NULL;
}
}
*out_model = m;
- const char *enc = encoding_from_root(root, detail, m ? m->info->api : EMBED_API_OPENAI);
+ const char *enc = encoding_from_root(root, detail, m ? m->info->api : FFWD_API_OPENAI);
int min_dim = m ? m->info->min_dim : 128;
int max_dim = m ? m->info->dim : 2560;
int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc);
@@ -330,7 +328,7 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re
if (n_inputs < 1)
ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item",
"value_error.list.min_items");
- else if (n_inputs > EMBED_API_MAX_STANDARD_INPUTS)
+ else if (n_inputs > FFWD_API_MAX_STANDARD_INPUTS)
ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items",
"value_error.list.max_items");
cJSON *item;
@@ -364,7 +362,7 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re
out->inputs[0].ids = out->tokens[0].ids;
out->inputs[0].n_tokens = out->tokens[0].n_tokens;
out->total_tokens = out->tokens[0].n_tokens;
- if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
+ if (out->tokens[0].n_tokens > FFWD_API_MAX_ITEM_TOKENS)
ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit",
"value_error.context_length");
}
@@ -378,14 +376,13 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re
idx++;
continue;
}
- if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) !=
- 0) {
+ if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) != 0) {
ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
} else {
out->inputs[idx].ids = out->tokens[idx].ids;
out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
out->total_tokens += out->tokens[idx].n_tokens;
- if (out->tokens[idx].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
+ if (out->tokens[idx].n_tokens > FFWD_API_MAX_ITEM_TOKENS)
ve_add(detail, loc, "input exceeds 32768 token limit",
"value_error.context_length");
}
@@ -393,7 +390,7 @@ void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_re
}
}
- if (out->total_tokens > EMBED_API_MAX_TOTAL_TOKENS)
+ if (out->total_tokens > FFWD_API_MAX_TOTAL_TOKENS)
ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
"value_error.context_length");
}
@@ -435,7 +432,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
if (n_docs < 1)
ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document",
"value_error.list.min_items");
- else if (n_docs > EMBED_API_MAX_CONTEXT_DOCS)
+ else if (n_docs > FFWD_API_MAX_CONTEXT_DOCS)
ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents",
"value_error.list.max_items");
cJSON *doc_arr;
@@ -449,8 +446,8 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
continue;
}
int doc_chunks = cJSON_GetArraySize(doc_arr);
- if (doc_chunks > EMBED_API_MAX_CONTEXT_CHUNKS - total_chunks)
- total_chunks = EMBED_API_MAX_CONTEXT_CHUNKS + 1;
+ if (doc_chunks > FFWD_API_MAX_CONTEXT_CHUNKS - total_chunks)
+ total_chunks = FFWD_API_MAX_CONTEXT_CHUNKS + 1;
else
total_chunks += doc_chunks;
if (doc_chunks < 1) {
@@ -473,7 +470,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
}
di++;
}
- if (total_chunks > EMBED_API_MAX_CONTEXT_CHUNKS)
+ if (total_chunks > FFWD_API_MAX_CONTEXT_CHUNKS)
ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks",
"value_error.list.max_items");
}
@@ -507,7 +504,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
ci++;
}
- if (doc_tokens > EMBED_API_MAX_ITEM_TOKENS) {
+ if (doc_tokens > FFWD_API_MAX_ITEM_TOKENS) {
char loc[64];
snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
ve_add(detail, loc, "document exceeds 32768 token limit",
@@ -519,6 +516,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
if (doc_valid) {
doc->n_tokens = (int)doc_tokens;
doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids));
+ int sep_id = ffwd_tok_context_separator_id(m->tok);
int pos = 0;
for (ci = 0; ci < n_chunks; ci++) {
doc->spans[ci].start = pos;
@@ -531,7 +529,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
(size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids));
pos += chunk_tokens[ci].n_tokens;
if (ci + 1 < n_chunks) {
- doc->ids[pos++] = m->context_separator_id;
+ doc->ids[pos++] = sep_id;
}
}
}
@@ -539,7 +537,7 @@ void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_
free(chunk_tokens);
di++;
}
- if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) {
+ if (request_tokens > FFWD_API_MAX_TOTAL_TOKENS) {
ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
"value_error.context_length");
} else {
@@ -617,7 +615,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request
if (n_documents < 1)
ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item",
"value_error.list.min_items");
- else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS)
+ else if (n_documents > FFWD_API_MAX_RERANK_DOCUMENTS)
ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items",
"value_error.list.max_items");
cJSON *document;
@@ -665,11 +663,10 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request
out->n_documents = n_documents;
out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) {
- ve_add(detail, "[\"body\",\"query\"]", "tokenization failed",
- "value_error.tokenization");
+ ve_add(detail, "[\"body\",\"query\"]", "tokenization failed", "value_error.tokenization");
} else {
out->query_tokens = out->query.n_tokens;
- if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS)
+ if (out->query_tokens > FFWD_API_MAX_ITEM_TOKENS)
ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit",
"value_error.context_length");
}
@@ -684,7 +681,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request
ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
} else {
int n = out->documents[i].n_tokens;
- if (n > EMBED_API_MAX_ITEM_TOKENS)
+ if (n > FFWD_API_MAX_ITEM_TOKENS)
ve_add(detail, loc, "document exceeds 32768 token limit",
"value_error.context_length");
if (out->document_tokens <= INT_MAX - n)
@@ -694,7 +691,7 @@ void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request
}
i++;
}
- if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) {
+ if (out->query_tokens > FFWD_API_MAX_TOTAL_TOKENS - out->document_tokens) {
ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit",
"value_error.context_length");
}
@@ -725,101 +722,17 @@ static int rerank_result_cmp(const void *a, const void *b) {
return ra->index - rb->index;
}
-static int execute_rerank_cpu(rerank_request *r, float *scores) {
- loaded_model *m = r->model;
- int dim = m->info->token_dim;
- int total_doc_vecs = 0;
- for (int i = 0; i < r->n_documents; i++)
- total_doc_vecs += r->documents[i].n_keep;
-
- float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
- float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
- int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
-
- /* Encode every candidate in one block-diagonal forward. The encoder packs
- * each document's kept token vectors back-to-back and fills offsets, so the
- * cost of N short documents collapses to a single packed forward. */
- const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
- const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
- int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
- int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
- for (int i = 0; i < r->n_documents; i++) {
- doc_ids[i] = r->documents[i].ids;
- keep[i] = r->documents[i].keep;
- n_tokens[i] = r->documents[i].n_tokens;
- n_keep[i] = r->documents[i].n_keep;
- }
-
- int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids,
- r->query.n_tokens, 1, query);
- if (rc == 0)
- rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens,
- keep, n_keep, r->n_documents, 1, docs, offsets);
- if (rc == 0)
- rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
- scores);
- free(n_keep);
- free(n_tokens);
- free(keep);
- free(doc_ids);
- free(offsets);
- free(docs);
- free(query);
- return rc;
-}
-
-#ifdef USE_MLX
-static int execute_rerank_mlx(rerank_request *r, float *scores) {
- loaded_model *m = r->model;
- embed_mlx_late_vectors_t *query =
- embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1);
-
- /* Encode every candidate in one padded transformer pass. The encoder packs
- * each document's kept token vectors back-to-back and fills offsets, so the
- * cost of N short documents collapses to a single batched forward. */
- const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
- const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
- int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
- int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
- int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
- for (int i = 0; i < r->n_documents; i++) {
- doc_ids[i] = r->documents[i].ids;
- keep[i] = r->documents[i].keep;
- n_tokens[i] = r->documents[i].n_tokens;
- n_keep[i] = r->documents[i].n_keep;
+void execute_rerank_request(rerank_request *r) {
+ if (!r || !r->model) {
+ if (r && r->j)
+ job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error");
+ return;
}
- embed_mlx_late_vectors_t *packed =
- query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep,
- r->n_documents, 1, offsets)
- : NULL;
- int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets,
- r->n_documents, scores)
- : -1;
- embed_mlx_late_vectors_free(packed);
- embed_mlx_late_vectors_free(query);
- free(offsets);
- free(n_keep);
- free(n_tokens);
- free(keep);
- free(doc_ids);
- return rc;
-}
-#endif
-
-#ifdef USE_CUDA
-static int execute_rerank_cuda(rerank_request *r, float *scores) {
- loaded_model *m = r->model;
- int dim = m->info->token_dim;
- int total_doc_vecs = 0;
- for (int i = 0; i < r->n_documents; i++)
- total_doc_vecs += r->documents[i].n_keep;
-
- float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
- float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
- int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
+ float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
- /* GPU encodes the query and every candidate (one packed forward); MaxSim
- * runs on the host, where the grouped-GEMM scorer beats a device graph. */
+ /* Flatten the per-document fields into the arrays the backend rerank takes;
+ * it encodes the query and every candidate in one packed forward and scores
+ * MaxSim, so N short documents collapse to a single batch. */
const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
@@ -830,43 +743,23 @@ static int execute_rerank_cuda(rerank_request *r, float *scores) {
n_tokens[i] = r->documents[i].n_tokens;
n_keep[i] = r->documents[i].n_keep;
}
-
- int rc =
- embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query);
- if (rc == 0)
- rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep,
- r->n_documents, 1, docs, offsets);
- if (rc == 0)
- rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
- scores);
+ ffwd_rerank_input_t in = {
+ .query_ids = r->query.ids,
+ .query_n_tokens = r->query.n_tokens,
+ .query_n_keep = r->query.n_keep,
+ .doc_ids = doc_ids,
+ .doc_n_tokens = n_tokens,
+ .doc_keep = keep,
+ .doc_n_keep = n_keep,
+ .n_docs = r->n_documents,
+ };
+ uint64_t t0 = nstime();
+ int rc = ffwd_rerank(r->model->backend, &in, scores);
+ r->j->infer_ns += nstime() - t0;
free(n_keep);
free(n_tokens);
free(keep);
free(doc_ids);
- free(offsets);
- free(docs);
- free(query);
- return rc;
-}
-#endif
-
-void execute_rerank_request(rerank_request *r) {
- if (!r || !r->model) {
- if (r && r->j)
- job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error");
- return;
- }
- float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
- uint64_t t0 = nstime();
-#if defined(USE_MLX)
- int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores);
-#elif defined(USE_CUDA)
- int rc =
- r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores);
-#else
- int rc = execute_rerank_cpu(r, scores);
-#endif
- r->j->infer_ns += nstime() - t0;
if (rc != 0) {
free(scores);
job_set_error(r->j, 500, "late-interaction reranking failed", "server_error");
diff --git a/src/server_http.c b/tools/server/http.c
index f155aec..a1d347f 100644
--- a/src/server_http.c
+++ b/tools/server/http.c
@@ -1,5 +1,5 @@
#include "server_internal.h"
-#include "server_util.h"
+#include "util.h"
#include <ctype.h>
#include <errno.h>
@@ -240,7 +240,7 @@ static int parse_request_headers(client *c) {
char *end = NULL;
errno = 0;
unsigned long long v = strtoull(cl, &end, 10);
- if (errno || !end || *end || v > EMBED_HTTP_MAX_BODY) {
+ if (errno || !end || *end || v > FFWD_HTTP_MAX_BODY) {
free(cl);
return -1;
}
@@ -255,7 +255,7 @@ static int complete_request(client *c) {
if (!c->header_done) {
ssize_t hend = find_header_end(c->in.ptr, c->in.len);
if (hend < 0) {
- if (c->in.len > EMBED_HTTP_MAX_HEADER)
+ if (c->in.len > FFWD_HTTP_MAX_HEADER)
return -1;
return 0;
}
@@ -264,7 +264,7 @@ static int complete_request(client *c) {
if (parse_request_headers(c) != 0)
return -1;
}
- if (c->content_length > EMBED_HTTP_MAX_BODY)
+ if (c->content_length > FFWD_HTTP_MAX_BODY)
return -1;
if (c->in.len < c->header_len + c->content_length)
return 0;
@@ -368,7 +368,7 @@ void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
(void)loop;
(void)mask;
client *c = clientData;
- char tmp[EMBED_HTTP_IO_BUF];
+ char tmp[FFWD_HTTP_IO_BUF];
for (;;) {
ssize_t n = read(fd, tmp, sizeof(tmp));
if (n > 0) {
diff --git a/src/server_json.c b/tools/server/json.c
index cca8fd3..4812164 100644
--- a/src/server_json.c
+++ b/tools/server/json.c
@@ -1,5 +1,5 @@
#include "server_internal.h"
-#include "server_util.h"
+#include "util.h"
#include <cjson/cJSON.h>
#include <ctype.h>
@@ -143,7 +143,7 @@ bool cjson_is_integer(cJSON *item) {
}
const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) {
- const char *dflt = api == EMBED_API_PERPLEXITY ? "base64_int8" : "float";
+ const char *dflt = api == FFWD_API_PERPLEXITY ? "base64_int8" : "float";
cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format");
if (!encoding)
return dflt;
@@ -153,13 +153,12 @@ const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api)
return dflt;
}
const char *v = encoding->valuestring;
- int ok = api == EMBED_API_PERPLEXITY
- ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
- !strcmp(v, "float"))
+ int ok = api == FFWD_API_PERPLEXITY
+ ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") || !strcmp(v, "float"))
: (!strcmp(v, "float") || !strcmp(v, "base64"));
if (!ok) {
ve_add(detail, "[\"body\",\"encoding_format\"]",
- api == EMBED_API_PERPLEXITY
+ api == FFWD_API_PERPLEXITY
? "value is not a valid enum member; permitted: "
"'base64_int8', 'base64_binary', 'float'"
: "value is not a valid enum member; permitted: 'float', 'base64'",
@@ -182,8 +181,7 @@ int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) {
return 0;
}
if (!cJSON_IsString(tt) || !tt->valuestring) {
- ve_add(detail, "[\"body\",\"text_type\"]", "text_type must be a string",
- "type_error.string");
+ ve_add(detail, "[\"body\",\"text_type\"]", "text_type must be a string", "type_error.string");
return 0;
}
const char *v = tt->valuestring;
@@ -196,8 +194,7 @@ int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) {
return 0;
}
-int dimensions_from_root(
- cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) {
+int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) {
int dims = max_dim;
cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions");
if (dimensions) {
@@ -208,8 +205,7 @@ int dimensions_from_root(
dims = dimensions->valueint;
if (dims < min_dim || dims > max_dim) {
char msg[96];
- snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim,
- max_dim);
+ snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim, max_dim);
ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range");
}
}
diff --git a/tools/server/main.c b/tools/server/main.c
new file mode 100644
index 0000000..1cbdc3b
--- /dev/null
+++ b/tools/server/main.c
@@ -0,0 +1,342 @@
+/* main.c - ffwd-server entry point.
+ *
+ * The frontend for the ffwd-server binary: argument parsing, model-spec
+ * handling, and main(). The server logic lives in server.c; this file only
+ * builds the ffwd_server_config_t and calls ffwd_run_server. It is split
+ * from server.c so the test suite can include server.c (for its internal
+ * functions) without a conflicting main() and without a compile-out guard. */
+#include "build.h"
+#include "server.h"
+#include "server_internal.h"
+#include "ffwd.h"
+#include "internal.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static void print_usage(const char *prog) {
+ fprintf(stderr,
+ "Usage: %s --model LABEL=DIR [options]\n"
+ "\n"
+ "Options:\n"
+ " --model LABEL=DIR[:key=val]\n"
+ " Standard embedding model (repeatable)\n"
+ " --contextual-model LABEL=DIR[:key=val]\n"
+ " Contextual embedding model (repeatable)\n"
+ " --late-model LABEL=DIR[:key=val]\n"
+ " Late-interaction rerank model (repeatable)\n"
+ " keys: api=openai|perplexity, min_dim=N,\n"
+ " query=TEXT (last key; \\n escapes newline)\n"
+ " --host HOST Bind host (default: 127.0.0.1)\n"
+ " --port N Bind port (default: 8000)\n"
+ " --api-key K Require Authorization: Bearer K\n"
+ " -b, --batch-size N Max texts or documents per inference batch\n"
+ " (default: 32)\n"
+ " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
+ " --batch-wait-us N First-arrival micro-batch deadline in us\n"
+ " (default: %d)\n",
+ prog, ffwd_default_batch_wait_us());
+ ffwd_server_help(stderr);
+ fprintf(stderr,
+ " -v, --verbose Verbose (-vv for debug)\n"
+ " -V, --version Print version and exit\n"
+ " --build-info Print build details and exit\n"
+ " -h, --help Show this help\n"
+ "\n"
+ "Examples:\n"
+ " %s --model docs=./model --port 8000\n"
+ " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n",
+ prog, prog);
+}
+
+typedef struct {
+ ffwd_server_model_spec_t *v;
+ int n;
+ int cap;
+} model_specs_t;
+
+static char *dup_range(const char *p, size_t n) {
+ char *s = (char *)malloc(n + 1);
+ if (!s)
+ return NULL;
+ memcpy(s, p, n);
+ s[n] = '\0';
+ return s;
+}
+
+static char *unescape_model_option(const char *s) {
+ size_t n = strlen(s);
+ char *out = (char *)malloc(n + 1);
+ if (!out)
+ return NULL;
+ size_t w = 0;
+ for (size_t r = 0; r < n; r++) {
+ if (s[r] == '\\' && r + 1 < n) {
+ r++;
+ if (s[r] == 'n')
+ out[w++] = '\n';
+ else if (s[r] == 't')
+ out[w++] = '\t';
+ else if (s[r] == 'r')
+ out[w++] = '\r';
+ else
+ out[w++] = s[r];
+ } else {
+ out[w++] = s[r];
+ }
+ }
+ out[w] = '\0';
+ return out;
+}
+
+static int parse_model_option(ffwd_server_model_spec_t *spec, const char *opt, const char *flag) {
+ const char *eq = strchr(opt, '=');
+ if (!eq || eq == opt || !eq[1]) {
+ fprintf(stderr, "%s option expects key=value: %s\n", flag, opt);
+ return -1;
+ }
+ size_t key_len = (size_t)(eq - opt);
+ const char *val = eq + 1;
+ if (key_len == 3 && strncmp(opt, "api", key_len) == 0) {
+ if (!strcmp(val, "openai")) {
+ spec->api = FFWD_SERVER_API_OPENAI;
+ } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) {
+ spec->api = FFWD_SERVER_API_PERPLEXITY;
+ } else if (!strcmp(val, "default")) {
+ spec->api = FFWD_SERVER_API_DEFAULT;
+ } else {
+ fprintf(stderr, "%s api must be openai or perplexity\n", flag);
+ return -1;
+ }
+ return 0;
+ }
+ if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) {
+ char *end = NULL;
+ long v = strtol(val, &end, 10);
+ if (!end || *end || v <= 0 || v > INT_MAX) {
+ fprintf(stderr, "%s min_dim must be a positive integer\n", flag);
+ return -1;
+ }
+ spec->min_dim = (int)v;
+ return 0;
+ }
+ fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt);
+ return -1;
+}
+
+static int append_model_spec(model_specs_t *specs,
+ const char *arg,
+ ffwd_server_model_kind_t kind,
+ const char *flag) {
+ const char *eq = strchr(arg, '=');
+ if (!eq || eq == arg || !eq[1]) {
+ fprintf(stderr, "%s expects LABEL=DIR\n", flag);
+ return -1;
+ }
+ if (specs->n == specs->cap) {
+ int new_cap = specs->cap ? specs->cap * 2 : 4;
+ void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
+ if (!p)
+ return -1;
+ specs->v = p;
+ specs->cap = new_cap;
+ }
+ size_t id_len = (size_t)(eq - arg);
+ const char *path_start = eq + 1;
+ const char *opts = strchr(path_start, ':');
+ size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start);
+ if (path_len == 0) {
+ fprintf(stderr, "%s expects a non-empty DIR\n", flag);
+ return -1;
+ }
+ ffwd_server_model_spec_t spec;
+ memset(&spec, 0, sizeof(spec));
+ spec.id = dup_range(arg, id_len);
+ spec.path = dup_range(path_start, path_len);
+ spec.kind = kind;
+ if (!spec.id || !spec.path) {
+ free((char *)spec.id);
+ free((char *)spec.path);
+ return -1;
+ }
+ const char *p = opts ? opts + 1 : NULL;
+ while (p && *p) {
+ if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) {
+ const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15);
+ spec.query_instruct = unescape_model_option(val);
+ if (!spec.query_instruct)
+ goto fail;
+ break;
+ }
+ const char *next = strchr(p, ':');
+ size_t opt_len = next ? (size_t)(next - p) : strlen(p);
+ char *opt = dup_range(p, opt_len);
+ if (!opt)
+ goto fail;
+ int rc = parse_model_option(&spec, opt, flag);
+ free(opt);
+ if (rc != 0)
+ goto fail;
+ p = next ? next + 1 : NULL;
+ }
+ specs->v[specs->n] = spec;
+ specs->n++;
+ return 0;
+
+fail:
+ free((char *)spec.id);
+ free((char *)spec.path);
+ free((char *)spec.query_instruct);
+ return -1;
+}
+
+static void free_model_specs(model_specs_t *specs) {
+ if (!specs)
+ return;
+ for (int i = 0; i < specs->n; i++) {
+ free((char *)specs->v[i].id);
+ free((char *)specs->v[i].path);
+ free((char *)specs->v[i].query_instruct);
+ }
+ free(specs->v);
+}
+
+int main(int argc, char *argv[]) {
+ int verbose = 0;
+ int batch_size = 0;
+ int max_batch_tokens = 0;
+ int batch_wait_us = -1;
+ /* Backend tuning flags parse into opts regardless of backend; the linked
+ * backend uses what it supports. */
+ ffwd_options_t opts = {.gpu_quant_group_size = 64};
+ const char *host = "127.0.0.1";
+ const char *api_key = NULL;
+ int port = 8000;
+ model_specs_t model_specs = {0};
+
+ const char *prog = ffwd_prog_name(argv[0]);
+ int arg = 1;
+ while (arg < argc && argv[arg][0] == '-') {
+ const char *f = argv[arg];
+ if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
+ ffwd_print_version(prog);
+ free_model_specs(&model_specs);
+ return 0;
+ } else if (!strcmp(f, "--build-info")) {
+ ffwd_print_build_info(prog);
+ free_model_specs(&model_specs);
+ return 0;
+ } else if (!strcmp(f, "--model")) {
+ if (arg + 1 >= argc ||
+ append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_STANDARD, f) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--contextual-model")) {
+ if (arg + 1 >= argc || append_model_spec(&model_specs, argv[++arg],
+ FFWD_SERVER_MODEL_CONTEXTUAL, f) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--late-model")) {
+ if (arg + 1 >= argc ||
+ append_model_spec(&model_specs, argv[++arg], FFWD_SERVER_MODEL_LATE, f) != 0) {
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--gpu-quant-bits")) {
+ opts.gpu_quant_bits = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--gpu-quant-group-size")) {
+ opts.gpu_quant_group_size = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--memory-utilization")) {
+ opts.memory_utilization = atof(argv[++arg]);
+ if (opts.memory_utilization <= 0.0) {
+ fprintf(stderr, "--memory-utilization must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--gpu-gemm-mode")) {
+ opts.gpu_gemm_mode = argv[++arg];
+ } else if (!strcmp(f, "--gpu-weight-dtype")) {
+ opts.gpu_weight_dtype = argv[++arg];
+ } else if (!strcmp(f, "--host")) {
+ host = argv[++arg];
+ } else if (!strcmp(f, "--port")) {
+ port = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--api-key")) {
+ api_key = argv[++arg];
+ } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
+ batch_size = atoi(argv[++arg]);
+ } else if (!strcmp(f, "--max-batch-tokens")) {
+ max_batch_tokens = atoi(argv[++arg]);
+ if (max_batch_tokens <= 0) {
+ fprintf(stderr, "--max-batch-tokens must be > 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "--batch-wait-us")) {
+ batch_wait_us = atoi(argv[++arg]);
+ if (batch_wait_us < 0) {
+ fprintf(stderr, "--batch-wait-us must be >= 0\n");
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
+ opts.n_threads = atoi(argv[++arg]);
+ } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
+ verbose++;
+ } else if (!strcmp(f, "-vv")) {
+ verbose = 2;
+ } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 0;
+ } else {
+ fprintf(stderr, "unknown option: %s\n", f);
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ arg++;
+ }
+
+ ffwd_verbose = verbose;
+
+ if (arg < argc) {
+ fprintf(stderr, "unexpected argument: %s\n", argv[arg]);
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ if (model_specs.n == 0) {
+ fprintf(stderr, "at least one --model LABEL=DIR is required\n");
+ print_usage(prog);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+ char err[256];
+ if (ffwd_init(&opts, err, sizeof(err)) != 0) {
+ fprintf(stderr, "%s\n", err);
+ free_model_specs(&model_specs);
+ return 1;
+ }
+
+ ffwd_server_config_t scfg = {
+ .models = model_specs.v,
+ .n_models = model_specs.n,
+ .host = host,
+ .port = port,
+ .batch_size = batch_size > 0 ? batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE,
+ .max_batch_tokens = max_batch_tokens,
+ .batch_wait_us = batch_wait_us,
+ .gpu_quantize_bits = opts.gpu_quant_bits,
+ .gpu_quantize_group_size = opts.gpu_quant_group_size,
+ .memory_utilization = opts.memory_utilization,
+ .api_key = api_key,
+ };
+ int rc = ffwd_run_server(&scfg);
+ free_model_specs(&model_specs);
+ return rc;
+}
diff --git a/tools/server/models.c b/tools/server/models.c
new file mode 100644
index 0000000..8fdc590
--- /dev/null
+++ b/tools/server/models.c
@@ -0,0 +1,342 @@
+#include "server_internal.h"
+#include "util.h"
+
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* ========================================================================
+ * Model load/lifecycle, and tokenize/inference dispatch
+ *
+ * Each loaded model owns resolved serving metadata for the operator-chosen
+ * label. Transformer behavior comes from the model files through ffwd_config;
+ * server-only facts absent from those files, such as output API family and
+ * Matryoshka floor, come from the load spec. The tokenize_* and model_ffwd_*
+ * helpers are the single dispatch point the request handlers call, hiding the
+ * compiled backend choice behind one signature.
+ * ======================================================================== */
+
+loaded_model *loaded_model_for_label(http_server *s, const char *label) {
+ if (!label)
+ return NULL;
+ for (int i = 0; i < s->n_models; i++) {
+ if (s->models[i].info && !strcmp(label, s->models[i].info->id))
+ return &s->models[i];
+ }
+ return NULL;
+}
+
+static model_kind model_kind_from_public(ffwd_server_model_kind_t kind) {
+ switch (kind) {
+ case FFWD_SERVER_MODEL_CONTEXTUAL:
+ return MODEL_KIND_CONTEXTUAL;
+ case FFWD_SERVER_MODEL_LATE:
+ return MODEL_KIND_LATE;
+ case FFWD_SERVER_MODEL_STANDARD:
+ default:
+ return MODEL_KIND_STANDARD;
+ }
+}
+
+static embedding_api_t embedding_api_from_public(ffwd_server_embedding_api_t api) {
+ return api == FFWD_SERVER_API_PERPLEXITY ? FFWD_API_PERPLEXITY : FFWD_API_OPENAI;
+}
+
+static int read_optional_text_file(const char *model_dir, const char *relative, char **out) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ server_log("ffwd-server: model path too long");
+ return -1;
+ }
+ FILE *f = fopen(path, "rb");
+ if (!f && errno == ENOENT)
+ return 0;
+ if (!f) {
+ server_log("ffwd-server: failed to open %s", path);
+ return -1;
+ }
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf) {
+ fclose(f);
+ return -1;
+ }
+ if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ free(buf);
+ fclose(f);
+ return -1;
+ }
+ fclose(f);
+ buf[sz] = '\0';
+ *out = buf;
+ return 1;
+}
+
+static int read_query_instruct_from_files(const char *model_dir, char **out) {
+ char *buf = NULL;
+ int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf);
+ if (rc <= 0)
+ return rc;
+ cJSON *root = cJSON_Parse(buf);
+ free(buf);
+ if (!root) {
+ server_log("ffwd-server: invalid config_sentence_transformers.json: %s", model_dir);
+ return -1;
+ }
+ cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts");
+ cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL;
+ if (query && !cJSON_IsString(query)) {
+ cJSON_Delete(root);
+ server_log("ffwd-server: prompts.query must be a string: %s", model_dir);
+ return -1;
+ }
+ if (query && query->valuestring && query->valuestring[0]) {
+ *out = xstrdup(query->valuestring);
+ cJSON_Delete(root);
+ return *out ? 1 : -1;
+ }
+ cJSON_Delete(root);
+ return 0;
+}
+
+static int init_model_info(model_info **out, const ffwd_server_model_spec_t *spec) {
+ model_info *info = xcalloc(1, sizeof(*info));
+ info->id = xstrdup(spec->id);
+ info->kind = model_kind_from_public(spec->kind);
+ info->api = embedding_api_from_public(spec->api);
+ info->min_dim = spec->min_dim;
+ if (spec->query_instruct) {
+ info->query_instruct = xstrdup(spec->query_instruct);
+ } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) {
+ free(info->id);
+ free(info);
+ return -1;
+ }
+ if (!info->id || (spec->query_instruct && !info->query_instruct)) {
+ free(info->query_instruct);
+ free(info->id);
+ free(info);
+ return -1;
+ }
+ *out = info;
+ return 0;
+}
+
+static void free_model_info(model_info *info) {
+ if (!info)
+ return;
+ free(info->id);
+ free(info->query_instruct);
+ free(info);
+}
+
+void free_token_bufs(token_buf *t, int n) {
+ if (!t)
+ return;
+ for (int i = 0; i < n; i++)
+ free(t[i].ids);
+}
+
+void embedding_request_free(embedding_request *r) {
+ if (!r)
+ return;
+ free_token_bufs(r->tokens, r->n_inputs);
+ free(r->tokens);
+ free(r->inputs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void contextual_request_free(contextual_request *r) {
+ if (!r)
+ return;
+ for (int i = 0; i < r->n_docs; i++) {
+ free(r->docs[i].ids);
+ free(r->docs[i].spans);
+ }
+ free(r->docs);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+void late_tokens_free(late_tokens *t) {
+ if (!t)
+ return;
+ free(t->ids);
+ free(t->keep);
+ memset(t, 0, sizeof(*t));
+}
+
+void rerank_request_free(rerank_request *r) {
+ if (!r)
+ return;
+ late_tokens_free(&r->query);
+ for (int i = 0; i < r->n_documents; i++)
+ late_tokens_free(&r->documents[i]);
+ free(r->documents);
+ if (r->root)
+ cJSON_Delete(r->root);
+ memset(r, 0, sizeof(*r));
+}
+
+/* Tokenization itself lives in libffwd (ffwd_tok). These thin wrappers only
+ * add the server's per-stage timing onto the job; the request handlers call
+ * them unchanged. */
+int tokenize_input(
+ loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
+ uint64_t t0 = nstime();
+ out->ids = ffwd_tokenize(m->tok, text, query_instruct, &out->n_tokens);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ if (!out->ids) {
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ return 0;
+}
+
+int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
+ return tokenize_input(m, j, text, NULL, out);
+}
+
+int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
+ uint64_t t0 = nstime();
+ int rc = ffwd_tokenize_late(m->tok, text, is_query, out);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ return rc;
+}
+
+int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out) {
+ return ffwd_encode_batch(m->backend, inputs, batch, out);
+}
+
+int model_ffwd_spans_batch(loaded_model *m,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ return ffwd_encode_spans_batch(m->backend, inputs, batch, out);
+}
+
+static int model_uses_dense_batches(const loaded_model *m) {
+ return ffwd_uses_dense_batches(m->backend);
+}
+
+/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
+ * pads each dense row to the longest input in the chunk. */
+int inference_batch_accepts_input(
+ const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
+ if (batch == 0)
+ return 1;
+ if (model_uses_dense_batches(m))
+ return next_tokens <= max_batch_tokens / (batch + 1);
+ return next_tokens <= max_batch_tokens - packed_tokens;
+}
+
+int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim) {
+ if (!m || !m->info || !config || config->hidden_size <= 0) {
+ if (m && m->info)
+ server_log("ffwd-server: model %s has invalid config", m->info->id);
+ return -1;
+ }
+ if (m->info->kind == MODEL_KIND_LATE) {
+ if (token_dim <= 0) {
+ server_log("ffwd-server: late model %s is missing token projection metadata",
+ m->info->id);
+ return -1;
+ }
+ } else if (token_dim != 0) {
+ server_log("ffwd-server: non-late model %s unexpectedly has token projection metadata",
+ m->info->id);
+ return -1;
+ }
+
+ int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size;
+ if (min_dim <= 0 || min_dim > config->hidden_size) {
+ server_log("ffwd-server: model %s has invalid min_dim=%d for dim=%d", m->info->id, min_dim,
+ config->hidden_size);
+ return -1;
+ }
+ m->info->dim = config->hidden_size;
+ m->info->min_dim = min_dim;
+ m->info->token_dim = token_dim;
+ m->info->attention_mode = config->attention_mode;
+ m->info->pooling_mode = config->pooling_mode;
+ m->info->normalize_embeddings = config->normalize_embeddings;
+ m->renormalize_truncated =
+ config->normalize_embeddings && config->pooling_mode == FFWD_POOL_LAST_TOKEN;
+ return 0;
+}
+
+/* Apply the loaded model's config to its serving metadata and validate it. CPU
+ * runs this in load_one_model; a GPU build runs it on the inference thread once
+ * ffwd_activate has created the context. */
+int finalize_loaded_model(loaded_model *m) {
+ const ffwd_config_t *config = ffwd_config(m->backend);
+ int token_dim = ffwd_token_dim(m->backend);
+ ffwd_tok_set_append_terminal(m->tok, config->append_terminal_token);
+ return configure_loaded_model(m, config, token_dim);
+}
+
+int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec) {
+ /* Grow the model list by one entry. */
+ loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models));
+ if (!nm)
+ return -1;
+ s->models = nm;
+ loaded_model *m = &s->models[s->n_models];
+ memset(m, 0, sizeof(*m));
+ s->n_models++;
+
+ if (init_model_info(&m->info, spec) != 0)
+ return -1;
+ const char *path = spec->path;
+ m->path = xstrdup(path);
+ char err[256];
+ /* The tokenizer (file probe, special-token resolution, [CLS]/[SEP] vs Qwen
+ * terminal-token layout) is owned by libffwd now. */
+ m->tok = ffwd_tok_open(path, m->info->kind == MODEL_KIND_LATE, err, sizeof(err));
+ if (!m->tok) {
+ server_log("ffwd-server: %s: %s", path, err);
+ return -1;
+ }
+
+ m->backend =
+ ffwd_open(path, m->info->kind == MODEL_KIND_LATE, &s->backend_opts, err, sizeof(err));
+ if (!m->backend) {
+ server_log("ffwd-server: %s: %s", path, err);
+ return -1;
+ }
+ /* CPU loads the model in open, so its config is ready to finalize now. A GPU
+ * build defers the load - and this finalize - to the inference thread. */
+ if (ffwd_config(m->backend))
+ return finalize_loaded_model(m);
+ return 0;
+}
+
+void free_models(http_server *s) {
+ for (int i = 0; i < s->n_models; i++) {
+ loaded_model *m = &s->models[i];
+ free_model_info(m->info);
+ free(m->path);
+ ffwd_tok_free(m->tok);
+ ffwd_free(m->backend);
+ }
+ free(s->models);
+ s->models = NULL;
+ s->n_models = 0;
+}
diff --git a/src/sbuf.c b/tools/server/sbuf.c
index f286d8c..b341c31 100644
--- a/src/sbuf.c
+++ b/tools/server/sbuf.c
@@ -1,5 +1,5 @@
#include "sbuf.h"
-#include "server_util.h"
+#include "util.h"
#include <stdarg.h>
#include <stdint.h>
diff --git a/src/sbuf.h b/tools/server/sbuf.h
index 0c8929e..3116090 100644
--- a/src/sbuf.h
+++ b/tools/server/sbuf.h
@@ -1,5 +1,5 @@
-#ifndef EMBED_SBUF_H
-#define EMBED_SBUF_H
+#ifndef FFWD_SBUF_H
+#define FFWD_SBUF_H
#include <stddef.h>
@@ -21,4 +21,4 @@ void sbuf_clear(sbuf *b);
void sbuf_printf(sbuf *b, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
void sbuf_free(sbuf *b);
-#endif /* EMBED_SBUF_H */
+#endif /* FFWD_SBUF_H */
diff --git a/src/server_schedule.c b/tools/server/schedule.c
index 1daf8dd..c13e0cf 100644
--- a/src/server_schedule.c
+++ b/tools/server/schedule.c
@@ -1,5 +1,5 @@
#include "server_internal.h"
-#include "server_util.h"
+#include "util.h"
#include <errno.h>
#include <pthread.h>
@@ -12,7 +12,7 @@
/* ========================================================================
* Job queue, micro-batching, and completion
*
- * dispatch_request (server_http.c) enqueues one job per framed request; the
+ * dispatch_request (http.c) enqueues one job per framed request; the
* inference worker thread (worker_main, server.c) drains them with
* collect_job_batch (a short first-arrival wait groups concurrent arrivals) and
* runs process_job_group, which routes each job to its endpoint handler and
diff --git a/tools/server/server.c b/tools/server/server.c
new file mode 100644
index 0000000..abff2c2
--- /dev/null
+++ b/tools/server/server.c
@@ -0,0 +1,414 @@
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+
+#include "server.h"
+#include "server_internal.h"
+#include "util.h"
+#include "sbuf.h"
+#include "ffwd.h"
+#include "ae.h"
+#include "anet.h"
+
+#include <arpa/inet.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <netinet/in.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+# include <sys/sysctl.h>
+#endif
+
+#include <cjson/cJSON.h>
+
+static volatile sig_atomic_t g_stop_requested = 0;
+static volatile sig_atomic_t g_signal_wfd = -1;
+
+static void stop_signal_handler(int sig) {
+ (void)sig;
+ if (g_stop_requested)
+ _exit(130);
+ int saved = errno;
+ g_stop_requested = 1;
+ int fd = (int)g_signal_wfd;
+ if (fd >= 0) {
+ const unsigned char byte = 1;
+ if (write(fd, &byte, 1) < 0) {
+ /* best-effort stop wakeup; a failed self-pipe write is ignored */
+ }
+ }
+ errno = saved;
+}
+
+static bool auth_ok(http_server *s, const char *auth) {
+ if (!s->api_key || !s->api_key[0])
+ return true;
+ if (!auth)
+ return false;
+ const char prefix[] = "Bearer ";
+ return !strncmp(auth, prefix, sizeof(prefix) - 1) &&
+ !strcmp(auth + sizeof(prefix) - 1, s->api_key);
+}
+
+void dispatch_request(client *c) {
+ aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
+
+ if (!strcmp(c->req.method, "OPTIONS")) {
+ append_http_response(c, 204, NULL, "", 0);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+ return;
+ }
+
+ if (!route_is_inference(c->req.method, c->req.path)) {
+ size_t len;
+ char *body = json_error_body("unknown endpoint", "invalid_request_error", &len);
+ append_http_response(c, 404, "application/json", body, len);
+ free(body);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+ return;
+ }
+
+ if (!auth_ok(c->srv, c->req.auth)) {
+ size_t len;
+ char *body = json_error_body("missing or invalid bearer token", "authentication_error", &len);
+ append_http_response(c, 401, "application/json", body, len);
+ free(body);
+ if (queue_write(c) != AE_OK)
+ close_client(c);
+ return;
+ }
+
+ job *j = xcalloc(1, sizeof(*j));
+ j->srv = c->srv;
+ j->c = c;
+ snprintf(j->method, sizeof(j->method), "%s", c->req.method);
+ snprintf(j->path, sizeof(j->path), "%s", c->req.path);
+ j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len);
+ j->body_len = c->req.body_len;
+ j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
+ j->created_ns = nstime();
+ client_incref(c);
+ enqueue_job(j);
+}
+
+static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
+ (void)loop;
+ (void)mask;
+ http_server *s = clientData;
+ char cip[64];
+ int cport = 0;
+ for (;;) {
+ int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport);
+ if (cfd == ANET_ERR) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK)
+ return;
+ if (errno == EINTR)
+ continue;
+ return;
+ }
+ set_nonblock(cfd);
+ client *c = xcalloc(1, sizeof(*c));
+ c->srv = s;
+ c->fd = cfd;
+ c->refcount = 1;
+ c->last_active_ms = mstime();
+ c->linked = 1;
+ c->next = s->clients;
+ if (s->clients)
+ s->clients->prev = c;
+ s->clients = c;
+ s->n_clients++;
+ if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK)
+ close_client(c);
+ }
+}
+
+static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
+ (void)loop;
+ (void)id;
+ http_server *s = clientData;
+ uint64_t now = mstime();
+ client *c = s->clients;
+ while (c) {
+ client *next = c->next;
+ /* refcount > 1 means a request from this client is queued or being
+ * processed by the worker; long batch queues (e.g. 4B long-document
+ * batches) must not be reaped as idle. */
+ if (!c->cancelled && c->refcount <= 1 &&
+ now - c->last_active_ms > FFWD_HTTP_CLIENT_TIMEOUT_MS)
+ close_client(c);
+ c = next;
+ }
+ return FFWD_HTTP_SWEEP_MS;
+}
+
+static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
+ (void)fd;
+ (void)mask;
+ http_server *s = clientData;
+ unsigned char tmp[128];
+ while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {
+ }
+ aeStop(loop);
+}
+
+static void *worker_main(void *arg) {
+ http_server *s = arg;
+ /* Create each model's context on this thread (a GPU build defers the load
+ * here so the context lives on the thread that uses it) and finalize the
+ * config a GPU build could not finalize at open time. CPU is loaded and
+ * finalized already, marked by info->dim being set. */
+ int rc = 0;
+ char load_err[256];
+ for (int i = 0; i < s->n_models; i++) {
+ loaded_model *m = &s->models[i];
+ if (ffwd_activate(m->backend, load_err, sizeof(load_err)) != 0) {
+ server_log("ffwd-server: failed to load model on worker: %s: %s", m->path, load_err);
+ rc = 1;
+ break;
+ }
+ if (m->info->dim == 0 && finalize_loaded_model(m) != 0) {
+ rc = 1;
+ break;
+ }
+ }
+ pthread_mutex_lock(&s->mu);
+ s->worker_init_rc = rc;
+ s->worker_ready = 1;
+ if (rc)
+ s->stopping = 1;
+ pthread_cond_broadcast(&s->cv);
+ pthread_mutex_unlock(&s->mu);
+ if (rc) {
+ for (int i = 0; i < s->n_models; i++)
+ ffwd_worker_free(s->models[i].backend);
+ return NULL;
+ }
+
+ for (;;) {
+ job *jobs[FFWD_SERVER_MICROBATCH_MAX_JOBS];
+ int n_jobs = collect_job_batch(s, jobs, FFWD_SERVER_MICROBATCH_MAX_JOBS);
+ if (n_jobs == 0)
+ break;
+ process_job_group(s, jobs, n_jobs);
+ }
+ for (int i = 0; i < s->n_models; i++)
+ ffwd_worker_free(s->models[i].backend);
+ return NULL;
+}
+
+static int listen_on(const char *host, int port) {
+ char portbuf[32];
+ snprintf(portbuf, sizeof(portbuf), "%d", port);
+ char err[ANET_ERR_LEN];
+ int fd = anetTcpServer(err, port, (char *)host, FFWD_HTTP_BACKLOG);
+ (void)portbuf;
+ if (fd == ANET_ERR) {
+ server_log("ffwd-server: listen failed on %s:%d: %s", host, port, err);
+ return -1;
+ }
+ set_nonblock(fd);
+ return fd;
+}
+
+static int validate_model_specs(const ffwd_server_config_t *cfg) {
+ for (int i = 0; i < cfg->n_models; i++) {
+ const ffwd_server_model_spec_t *spec = &cfg->models[i];
+ if (!spec->id || !spec->id[0]) {
+ fprintf(stderr, "ffwd-server: model label must not be empty\n");
+ return -1;
+ }
+ if (!spec->path || !spec->path[0]) {
+ fprintf(stderr, "ffwd-server: model %s has an empty path\n", spec->id);
+ return -1;
+ }
+ if (spec->kind < FFWD_SERVER_MODEL_STANDARD || spec->kind > FFWD_SERVER_MODEL_LATE) {
+ fprintf(stderr, "ffwd-server: model %s has an invalid kind\n", spec->id);
+ return -1;
+ }
+ if (spec->api < FFWD_SERVER_API_DEFAULT || spec->api > FFWD_SERVER_API_PERPLEXITY) {
+ fprintf(stderr, "ffwd-server: model %s has an invalid api\n", spec->id);
+ return -1;
+ }
+ if (spec->min_dim < 0) {
+ fprintf(stderr, "ffwd-server: model %s has an invalid min_dim\n", spec->id);
+ return -1;
+ }
+ for (int j = 0; j < i; j++) {
+ if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) {
+ fprintf(stderr, "ffwd-server: duplicate model label: %s\n", spec->id);
+ return -1;
+ }
+ }
+ }
+ return 0;
+}
+
+/* Hold the model paths in one array so the backend preflight can size the set
+ * against host memory in a single pass. */
+static int run_memory_preflight(const ffwd_server_config_t *cfg, const ffwd_options_t *opts) {
+ const char **paths = (const char **)malloc((size_t)cfg->n_models * sizeof(*paths));
+ if (!paths)
+ return -1;
+ for (int i = 0; i < cfg->n_models; i++)
+ paths[i] = cfg->models[i].path;
+ char err[256] = {0};
+ int rc = ffwd_preflight(paths, cfg->n_models, opts, err, sizeof(err));
+ free(paths);
+ if (rc != 0 && err[0])
+ server_log("ffwd-server: %s", err);
+ return rc;
+}
+
+int ffwd_run_server(const ffwd_server_config_t *cfg) {
+ if (!cfg || !cfg->models || cfg->n_models <= 0) {
+ fprintf(stderr, "ffwd-server: at least one model label and path is required\n");
+ return 1;
+ }
+ ffwd_options_t backend_opts = {
+ .gpu_quant_bits = cfg->gpu_quantize_bits,
+ .gpu_quant_group_size = cfg->gpu_quantize_group_size > 0 ? cfg->gpu_quantize_group_size : 64,
+ .memory_utilization = cfg->memory_utilization,
+ };
+ if (validate_model_specs(cfg) != 0 || run_memory_preflight(cfg, &backend_opts) != 0)
+ return 1;
+
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1";
+ s.port = cfg->port > 0 ? cfg->port : 8000;
+ s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : FFWD_SERVER_DEFAULT_BATCH_SIZE;
+ s.max_batch_tokens =
+ cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS;
+ s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us : ffwd_default_batch_wait_us();
+ s.backend_opts = backend_opts;
+ if (cfg->api_key && cfg->api_key[0])
+ s.api_key = xstrdup(cfg->api_key);
+ else {
+ const char *env_key = getenv("FFWD_API_KEY");
+ if (env_key && env_key[0])
+ s.api_key = xstrdup(env_key);
+ }
+
+ pthread_mutex_init(&s.mu, NULL);
+ pthread_cond_init(&s.cv, NULL);
+
+ for (int i = 0; i < cfg->n_models; i++) {
+ if (load_one_model(&s, &cfg->models[i]) != 0) {
+ free_models(&s);
+ return 1;
+ }
+ }
+
+ if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) {
+ perror("pipe");
+ free_models(&s);
+ return 1;
+ }
+ set_nonblock(s.completion_pipe[0]);
+ set_nonblock(s.completion_pipe[1]);
+ set_nonblock(s.signal_pipe[0]);
+ set_nonblock(s.signal_pipe[1]);
+
+ struct sigaction sa;
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_handler = stop_signal_handler;
+ sigemptyset(&sa.sa_mask);
+ sigaction(SIGINT, &sa, NULL);
+ sigaction(SIGTERM, &sa, NULL);
+ signal(SIGPIPE, SIG_IGN);
+ g_signal_wfd = s.signal_pipe[1];
+
+ s.loop = aeCreateEventLoop(FFWD_HTTP_SETSIZE);
+ if (!s.loop) {
+ fprintf(stderr, "ffwd-server: failed to create event loop\n");
+ free_models(&s);
+ return 1;
+ }
+
+ if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) {
+ fprintf(stderr, "ffwd-server: failed to start worker\n");
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ return 1;
+ }
+ pthread_mutex_lock(&s.mu);
+ while (!s.worker_ready)
+ pthread_cond_wait(&s.cv, &s.mu);
+ int worker_init_rc = s.worker_init_rc;
+ pthread_mutex_unlock(&s.mu);
+ if (worker_init_rc) {
+ pthread_join(s.worker, NULL);
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 1;
+ }
+
+ s.listen_fd = listen_on(s.host, s.port);
+ if (s.listen_fd < 0) {
+ pthread_mutex_lock(&s.mu);
+ s.stopping = 1;
+ pthread_cond_signal(&s.cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.worker, NULL);
+ aeDeleteEventLoop(s.loop);
+ free_models(&s);
+ return 1;
+ }
+
+ aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s);
+ aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s);
+ aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s);
+ aeCreateTimeEvent(s.loop, FFWD_HTTP_SWEEP_MS, timeout_cb, &s, NULL);
+
+ server_log("ffwd-server: listening on http://%s:%d", s.host, s.port);
+ aeMain(s.loop);
+
+ server_log("ffwd-server: shutdown requested");
+ if (s.listen_fd >= 0)
+ close(s.listen_fd);
+ client *c = s.clients;
+ while (c) {
+ client *next = c->next;
+ close_client(c);
+ c = next;
+ }
+
+ pthread_mutex_lock(&s.mu);
+ s.stopping = 1;
+ pthread_cond_signal(&s.cv);
+ pthread_mutex_unlock(&s.mu);
+ pthread_join(s.worker, NULL);
+
+ aeDeleteEventLoop(s.loop);
+ close(s.completion_pipe[0]);
+ close(s.completion_pipe[1]);
+ close(s.signal_pipe[0]);
+ close(s.signal_pipe[1]);
+ free_models(&s);
+ free(s.api_key);
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+ return 0;
+}
diff --git a/tools/server/server.h b/tools/server/server.h
new file mode 100644
index 0000000..b8ec7e4
--- /dev/null
+++ b/tools/server/server.h
@@ -0,0 +1,57 @@
+#ifndef FFWD_SERVER_H
+#define FFWD_SERVER_H
+
+/* Public-API export annotation. The libraries are built with
+ * -fvisibility=hidden, so only declarations carrying FFWD_API are exported
+ * from the shared library; everything else stays internal. */
+#ifndef FFWD_API
+# if defined(__GNUC__)
+# define FFWD_API __attribute__((visibility("default")))
+# else
+# define FFWD_API
+# endif
+#endif
+
+#include <stdbool.h>
+
+typedef enum {
+ FFWD_SERVER_MODEL_STANDARD = 0,
+ FFWD_SERVER_MODEL_CONTEXTUAL = 1,
+ FFWD_SERVER_MODEL_LATE = 2,
+} ffwd_server_model_kind_t;
+
+typedef enum {
+ FFWD_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */
+ FFWD_SERVER_API_OPENAI = 1,
+ FFWD_SERVER_API_PERPLEXITY = 2,
+} ffwd_server_embedding_api_t;
+
+typedef struct {
+ const char *id; /* served label; requests match this string exactly */
+ const char *path;
+ ffwd_server_model_kind_t kind;
+ ffwd_server_embedding_api_t api;
+ int min_dim; /* 0 means the loaded model's full hidden size */
+ const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */
+} ffwd_server_model_spec_t;
+
+typedef struct {
+ const ffwd_server_model_spec_t *models;
+ int n_models;
+ const char *host;
+ int port;
+ int batch_size;
+ int max_batch_tokens;
+ int batch_wait_us;
+ /* GPU build tuning; ignored by backends that do not use it. */
+ int gpu_quantize_bits;
+ int gpu_quantize_group_size;
+ double memory_utilization; /* fraction of physical memory the GPU preflight
+ may plan for; 0 = default 0.90, above 1.0
+ allows overcommit */
+ const char *api_key;
+} ffwd_server_config_t;
+
+FFWD_API int ffwd_run_server(const ffwd_server_config_t *cfg);
+
+#endif
diff --git a/src/server_internal.h b/tools/server/server_internal.h
index 1f106ca..91b51c5 100644
--- a/src/server_internal.h
+++ b/tools/server/server_internal.h
@@ -1,27 +1,17 @@
-#ifndef EMBED_SERVER_INTERNAL_H
-#define EMBED_SERVER_INTERNAL_H
+#ifndef FFWD_SERVER_INTERNAL_H
+#define FFWD_SERVER_INTERNAL_H
-/* Declarations shared across the embed-server translation units (HTTP framing,
+/* Declarations shared across the ffwd-server translation units (HTTP framing,
* JSON validation, output encoding, scheduling, model lifecycle, endpoint
* handlers). The server is split by concern but compiles as several objects
* linked into one binary; this header is what they share. Not a public API -
* the CLI and library do not include it. */
-#include "embed.h"
+#include "ffwd.h"
#include "server.h"
#include "sbuf.h"
-#include "tokenizer_bpe.h"
-#include "tokenizer_sentencepiece.h"
-#include "tokenizer_wordpiece.h"
-#ifdef USE_MLX
-#include "mlx.h"
-#endif
-#ifdef USE_CUDA
-#include "cuda.h"
-#endif
-
-#include "deps/ae/ae.h"
+#include "ae.h"
#include <cjson/cJSON.h>
#include <pthread.h>
@@ -29,45 +19,43 @@
#include <stddef.h>
#include <stdint.h>
-#define EMBED_HTTP_IO_BUF 8192
-#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
-#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
-#define EMBED_HTTP_MAX_PATH 255u
-#define EMBED_HTTP_BACKLOG 128
-#define EMBED_HTTP_SETSIZE 10240
-#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000
-#define EMBED_HTTP_SWEEP_MS 1000
+#define FFWD_HTTP_IO_BUF 8192
+#define FFWD_HTTP_MAX_HEADER (64u * 1024u)
+#define FFWD_HTTP_MAX_BODY (64u * 1024u * 1024u)
+#define FFWD_HTTP_MAX_PATH 255u
+#define FFWD_HTTP_BACKLOG 128
+#define FFWD_HTTP_SETSIZE 10240
+#define FFWD_HTTP_CLIENT_TIMEOUT_MS 30000
+#define FFWD_HTTP_SWEEP_MS 1000
-#define EMBED_API_MAX_STANDARD_INPUTS 512
-#define EMBED_API_MAX_CONTEXT_DOCS 512
-#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
-#define EMBED_API_MAX_RERANK_DOCUMENTS 1000
-#define EMBED_API_MAX_ITEM_TOKENS 32768
-#define EMBED_API_MAX_TOTAL_TOKENS 120000
-#define EMBED_LATE_QUERY_TOKENS 32
-#define EMBED_LATE_MASK_TOKEN_ID 151642
-#define EMBED_LATE_QUERY_PREFIX_ID 151669
-#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
+#define FFWD_API_MAX_STANDARD_INPUTS 512
+#define FFWD_API_MAX_CONTEXT_DOCS 512
+#define FFWD_API_MAX_CONTEXT_CHUNKS 16000
+#define FFWD_API_MAX_RERANK_DOCUMENTS 1000
+#define FFWD_API_MAX_ITEM_TOKENS 32768
+#define FFWD_API_MAX_TOTAL_TOKENS 120000
+/* FFWD_LATE_* and FFWD_CONTEXT_SEPARATOR_TOKEN_ID now live in ffwd.h, owned
+ * by the tokenizer (ffwd_tok). */
/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
* short-request throughput over 8 with no long-document penalty, while 128
* was no faster and inflated long-document tail latency. */
-#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
-#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
+#define FFWD_SERVER_DEFAULT_BATCH_SIZE 32
+#define FFWD_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
/* Microseconds the worker waits for more requests before dispatching a batch.
* CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
* busy; MLX and CPU dispatch immediately, where waiting only adds latency.
* Override with --batch-wait-us. */
-#define EMBED_SERVER_BATCH_WAIT_US 0
-#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
-#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
-#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
-#define EMBED_MLX_RESIDENT_MULTIPLIER 2
+#define FFWD_SERVER_BATCH_WAIT_US 0
+#define FFWD_SERVER_CUDA_BATCH_WAIT_US 1000
+#define FFWD_SERVER_MICROBATCH_MAX_JOBS 128
+#define FFWD_MLX_MEMORY_BUDGET_PERCENT 90
+#define FFWD_MLX_RESIDENT_MULTIPLIER 2
typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
typedef enum {
- EMBED_API_OPENAI = 0,
- EMBED_API_PERPLEXITY = 1,
+ FFWD_API_OPENAI = 0,
+ FFWD_API_PERPLEXITY = 1,
} embedding_api_t;
typedef struct {
@@ -76,8 +64,8 @@ typedef struct {
int dim;
int min_dim;
int token_dim;
- embed_attention_mode_t attention_mode;
- embed_pooling_mode_t pooling_mode;
+ ffwd_attention_mode_t attention_mode;
+ ffwd_pooling_mode_t pooling_mode;
int normalize_embeddings;
embedding_api_t api;
char *query_instruct;
@@ -86,38 +74,9 @@ typedef struct {
typedef struct {
model_info *info;
char *path;
- embed_tokenizer_t *tok;
- embed_tokenizer_workspace_t *tok_ws;
- /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model.
- * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the
- * Qwen byte-level BPE fields above stay NULL. */
- wordpiece_tokenizer_t *wp_tok;
- wordpiece_workspace_t *wp_tok_ws;
- sentencepiece_tokenizer_t *sp_tok;
- sentencepiece_workspace_t *sp_tok_ws;
- int cls_id;
- int sep_id;
- int context_separator_id;
- int late_mask_id;
- int late_query_prefix_id;
- int late_document_prefix_id;
- int late_skip_ids[64];
- int n_late_skip_ids;
- int append_terminal_token;
- int terminal_token_id;
- int renormalize_truncated;
- embed_model_t *cpu_model;
- embed_workspace_t *cpu_ws;
- embed_late_model_t *cpu_late_model;
- embed_late_workspace_t *cpu_late_ws;
-#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
- embed_mlx_late_ctx_t *mlx_late_ctx;
-#endif
-#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
- embed_cuda_late_ctx_t *cuda_late_ctx;
-#endif
+ ffwd_tok_t *tok; /* tokenizer: text -> ids + the model's special tokens */
+ ffwd_t *backend; /* compute handle */
+ int renormalize_truncated; /* serving policy: re-normalize a Matryoshka cut */
} loaded_model;
typedef struct client client;
@@ -136,10 +95,7 @@ typedef struct {
char *api_key;
loaded_model *models;
int n_models;
- int use_mlx;
- int use_cuda;
- int mlx_quantize_bits;
- int mlx_quantize_group_size;
+ ffwd_options_t backend_opts;
pthread_t worker;
pthread_mutex_t mu;
@@ -158,7 +114,7 @@ typedef struct {
typedef struct {
char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
+ char path[FFWD_HTTP_MAX_PATH + 1];
char version[16];
char *body;
size_t body_len;
@@ -188,7 +144,7 @@ struct job {
http_server *srv;
client *c;
char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
+ char path[FFWD_HTTP_MAX_PATH + 1];
char *body;
size_t body_len;
char *auth;
@@ -222,7 +178,7 @@ typedef struct {
const char *encoding;
int n_inputs;
token_buf *tokens;
- embed_input_t *inputs;
+ ffwd_input_t *inputs;
int total_tokens;
int ready;
} embedding_request;
@@ -230,7 +186,7 @@ typedef struct {
typedef struct {
int *ids;
int n_tokens;
- embed_span_t *spans;
+ ffwd_span_t *spans;
int n_spans;
} contextual_doc;
@@ -247,12 +203,8 @@ typedef struct {
int ready;
} contextual_request;
-typedef struct {
- int *ids;
- int n_tokens;
- int *keep;
- int n_keep;
-} late_tokens;
+/* The server's late_tokens is exactly what ffwd_tokenize_late fills. */
+typedef ffwd_late_tokens_t late_tokens;
typedef struct {
job *j;
@@ -278,8 +230,7 @@ cJSON *parse_json_body(job *j, cJSON *detail);
bool cjson_is_integer(cJSON *item);
const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api);
int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct);
-int dimensions_from_root(
- cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding);
+int dimensions_from_root(cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding);
/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */
int set_nonblock(int fd);
@@ -317,19 +268,21 @@ void embedding_request_free(embedding_request *r);
void contextual_request_free(contextual_request *r);
void late_tokens_free(late_tokens *t);
void rerank_request_free(rerank_request *r);
+/* Thin timing wrappers over libffwd's ffwd_tokenize / ffwd_tokenize_late. */
int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out);
int tokenize_input(
loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out);
int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out);
-int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out);
-int model_embed_spans_batch(loaded_model *m,
- const embed_context_input_t *inputs,
- int batch,
- float *out);
+int model_ffwd_batch(loaded_model *m, const ffwd_input_t *inputs, int batch, float *out);
+int model_ffwd_spans_batch(loaded_model *m,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out);
int inference_batch_accepts_input(
const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens);
-int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim);
-int load_one_model(http_server *s, const embed_server_model_spec_t *spec);
+int configure_loaded_model(loaded_model *m, const ffwd_config_t *config, int token_dim);
+int finalize_loaded_model(loaded_model *m);
+int load_one_model(http_server *s, const ffwd_server_model_spec_t *spec);
void free_models(http_server *s);
/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */
@@ -358,4 +311,4 @@ int collect_job_batch(http_server *s, job **jobs, int max_jobs);
void process_job_group(http_server *s, job **jobs, int n_jobs);
void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
-#endif /* EMBED_SERVER_INTERNAL_H */
+#endif /* FFWD_SERVER_INTERNAL_H */
diff --git a/tools/server/test_base64.c b/tools/server/test_base64.c
new file mode 100644
index 0000000..3519c0c
--- /dev/null
+++ b/tools/server/test_base64.c
@@ -0,0 +1,51 @@
+/* test_base64.c - base64_encode against RFC 4648 vectors and binary inputs. */
+
+#include "base64.h"
+#include "test_util.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+static void test_base64_rfc4648(void) {
+ /* RFC 4648 test vectors. */
+ static const struct {
+ const char *in, *out;
+ } cases[] = {
+ {"", ""},
+ {"f", "Zg=="},
+ {"fo", "Zm8="},
+ {"foo", "Zm9v"},
+ {"foob", "Zm9vYg=="},
+ {"fooba", "Zm9vYmE="},
+ {"foobar", "Zm9vYmFy"},
+ };
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ char *got = base64_encode((const unsigned char *)cases[i].in, strlen(cases[i].in));
+ TEST_ASSERT(strcmp(got, cases[i].out) == 0);
+ free(got);
+ }
+}
+
+static void test_base64_binary_vectors(void) {
+ static const struct {
+ unsigned char bytes[5];
+ size_t n;
+ const char *out;
+ } cases[] = {
+ {{0xff}, 1, "/w=="},
+ {{0xff, 0xee}, 2, "/+4="},
+ {{0xff, 0xee, 0xdd}, 3, "/+7d"},
+ {{0x00, 0x01, 0x02, 0x03, 0x04}, 5, "AAECAwQ="},
+ };
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ char *got = base64_encode(cases[i].bytes, cases[i].n);
+ TEST_ASSERT(strcmp(got, cases[i].out) == 0);
+ free(got);
+ }
+}
+
+int main(void) {
+ test_base64_rfc4648();
+ test_base64_binary_vectors();
+ return TEST_REPORT("base64");
+}
diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c
new file mode 100644
index 0000000..a7db9c4
--- /dev/null
+++ b/tools/server/test_encode.c
@@ -0,0 +1,90 @@
+/* test_encode.c - embedding quantization and the output encodings
+ * (base64_int8, base64_binary, base64 float32) plus float rendering. */
+
+#include "base64.h"
+#include "server_internal.h"
+#include "test_util.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+static void test_quantize_int8_tanh(void) {
+ TEST_ASSERT(quantize_int8_tanh(0.0f) == 0);
+ /* tanh saturates to +-1, scaled by 127. */
+ TEST_ASSERT(quantize_int8_tanh(100.0f) == 127);
+ TEST_ASSERT(quantize_int8_tanh(-100.0f) == -127);
+ /* Symmetry. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == -quantize_int8_tanh(-0.5f));
+ /* round(tanh(0.5) * 127) = round(58.69) = 59. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == 59);
+}
+
+static void test_encode_embedding_int8(void) {
+ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
+ signed char expect[4] = {0, 59, -59, 127};
+ char *got = encode_embedding(emb, 4, "base64_int8");
+ char *want = base64_encode((const unsigned char *)expect, 4);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_encode_embedding_binary(void) {
+ /* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */
+ float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f};
+ unsigned char expect = 0x85; /* bits 0, 2, 7 set */
+ char *got = encode_embedding(emb, 8, "base64_binary");
+ char *want = base64_encode(&expect, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+
+ /* Zero counts as non-negative (sign bit set). */
+ float zeros[8] = {0};
+ unsigned char all = 0xff;
+ got = encode_embedding(zeros, 8, "base64_binary");
+ want = base64_encode(&all, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_encode_embedding_base64_float32(void) {
+ /* OpenAI/DashScope "base64" is base64 of the raw little-endian float32
+ * vector - lossless, unlike the int8 formats. */
+ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
+ /* Reference bytes via memcpy: a float->byte reinterpret cast confuses the
+ * static analyzer's uninitialized-read model. */
+ unsigned char bytes[sizeof emb];
+ memcpy(bytes, emb, sizeof emb);
+ char *want = base64_encode(bytes, sizeof emb);
+ char *got = encode_embedding(emb, 4, "base64");
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_append_embedding_value_float(void) {
+ float emb[1] = {0.5f};
+
+ /* OpenAI-compatible models render the true float32 value. */
+ sbuf b = {0};
+ append_embedding_value(&b, 0, emb, 1, "float", FFWD_API_OPENAI);
+ TEST_ASSERT(strstr(b.ptr, "\"embedding\":[0.5]") != NULL);
+ sbuf_free(&b);
+
+ /* Perplexity-compatible models render the int8-decoded view. */
+ sbuf b2 = {0};
+ append_embedding_value(&b2, 0, emb, 1, "float", FFWD_API_PERPLEXITY);
+ TEST_ASSERT(strstr(b2.ptr, "0.4609375") != NULL);
+ sbuf_free(&b2);
+}
+
+int main(void) {
+ test_quantize_int8_tanh();
+ test_encode_embedding_int8();
+ test_encode_embedding_binary();
+ test_encode_embedding_base64_float32();
+ test_append_embedding_value_float();
+ return TEST_REPORT("encode");
+}
diff --git a/tools/server/test_json.c b/tools/server/test_json.c
new file mode 100644
index 0000000..76a0f45
--- /dev/null
+++ b/tools/server/test_json.c
@@ -0,0 +1,143 @@
+/* test_json.c - request validation, JSON error bodies, and 422 detail
+ * handling (encoding_format, text_type, error escaping, job_set_422). */
+
+#include "server_internal.h"
+#include "test_util.h"
+#include "util.h" /* xstrdup */
+
+#include <stdlib.h>
+#include <string.h>
+
+static const char *TEST_QWEN3_QUERY_INSTRUCT =
+ "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:";
+
+static void test_json_error_body_escaping(void) {
+ size_t len = 0;
+ char *body = json_error_body("line\nquote\" tab\t ctrl\001 slash\\", "bad\ntype", &len);
+ TEST_ASSERT(body != NULL);
+ if (!body)
+ return;
+ TEST_ASSERT(len == strlen(body));
+ TEST_ASSERT(strchr(body, '\n') == NULL);
+ TEST_ASSERT(strchr(body, '\t') == NULL);
+ cJSON *root = cJSON_Parse(body);
+ TEST_ASSERT(root != NULL);
+ if (!root) {
+ free(body);
+ return;
+ }
+ cJSON *err = cJSON_GetObjectItemCaseSensitive(root, "error");
+ cJSON *msg = err ? cJSON_GetObjectItemCaseSensitive(err, "message") : NULL;
+ cJSON *type = err ? cJSON_GetObjectItemCaseSensitive(err, "type") : NULL;
+ if (!msg || !type || !cJSON_IsString(msg) || !cJSON_IsString(type)) {
+ TEST_ASSERT(0 && "error body fields must be strings");
+ cJSON_Delete(root);
+ free(body);
+ return;
+ }
+ TEST_ASSERT(strcmp(msg->valuestring, "line\nquote\" tab\t ctrl\001 slash\\") == 0);
+ TEST_ASSERT(strcmp(type->valuestring, "bad\ntype") == 0);
+ cJSON_Delete(root);
+ free(body);
+}
+
+static void test_job_set_422_replaces_response(void) {
+ job j;
+ memset(&j, 0, sizeof(j));
+ j.response = xstrdup("old response");
+ j.response_len = strlen(j.response);
+ j.status = 500;
+
+ cJSON *detail = cJSON_CreateArray();
+ ve_add(detail, "[\"body\",\"field\"]", "bad value", "value_error");
+ job_set_422(&j, detail);
+ TEST_ASSERT(j.status == 422);
+ TEST_ASSERT(j.response != NULL);
+ if (j.response) {
+ TEST_ASSERT(j.response_len == strlen(j.response));
+ TEST_ASSERT(strstr(j.response, "bad value") != NULL);
+ }
+
+ cJSON_Delete(detail);
+ free(j.response);
+}
+
+static void test_encoding_from_root_family(void) {
+ cJSON *detail = cJSON_CreateArray();
+
+ /* Default encoding follows the explicit serving API. */
+ cJSON *empty = cJSON_CreateObject();
+ TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_OPENAI), "float") == 0);
+ TEST_ASSERT(strcmp(encoding_from_root(empty, detail, FFWD_API_PERPLEXITY), "base64_int8") ==
+ 0);
+ cJSON_Delete(empty);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
+
+ /* OpenAI rejects int8 formats; Perplexity rejects the OpenAI base64. */
+ cJSON *o = cJSON_CreateObject();
+ cJSON_AddStringToObject(o, "encoding_format", "base64_int8");
+ encoding_from_root(o, detail, FFWD_API_OPENAI);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 1);
+ cJSON_Delete(o);
+
+ cJSON *p = cJSON_CreateObject();
+ cJSON_AddStringToObject(p, "encoding_format", "base64");
+ encoding_from_root(p, detail, FFWD_API_PERPLEXITY);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 2);
+ cJSON_Delete(p);
+
+ /* An unknown format is rejected. */
+ cJSON *bad = cJSON_CreateObject();
+ cJSON_AddStringToObject(bad, "encoding_format", "base64_fp16");
+ encoding_from_root(bad, detail, FFWD_API_OPENAI);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 3);
+ cJSON_Delete(bad);
+
+ cJSON_Delete(detail);
+}
+
+static void test_text_type(void) {
+ cJSON *detail = cJSON_CreateArray();
+
+ /* Absent: no-op for both model types, no error. */
+ cJSON *empty = cJSON_CreateObject();
+ TEST_ASSERT(text_type_is_query(empty, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0);
+ TEST_ASSERT(text_type_is_query(empty, detail, NULL) == 0);
+ cJSON_Delete(empty);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
+
+ /* Qwen3: query -> 1, document -> 0, both accepted. */
+ cJSON *q = cJSON_CreateObject();
+ cJSON_AddStringToObject(q, "text_type", "query");
+ TEST_ASSERT(text_type_is_query(q, detail, TEST_QWEN3_QUERY_INSTRUCT) == 1);
+ cJSON_Delete(q);
+ cJSON *d = cJSON_CreateObject();
+ cJSON_AddStringToObject(d, "text_type", "document");
+ TEST_ASSERT(text_type_is_query(d, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0);
+ cJSON_Delete(d);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 0);
+
+ /* Invalid enum value on Qwen3 is reported. */
+ cJSON *bad = cJSON_CreateObject();
+ cJSON_AddStringToObject(bad, "text_type", "passage");
+ TEST_ASSERT(text_type_is_query(bad, detail, TEST_QWEN3_QUERY_INSTRUCT) == 0);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 1);
+ cJSON_Delete(bad);
+
+ /* text_type on a model with no query instruction is rejected. */
+ cJSON *p = cJSON_CreateObject();
+ cJSON_AddStringToObject(p, "text_type", "query");
+ TEST_ASSERT(text_type_is_query(p, detail, NULL) == 0);
+ TEST_ASSERT(cJSON_GetArraySize(detail) == 2);
+ cJSON_Delete(p);
+
+ cJSON_Delete(detail);
+}
+
+int main(void) {
+ test_json_error_body_escaping();
+ test_job_set_422_replaces_response();
+ test_encoding_from_root_family();
+ test_text_type();
+ return TEST_REPORT("json");
+}
diff --git a/tools/server/test_models.c b/tools/server/test_models.c
new file mode 100644
index 0000000..3dce62b
--- /dev/null
+++ b/tools/server/test_models.c
@@ -0,0 +1,26 @@
+/* test_models.c - resolving a served model by its request label. */
+
+#include "server_internal.h"
+#include "test_util.h"
+
+#include <string.h>
+
+static void test_loaded_model_lookup(void) {
+ model_info info;
+ memset(&info, 0, sizeof(info));
+ info.id = "served-label";
+ loaded_model models[1];
+ memset(models, 0, sizeof(models));
+ models[0].info = &info;
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.models = models;
+ s.n_models = 1;
+ TEST_ASSERT(loaded_model_for_label(&s, "served-label") == &models[0]);
+ TEST_ASSERT(loaded_model_for_label(&s, "compiled-name") == NULL);
+}
+
+int main(void) {
+ test_loaded_model_lookup();
+ return TEST_REPORT("models");
+}
diff --git a/tools/server/test_sbuf.c b/tools/server/test_sbuf.c
new file mode 100644
index 0000000..65d0eba
--- /dev/null
+++ b/tools/server/test_sbuf.c
@@ -0,0 +1,35 @@
+/* test_sbuf.c - growable string buffer growth, formatting, and clearing. */
+
+#include "sbuf.h"
+#include "test_util.h"
+
+#include <string.h>
+
+static void test_sbuf_growth_and_formatting(void) {
+ sbuf b = {0};
+ sbuf_puts(&b, "ab");
+ sbuf_putc(&b, 'c');
+ TEST_ASSERT(b.len == 3);
+ TEST_ASSERT(strcmp(b.ptr, "abc") == 0);
+
+ size_t old_cap = b.cap;
+ sbuf_printf(&b, "-%04d-%s", 7, "tail");
+ TEST_ASSERT(strcmp(b.ptr, "abc-0007-tail") == 0);
+ TEST_ASSERT(b.cap == old_cap);
+
+ char big[600];
+ memset(big, 'x', sizeof(big) - 1);
+ big[sizeof(big) - 1] = '\0';
+ sbuf_puts(&b, big);
+ TEST_ASSERT(b.len == strlen("abc-0007-tail") + sizeof(big) - 1);
+ TEST_ASSERT(b.ptr[b.len] == '\0');
+ sbuf_clear(&b);
+ TEST_ASSERT(b.len == 0);
+ TEST_ASSERT(b.ptr && b.ptr[0] == '\0');
+ sbuf_free(&b);
+}
+
+int main(void) {
+ test_sbuf_growth_and_formatting();
+ return TEST_REPORT("sbuf");
+}
diff --git a/tools/server/test_schedule.c b/tools/server/test_schedule.c
new file mode 100644
index 0000000..1f799f1
--- /dev/null
+++ b/tools/server/test_schedule.c
@@ -0,0 +1,73 @@
+/* test_schedule.c - job queue micro-batching: the batch-wait deadline and the
+ * zero-wait fast path through enqueue_job / collect_job_batch. */
+
+#include "server_internal.h"
+#include "test_util.h"
+#include "util.h" /* nstime */
+
+#include <pthread.h>
+#include <stdio.h>
+#include <string.h>
+
+static void test_collect_job_batch_deadline(void) {
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.batch_size = 32;
+ s.batch_wait_us = 1000;
+ pthread_mutex_init(&s.mu, NULL);
+ pthread_cond_init(&s.cv, NULL);
+
+ uint64_t first_ns = nstime() - 5000000u;
+ job first = {.srv = &s, .created_ns = first_ns};
+ job before_deadline = {.srv = &s, .created_ns = first_ns + 500000u};
+ job after_deadline = {.srv = &s, .created_ns = first_ns + 2000000u};
+ snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings");
+ snprintf(before_deadline.path, sizeof(before_deadline.path), "%s", "/v1/embeddings");
+ snprintf(after_deadline.path, sizeof(after_deadline.path), "%s", "/v1/embeddings");
+ enqueue_job(&first);
+ enqueue_job(&before_deadline);
+ enqueue_job(&after_deadline);
+
+ job *batch[3] = {0};
+ TEST_ASSERT(collect_job_batch(&s, batch, 3) == 2);
+ TEST_ASSERT(batch[0] == &first);
+ TEST_ASSERT(batch[1] == &before_deadline);
+ TEST_ASSERT(s.job_head == &after_deadline);
+ TEST_ASSERT(s.job_tail == &after_deadline);
+
+ s.job_head = NULL;
+ s.job_tail = NULL;
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+}
+
+static void test_collect_job_batch_zero_wait(void) {
+ http_server s;
+ memset(&s, 0, sizeof(s));
+ s.batch_size = 32;
+ pthread_mutex_init(&s.mu, NULL);
+ pthread_cond_init(&s.cv, NULL);
+
+ job first = {.srv = &s, .created_ns = nstime()};
+ job second = {.srv = &s, .created_ns = nstime()};
+ snprintf(first.path, sizeof(first.path), "%s", "/v1/embeddings");
+ snprintf(second.path, sizeof(second.path), "%s", "/v1/embeddings");
+ enqueue_job(&first);
+ enqueue_job(&second);
+
+ job *batch[2] = {0};
+ TEST_ASSERT(collect_job_batch(&s, batch, 2) == 2);
+ TEST_ASSERT(batch[0] == &first);
+ TEST_ASSERT(batch[1] == &second);
+ TEST_ASSERT(s.job_head == NULL);
+ TEST_ASSERT(s.job_tail == NULL);
+
+ pthread_cond_destroy(&s.cv);
+ pthread_mutex_destroy(&s.mu);
+}
+
+int main(void) {
+ test_collect_job_batch_deadline();
+ test_collect_job_batch_zero_wait();
+ return TEST_REPORT("schedule");
+}
diff --git a/tools/server/test_util.h b/tools/server/test_util.h
new file mode 100644
index 0000000..c6b6a13
--- /dev/null
+++ b/tools/server/test_util.h
@@ -0,0 +1,26 @@
+#ifndef FFWD_SERVER_TEST_UTIL_H
+#define FFWD_SERVER_TEST_UTIL_H
+
+/* Shared harness for the ffwd-server unit tests. TEST_ASSERT records failures
+ * (rather than aborting) so a single run reports every failure, not just the
+ * first. Each test file includes this, runs its checks, and ends main with
+ * `return TEST_REPORT("label");`, which prints a summary and yields the exit
+ * code (0 when clean, 1 otherwise). */
+
+#include <stdio.h>
+
+static int test_failures = 0;
+
+#define TEST_ASSERT(cond) \
+ do { \
+ if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+ test_failures++; \
+ } \
+ } while (0)
+
+#define TEST_REPORT(label) \
+ (test_failures ? (fprintf(stderr, "%s: %d failure(s)\n", (label), test_failures), 1) \
+ : (printf("ok: %s\n", (label)), 0))
+
+#endif /* FFWD_SERVER_TEST_UTIL_H */
diff --git a/src/server_util.c b/tools/server/util.c
index bc20804..1ec4a2a 100644
--- a/src/server_util.c
+++ b/tools/server/util.c
@@ -1,4 +1,4 @@
-#include "server_util.h"
+#include "util.h"
#include <stdarg.h>
#include <stdio.h>
@@ -8,7 +8,7 @@
#include <time.h>
void die_oom(void) {
- fprintf(stderr, "embed-server: out of memory\n");
+ fprintf(stderr, "ffwd-server: out of memory\n");
exit(1);
}
diff --git a/src/server_util.h b/tools/server/util.h
index 955903c..3175cd0 100644
--- a/src/server_util.h
+++ b/tools/server/util.h
@@ -1,21 +1,21 @@
-#ifndef EMBED_SERVER_UTIL_H
-#define EMBED_SERVER_UTIL_H
+#ifndef FFWD_SERVER_UTIL_H
+#define FFWD_SERVER_UTIL_H
#include <stdint.h>
#include <stddef.h>
/* Die-on-OOM allocation, monotonic clocks, and timestamped logging shared by
- * the embed-server translation units. The x* allocators abort the process on
+ * the ffwd-server translation units. The x* allocators abort the process on
* failure rather than returning NULL, so callers never null-check them; the
* attributes below carry that contract across translation-unit boundaries so
* the static analyzer does not flag the (correctly) unchecked uses. */
#if defined(__GNUC__) || defined(__clang__)
-#define SERVER_NORETURN __attribute__((noreturn))
-#define SERVER_RETURNS_NONNULL __attribute__((returns_nonnull))
+# define SERVER_NORETURN __attribute__((noreturn))
+# define SERVER_RETURNS_NONNULL __attribute__((returns_nonnull))
#else
-#define SERVER_NORETURN
-#define SERVER_RETURNS_NONNULL
+# define SERVER_NORETURN
+# define SERVER_RETURNS_NONNULL
#endif
SERVER_NORETURN void die_oom(void);
@@ -33,4 +33,4 @@ double ns_to_ms(uint64_t ns);
/* printf-style line logger: timestamp prefix, newline appended, to stderr. */
void server_log(const char *fmt, ...) __attribute__((format(printf, 1, 2)));
-#endif /* EMBED_SERVER_UTIL_H */
+#endif /* FFWD_SERVER_UTIL_H */