diff options
| -rw-r--r-- | Makefile | 187 | ||||
| -rw-r--r-- | bench/bench.h | 2 | ||||
| -rw-r--r-- | bench/bench_kernels.c | 26 | ||||
| -rw-r--r-- | bench/bench_model.c | 4 | ||||
| -rw-r--r-- | bench/bench_tokenizer.c | 24 | ||||
| -rw-r--r-- | qwen_kernels_impl.h | 53 | ||||
| -rw-r--r-- | src/build.h (renamed from embed_build.h) | 0 | ||||
| -rw-r--r-- | src/cli.c (renamed from embed_cli.c) | 33 | ||||
| -rw-r--r-- | src/config.c (renamed from embed_config.c) | 2 | ||||
| -rw-r--r-- | src/config.h (renamed from embed_config.h) | 0 | ||||
| -rw-r--r-- | src/cuda.cu (renamed from embed_cuda.cu) | 6 | ||||
| -rw-r--r-- | src/cuda.h (renamed from embed_cuda.h) | 0 | ||||
| -rw-r--r-- | src/embed.c (renamed from embed.c) | 69 | ||||
| -rw-r--r-- | src/embed.h (renamed from embed.h) | 1 | ||||
| -rw-r--r-- | src/internal.h (renamed from embed_internal.h) | 2 | ||||
| -rw-r--r-- | src/kernels.c (renamed from qwen_kernels.c) | 172 | ||||
| -rw-r--r-- | src/kernels.h (renamed from qwen_kernels.h) | 60 | ||||
| -rw-r--r-- | src/kernels_avx.c (renamed from qwen_kernels_avx.c) | 16 | ||||
| -rw-r--r-- | src/kernels_generic.c (renamed from qwen_kernels_generic.c) | 14 | ||||
| -rw-r--r-- | src/kernels_impl.h | 53 | ||||
| -rw-r--r-- | src/kernels_neon.c (renamed from qwen_kernels_neon.c) | 14 | ||||
| -rw-r--r-- | src/mlx.c (renamed from embed_mlx.c) | 6 | ||||
| -rw-r--r-- | src/mlx.h (renamed from embed_mlx.h) | 0 | ||||
| -rw-r--r-- | src/safetensors.c (renamed from qwen_safetensors.c) | 2 | ||||
| -rw-r--r-- | src/safetensors.h (renamed from qwen_safetensors.h) | 8 | ||||
| -rw-r--r-- | src/server.c (renamed from embed_server.c) | 45 | ||||
| -rw-r--r-- | src/server.h (renamed from embed_server.h) | 0 | ||||
| -rw-r--r-- | src/tokenizer_bpe.c (renamed from qwen_tokenizer.c) | 86 | ||||
| -rw-r--r-- | src/tokenizer_bpe.h (renamed from qwen_tokenizer.h) | 36 | ||||
| -rw-r--r-- | src/tokenizer_unicode.h (renamed from wordpiece_unicode.h) | 0 | ||||
| -rw-r--r-- | src/tokenizer_wordpiece.c (renamed from wordpiece_tokenizer.c) | 4 | ||||
| -rw-r--r-- | src/tokenizer_wordpiece.h (renamed from wordpiece_tokenizer.h) | 2 | ||||
| -rw-r--r-- | tests/test_kernels.c | 96 | ||||
| -rw-r--r-- | tests/test_late.c | 12 | ||||
| -rw-r--r-- | tests/test_safetensors.c | 2 | ||||
| -rw-r--r-- | tests/test_server.c | 30 | ||||
| -rw-r--r-- | tests/test_tokenizer.c | 122 | ||||
| -rw-r--r-- | tests/test_wordpiece.c | 2 | ||||
| -rw-r--r-- | tests/test_workspace.c | 14 |
39 files changed, 580 insertions, 625 deletions
@@ -8,8 +8,8 @@ BUILD_OS ?= $(shell uname -s | tr '[:upper:]' '[:lower:]') BUILD_ARCH ?= $(shell uname -m) # Version and build identity compiled into the embed and embed-server binaries. -# Applied on the embed_cli.o and embed_server.o recipes. VERSION is set for -# release builds; otherwise the C fallbacks apply. +# Applied on the cli.o and server.o recipes. VERSION is set for release builds; +# otherwise the C fallbacks apply. VERSION_CFLAGS = \ -DEMBED_BUILD_DATE=\"$(BUILD_DATE)\" \ -DEMBED_BUILD_COMMIT=\"$(BUILD_COMMIT)\" \ @@ -56,11 +56,11 @@ BACKEND_LDFLAGS = -framework Accelerate -framework Metal -framework Foundation \ -L$(MLXC_PREFIX)/lib -lmlxc \ -L$(MLX_PREFIX)/lib -lmlx \ -Wl,-rpath,$(MLX_PREFIX)/lib -Wl,-rpath,$(MLXC_PREFIX)/lib -EXTRA_OBJS = embed_mlx.o +EXTRA_OBJS = src/mlx.o else ifeq ($(BACKEND),cuda) BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include BACKEND_LDFLAGS = -L$(CUDA_HOME)/lib64 -lopenblas -lcudart -lcublas -EXTRA_OBJS = embed_cuda.o +EXTRA_OBJS = src/cuda.o else ifeq ($(UNAME_S),Darwin) BACKEND_CFLAGS = -DUSE_BLAS -DACCELERATE_NEW_LAPACK BACKEND_LDFLAGS = -framework Accelerate @@ -69,35 +69,30 @@ BACKEND_CFLAGS = -DUSE_BLAS -DUSE_OPENBLAS -I/usr/include/openblas BACKEND_LDFLAGS = -lopenblas endif -CFLAGS = $(CFLAGS_BASE) $(BACKEND_CFLAGS) +# -Isrc so frontends, tests, and benches resolve the public/internal headers +# now that every source lives under src/. +CFLAGS = $(CFLAGS_BASE) $(BACKEND_CFLAGS) -Isrc LDFLAGS = -lm -lpthread $(BACKEND_LDFLAGS) -# Source files -SRCS = embed.c \ - embed_config.c \ - qwen_kernels.c \ - qwen_kernels_generic.c \ - qwen_kernels_neon.c \ - qwen_kernels_avx.c \ - qwen_tokenizer.c \ - wordpiece_tokenizer.c \ - qwen_safetensors.c \ - deps/ae/ae.c \ - deps/ae/anet.c \ - deps/ae/monotonic.c +# Source groups. CORE_SRCS is the engine the tests and benches reuse; keeping it +# in one variable stops the per-test source lists from drifting apart. +KERNEL_SRCS = src/kernels.c src/kernels_generic.c src/kernels_neon.c src/kernels_avx.c +TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c +CORE_SRCS = src/embed.c src/config.c src/safetensors.c $(KERNEL_SRCS) -MLX_SRCS = embed_mlx.c -CUDA_SRCS = embed_cuda.cu +SRCS = $(CORE_SRCS) $(TOKENIZER_SRCS) \ + deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c + +MLX_SRCS = src/mlx.c +CUDA_SRCS = src/cuda.cu OBJS = $(SRCS:.c=.o) MLX_OBJS = $(MLX_SRCS:.c=.o) +HEADERS = $(wildcard src/*.h) TARGET = embed SERVER_TARGET = embed-server LIB = libembed.a -KERNEL_SRCS = qwen_kernels.c qwen_kernels_generic.c \ - qwen_kernels_neon.c qwen_kernels_avx.c - ifeq ($(shell uname -s),Darwin) SHARED_LIB = libembed.dylib SHARED_FLAGS = -dynamiclib @@ -137,11 +132,28 @@ else gpu: cuda endif +# ============================================================================= +# Compile rules +# ============================================================================= +# Generic rule for the plain core/kernel/tokenizer/safetensors/mlx objects. +# Sibling headers in src/ resolve via the quote-include search; -Isrc (in +# CFLAGS) makes it explicit. -arch=native for CUDA targets the GPU at build. +src/%.o: src/%.c $(HEADERS) + $(CC) $(CFLAGS) -c -o $@ $< + # -fPIC so the same object links into both binaries and the shared library. -# -arch=native targets the GPU present at build time (we always build CUDA -# on the machine that runs it), avoiding first-load PTX JIT. -embed_cuda.o: embed_cuda.cu embed_cuda.h embed_internal.h embed.h - $(NVCC) -O3 -std=c++17 -arch=native -Xcompiler -fPIC,-fvisibility=hidden -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $< +src/cuda.o: src/cuda.cu src/cuda.h src/internal.h src/embed.h + $(NVCC) -O3 -std=c++17 -arch=native -Xcompiler -fPIC,-fvisibility=hidden -DUSE_BLAS -DUSE_OPENBLAS -DUSE_CUDA -Isrc -I/usr/include/openblas -I$(CUDA_HOME)/include -x cu -c -o $@ $< + +# server.c reaches deps/ae via "deps/ae/ae.h", so it needs -I. (repo root). +src/server.o: src/server.c $(HEADERS) deps/ae/ae.h deps/ae/anet.h + $(CC) $(CFLAGS) $(VERSION_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae -c -o $@ $< + +src/cli.o: src/cli.c $(HEADERS) + $(CC) $(CFLAGS) $(VERSION_CFLAGS) -c -o $@ $< + +deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h + $(CC) $(CFLAGS) -Ideps/ae -c -o $@ $< # ============================================================================= # Shared library (built by every backend target from the same objects) @@ -149,6 +161,17 @@ embed_cuda.o: embed_cuda.cu embed_cuda.h embed_internal.h embed.h $(SHARED_LIB): $(OBJS) $(EXTRA_OBJS) $(CC) $(CFLAGS) $(SHARED_FLAGS) -o $@ $^ $(LDFLAGS) +# ============================================================================= +# Static library and binary link +# ============================================================================= +$(LIB): $(OBJS) $(EXTRA_OBJS) + $(AR) rcs $@ $^ + +$(TARGET): $(LIB) src/cli.o + $(CC) $(CFLAGS) -o $@ src/cli.o $(LIB) $(LDFLAGS) + +$(SERVER_TARGET): $(LIB) src/server.o + $(CC) $(CFLAGS) -o $@ src/server.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) # ============================================================================= # Tests (no model files required) @@ -162,29 +185,23 @@ TEST_BLAS_LDFLAGS = -lopenblas endif # Source lists shared by test and coverage targets so the two cannot drift. -TEST_CC_FLAGS = -Wall -Wextra -O2 -I. +TEST_CC_FLAGS = -Wall -Wextra -O2 -Isrc TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS) -TEST_TOKENIZER_SRCS = tests/test_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS) -TEST_WORDPIECE_SRCS = tests/test_wordpiece.c wordpiece_tokenizer.c -TEST_SAFETENSORS_SRCS = tests/test_safetensors.c qwen_safetensors.c -TEST_BF16_SRCS = tests/test_bf16_model.c embed.c embed_config.c qwen_safetensors.c \ - $(KERNEL_SRCS) -TEST_QWEN3_SRCS = tests/test_qwen3.c embed.c embed_config.c qwen_safetensors.c \ - $(KERNEL_SRCS) -TEST_QWEN2_SRCS = tests/test_qwen2.c embed.c embed_config.c qwen_safetensors.c \ - $(KERNEL_SRCS) -TEST_CLS_SRCS = tests/test_cls.c embed.c embed_config.c qwen_safetensors.c \ - $(KERNEL_SRCS) -TEST_WORKSPACE_SRCS = tests/test_workspace.c embed.c embed_config.c qwen_tokenizer.c \ - qwen_safetensors.c $(KERNEL_SRCS) -TEST_LATE_SRCS = tests/test_late.c embed.c embed_config.c qwen_tokenizer.c \ - qwen_safetensors.c $(KERNEL_SRCS) -TEST_SERVER_SRCS = tests/test_server.c embed.c embed_config.c \ - qwen_tokenizer.c wordpiece_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) \ +TEST_TOKENIZER_SRCS = tests/test_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS) +TEST_WORDPIECE_SRCS = tests/test_wordpiece.c src/tokenizer_wordpiece.c +TEST_SAFETENSORS_SRCS = tests/test_safetensors.c src/safetensors.c +TEST_BF16_SRCS = tests/test_bf16_model.c $(CORE_SRCS) +TEST_QWEN3_SRCS = tests/test_qwen3.c $(CORE_SRCS) +TEST_QWEN2_SRCS = tests/test_qwen2.c $(CORE_SRCS) +TEST_CLS_SRCS = tests/test_cls.c $(CORE_SRCS) +TEST_WORKSPACE_SRCS = tests/test_workspace.c $(CORE_SRCS) src/tokenizer_bpe.c +TEST_LATE_SRCS = tests/test_late.c $(CORE_SRCS) src/tokenizer_bpe.c +# test_server.c #includes ../src/server.c, so server is not a separate object +# here; it needs -I. for server.c's "deps/ae/ae.h" plus the ae sources. +TEST_SERVER_SRCS = tests/test_server.c $(CORE_SRCS) $(TOKENIZER_SRCS) \ deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c # The CLI check builds the real CLI (CPU backend) plus a driver that runs it. -TEST_CLI_BIN_SRCS = embed_cli.c embed.c embed_config.c \ - qwen_tokenizer.c wordpiece_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) +TEST_CLI_BIN_SRCS = src/cli.c $(CORE_SRCS) $(TOKENIZER_SRCS) test: $(CC) $(TEST_CC_FLAGS) -o tests/test_kernels_generic \ @@ -222,7 +239,7 @@ test: $(TEST_CLI_BIN_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CC) $(TEST_CC_FLAGS) -o tests/test_cli tests/test_cli.c -lm ./tests/test_cli ./tests/cli_under_test - $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -Ideps/ae \ + $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae \ -o tests/test_server $(TEST_SERVER_SRCS) \ -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CJSON_LDFLAGS) ./tests/test_server @@ -237,15 +254,12 @@ test-safetensors: $(TEST_SAFETENSORS_SRCS) -lm -lpthread ./tests/test_safetensors - # ============================================================================= # Test-suite line coverage (requires clang + llvm-cov/llvm-profdata. Writes a # per-file text summary to stdout and an HTML report to coverage/html/index.html.) # ============================================================================= COV_DIR = coverage # Source-based coverage; -O0 so no line is folded away by the optimizer. -# The "." compilation dir keeps report paths relative to the repo root; -# run llvm-cov from the repo root so sources resolve. COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \ -fcoverage-compilation-dir=. COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \ @@ -297,7 +311,7 @@ coverage: $(TEST_CLI_BIN_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CC) $(TEST_CC_FLAGS) -o tests/test_cli tests/test_cli.c -lm LLVM_PROFILE_FILE=$(COV_DIR)/cli_%p.profraw ./tests/test_cli ./tests/cli_under_test - $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -Ideps/ae \ + $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae \ -o tests/test_server $(TEST_SERVER_SRCS) \ -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CJSON_LDFLAGS) LLVM_PROFILE_FILE=$(COV_DIR)/server.profraw ./tests/test_server @@ -317,8 +331,8 @@ coverage: @echo "HTML report: $(COV_DIR)/html/index.html" bench-tokenizer: - $(CC) -Wall -Wextra -O3 -march=native -I. -o bench/bench_tokenizer \ - bench/bench_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS) -lm -lpthread + $(CC) -Wall -Wextra -O3 -march=native -Isrc -o bench/bench_tokenizer \ + bench/bench_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS) -lm -lpthread @echo "run: bench/bench_tokenizer <model_dir> [runs]" # ============================================================================= @@ -329,7 +343,7 @@ bench-tokenizer: # Dirtiness ignores .gitignore so local-only ignore entries do not mark # every record as -dirty. BENCH_STAMP = $(shell git rev-parse --short HEAD 2>/dev/null || echo nogit)$(shell git diff --quiet -- ':!.gitignore' 2>/dev/null || echo -dirty)-$(shell date +%Y%m%d-%H%M%S) -BENCH_CC_FLAGS = -Wall -Wextra $(CFLAGS_BASE) $(TEST_BLAS_CFLAGS) -I. +BENCH_CC_FLAGS = -Wall -Wextra $(CFLAGS_BASE) $(TEST_BLAS_CFLAGS) -Isrc bench: $(CC) $(BENCH_CC_FLAGS) -o bench/bench_kernels \ @@ -341,7 +355,7 @@ bench: bench-model: @test -n "$(MODEL_DIR)" || { echo "usage: make bench-model MODEL_DIR=/path/to/model-dir"; exit 1; } $(CC) $(BENCH_CC_FLAGS) -o bench/bench_model \ - bench/bench_model.c embed.c embed_config.c qwen_safetensors.c $(KERNEL_SRCS) \ + bench/bench_model.c $(CORE_SRCS) \ -lm -lpthread $(TEST_BLAS_LDFLAGS) @mkdir -p bench/results ./bench/bench_model "$(MODEL_DIR)" --json bench/results/model-$(BENCH_STAMP).json @@ -351,74 +365,19 @@ bench-model: # ============================================================================= SANITIZE ?= address,undefined -debug: CFLAGS = -Wall -Wextra -g -O0 -DDEBUG -fsanitize=$(SANITIZE) +debug: CFLAGS = -Wall -Wextra -g -O0 -DDEBUG -Isrc -fsanitize=$(SANITIZE) debug: LDFLAGS = -lm -lpthread -fsanitize=$(SANITIZE) debug: $(MAKE) clean $(MAKE) $(TARGET) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" # ============================================================================= -# Static library and binary link -# ============================================================================= -$(LIB): $(OBJS) $(EXTRA_OBJS) - $(AR) rcs $@ $^ - -$(TARGET): $(LIB) embed_cli.o - $(CC) $(CFLAGS) -o $@ embed_cli.o $(LIB) $(LDFLAGS) - -$(SERVER_TARGET): $(LIB) embed_server.o - $(CC) $(CFLAGS) -o $@ embed_server.o $(LIB) $(LDFLAGS) $(CJSON_LDFLAGS) - -# ============================================================================= -# Compile rules -# ============================================================================= -embed.o: embed.c embed.h embed_config.h qwen_kernels.h qwen_safetensors.h - $(CC) $(CFLAGS) -Ideps/ae -c -o $@ $< - -embed_server.o: embed_server.c embed_server.h embed_build.h embed.h embed_mlx.h qwen_tokenizer.h wordpiece_tokenizer.h deps/ae/ae.h deps/ae/anet.h - $(CC) $(CFLAGS) $(VERSION_CFLAGS) $(CJSON_CFLAGS) -Ideps/ae -c -o $@ $< - -embed_mlx.o: embed_mlx.c embed_mlx.h embed_config.h embed.h qwen_safetensors.h - $(CC) $(CFLAGS) -c -o $@ $< - -embed_cli.o: embed_cli.c embed_build.h embed.h qwen_kernels.h qwen_tokenizer.h wordpiece_tokenizer.h - $(CC) $(CFLAGS) $(VERSION_CFLAGS) -Ideps/ae -c -o $@ $< - - -qwen_kernels.o: qwen_kernels.c qwen_kernels.h qwen_kernels_impl.h - $(CC) $(CFLAGS) -c -o $@ $< - -qwen_kernels_generic.o: qwen_kernels_generic.c qwen_kernels_impl.h - $(CC) $(CFLAGS) -c -o $@ $< - -qwen_kernels_neon.o: qwen_kernels_neon.c qwen_kernels_impl.h - $(CC) $(CFLAGS) -c -o $@ $< - -qwen_kernels_avx.o: qwen_kernels_avx.c qwen_kernels_impl.h - $(CC) $(CFLAGS) -c -o $@ $< - -qwen_tokenizer.o: qwen_tokenizer.c qwen_tokenizer.h qwen_kernels.h - $(CC) $(CFLAGS) -c -o $@ $< - -wordpiece_tokenizer.o: wordpiece_tokenizer.c wordpiece_tokenizer.h wordpiece_unicode.h - $(CC) $(CFLAGS) -c -o $@ $< - -qwen_safetensors.o: qwen_safetensors.c qwen_safetensors.h - $(CC) $(CFLAGS) -c -o $@ $< - -embed_config.o: embed_config.c embed_config.h embed.h - $(CC) $(CFLAGS) -c -o $@ $< - -deps/ae/%.o: deps/ae/%.c deps/ae/ae.h deps/ae/anet.h deps/ae/monotonic.h - $(CC) $(CFLAGS) -Ideps/ae -c -o $@ $< - -# ============================================================================= clean: - rm -f $(OBJS) embed_mlx.o embed_cuda.o embed_cli.o embed_server.o \ + rm -f src/*.o deps/ae/*.o \ $(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \ tests/test_kernels_generic tests/test_kernels_blas \ tests/test_safetensors tests/test_bf16_model tests/test_server \ tests/test_qwen3 tests/test_qwen2 \ - tests/test_tokenizer tests/test_workspace tests/test_cli tests/test_late \ + tests/test_tokenizer tests/test_wordpiece tests/test_workspace tests/test_cli tests/test_late \ tests/cli_under_test bench/bench_tokenizer bench/bench_kernels bench/bench_model rm -rf $(COV_DIR) diff --git a/bench/bench.h b/bench/bench.h index 6089174..1d2b8f2 100644 --- a/bench/bench.h +++ b/bench/bench.h @@ -8,7 +8,7 @@ * float *x = make_input(); // setup, not measured * bench_begin(b); * for (long i = 0; i < b->n; i++) - * bench_sink += qwen_dot_f32(x, x, 1024); + * bench_sink += embed_dot_f32(x, x, 1024); * free(x); * } * diff --git a/bench/bench_kernels.c b/bench/bench_kernels.c index 4639d18..2b2fe47 100644 --- a/bench/bench_kernels.c +++ b/bench/bench_kernels.c @@ -8,11 +8,11 @@ */ #include "bench.h" -#include "qwen_kernels.h" +#include "kernels.h" #include <stdint.h> -int qwen_verbose = 0; /* defined by embed.c in full builds */ +int embed_verbose = 0; /* defined by embed.c in full builds */ enum { HIDDEN = 1024, @@ -71,7 +71,7 @@ static void bm_linear_f32(bench_state_t *b, int seq, int in, int out) { float *y = alloc_f32((size_t)seq * out, 3); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_linear_nobias(y, x, w, seq, in, out); + embed_linear_nobias(y, x, w, seq, in, out); bench_sink += y[0]; free(y); free(w); @@ -91,7 +91,7 @@ static void bm_linear_bf16(bench_state_t *b, int seq, int in, int out) { float *y = alloc_f32((size_t)seq * out, 6); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_linear_nobias_bf16(y, x, w, seq, in, out); + embed_linear_nobias_bf16(y, x, w, seq, in, out); bench_sink += y[0]; free(y); free(w); @@ -111,7 +111,7 @@ static void bm_qkv_bf16(bench_state_t *b) { float *v = alloc_f32((size_t)SEQ * KV_DIM, 13); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM); + embed_linear_nobias_bf16_qkv(q, k, v, x, wq, wk, wv, SEQ, HIDDEN, Q_DIM, KV_DIM); bench_sink += q[0] + k[0] + v[0]; free(v); free(k); @@ -128,7 +128,7 @@ static void bm_bf16_widen(bench_state_t *b) { float *dst = alloc_f32(n, 15); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_bf16_to_f32_buf(dst, src, n); + embed_bf16_to_f32_buf(dst, src, n); bench_sink += dst[0]; free(dst); free(src); @@ -143,7 +143,7 @@ static void bm_attn_scores(bench_state_t *b) { float *s = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 18); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ); + embed_matmul_t(s, qm, km, ATTN_SEQ, HEAD_DIM, ATTN_SEQ); bench_sink += s[0]; free(s); free(km); @@ -154,7 +154,7 @@ static void bm_softmax(bench_state_t *b) { float *x = alloc_f32((size_t)ATTN_SEQ * ATTN_SEQ, 19); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_softmax(x, ATTN_SEQ, ATTN_SEQ); + embed_softmax(x, ATTN_SEQ, ATTN_SEQ); bench_sink += x[0]; free(x); } @@ -165,7 +165,7 @@ static void bm_rope(bench_state_t *b) { float *sinv = alloc_f32((size_t)ATTN_SEQ * HEAD_DIM, 22); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM); + embed_apply_rope_neox(x, cosv, sinv, ATTN_SEQ, Q_DIM / HEAD_DIM, HEAD_DIM); bench_sink += x[0]; free(sinv); free(cosv); @@ -180,7 +180,7 @@ static void bm_rms_norm(bench_state_t *b) { float *y = alloc_f32((size_t)SEQ * HIDDEN, 25); bench_begin(b); for (long i = 0; i < b->n; i++) - qwen_rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f); + embed_rms_norm(y, x, w, SEQ, HIDDEN, 1e-6f); bench_sink += y[0]; free(y); free(w); @@ -197,7 +197,7 @@ static void bm_silu_mul(bench_state_t *b) { bench_begin(b); for (long i = 0; i < b->n; i++) { memcpy(gate, gate0, n * sizeof(float)); - qwen_silu_mul_inplace(gate, up, (int)n); + embed_silu_mul_inplace(gate, up, (int)n); } bench_sink += gate[0]; free(up); @@ -210,7 +210,7 @@ static void bm_dot(bench_state_t *b) { float *y = alloc_f32(HIDDEN, 29); bench_begin(b); for (long i = 0; i < b->n; i++) - bench_sink += qwen_dot_f32(x, y, HIDDEN); + bench_sink += embed_dot_f32(x, y, HIDDEN); free(y); free(x); } @@ -240,7 +240,7 @@ int main(int argc, char **argv) { } if (threads < 1) threads = 1; - qwen_set_threads(threads); + embed_set_threads(threads); char meta[128]; #ifdef USE_BLAS diff --git a/bench/bench_model.c b/bench/bench_model.c index 5cff015..b2d4769 100644 --- a/bench/bench_model.c +++ b/bench/bench_model.c @@ -11,7 +11,7 @@ #include "bench.h" #include "embed.h" -#include "qwen_kernels.h" +#include "kernels.h" static embed_model_t *g_model; static embed_workspace_t *g_ws; @@ -67,7 +67,7 @@ int main(int argc, char **argv) { } if (threads < 1) threads = 1; - qwen_set_threads(threads); + embed_set_threads(threads); g_model = embed_model_load(argv[1]); if (!g_model) { diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index fb6d37a..bf4d0b7 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -1,14 +1,14 @@ /* bench/bench_tokenizer.c - tokenizer encode-path benchmark. * Build via `make bench-tokenizer`; needs a model dir (vocab.json). */ -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include <stdio.h> #include <stdlib.h> #include <string.h> #include <time.h> -int qwen_verbose = 0; +int embed_verbose = 0; static double now_ms(void) { struct timespec ts; @@ -45,8 +45,8 @@ int main(int argc, char **argv) { int n_texts = argc - 3; char **texts = argv + 3; - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab); - qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new(); + embed_tokenizer_t *tok = embed_tokenizer_load(vocab); + embed_tokenizer_workspace_t *ws = embed_tokenizer_workspace_new(); if (!tok || !ws) { fprintf(stderr, "failed to initialize tokenizer\n"); goto cleanup; @@ -61,13 +61,13 @@ int main(int argc, char **argv) { int tokens_per_pass = 0; for (int i = 0; i < n_texts; i++) { - gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]); + gold[i] = embed_tokenizer_encode(tok, texts[i], &gold_n[i]); if (!gold[i] || gold_n[i] <= 0) goto cleanup; tokens_per_pass += gold_n[i]; int n_ws = 0; - int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws); + int *ids_ws = embed_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws); if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) { free(ids_ws); goto cleanup; @@ -81,7 +81,7 @@ int main(int argc, char **argv) { if (!bufs[i]) goto cleanup; int n_into = 0; - if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 || + if (embed_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 || !same_ids(gold[i], gold_n[i], bufs[i], n_into)) goto cleanup; } @@ -91,7 +91,7 @@ int main(int argc, char **argv) { for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int *ids = qwen_tokenizer_encode(tok, texts[i], &n); + int *ids = embed_tokenizer_encode(tok, texts[i], &n); if (!ids || n != gold_n[i]) { free(ids); goto cleanup; @@ -107,7 +107,7 @@ int main(int argc, char **argv) { for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n); + int *ids = embed_tokenizer_encode_with_workspace(tok, ws, texts[i], &n); if (!ids || n != gold_n[i]) { free(ids); goto cleanup; @@ -123,7 +123,7 @@ int main(int argc, char **argv) { for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n); + int rc = embed_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n); if (rc != 0 || n != gold_n[i]) goto cleanup; into_tokens += n; @@ -152,7 +152,7 @@ cleanup: free(gold_n); free(bufs); free(caps); - qwen_tokenizer_workspace_free(ws); - qwen_tokenizer_free(tok); + embed_tokenizer_workspace_free(ws); + embed_tokenizer_free(tok); return status; } diff --git a/qwen_kernels_impl.h b/qwen_kernels_impl.h deleted file mode 100644 index 217f1e8..0000000 --- a/qwen_kernels_impl.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * qwen_kernels_impl.h - internal architecture dispatch for hot kernels - */ - -#ifndef QWEN_KERNELS_IMPL_H -#define QWEN_KERNELS_IMPL_H - -#include <stdint.h> - -void qwen_bf16_matvec_fused_generic( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -float qwen_dot_f32_generic(const float *a, const float *b, int n); -void qwen_vec_scale_inplace_generic(float *dst, float scale, int n); -void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); -void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n); - -#ifdef __ARM_NEON -void qwen_bf16_matvec_fused_neon( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -float qwen_dot_f32_neon(const float *a, const float *b, int n); -void qwen_vec_scale_inplace_neon(float *dst, float scale, int n); -void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); -void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n); - -#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_neon -#define qwen_dot_f32_impl qwen_dot_f32_neon -#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_neon -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_neon -#define qwen_vec_scale_add_impl qwen_vec_scale_add_neon - -#elif defined(__AVX2__) && defined(__FMA__) -void qwen_bf16_matvec_fused_avx( - float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); -float qwen_dot_f32_avx(const float *a, const float *b, int n); -void qwen_vec_scale_inplace_avx(float *dst, float scale, int n); -void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); -void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n); - -#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_avx -#define qwen_dot_f32_impl qwen_dot_f32_avx -#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_avx -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_avx -#define qwen_vec_scale_add_impl qwen_vec_scale_add_avx - -#else -#define qwen_bf16_matvec_fused_impl qwen_bf16_matvec_fused_generic -#define qwen_dot_f32_impl qwen_dot_f32_generic -#define qwen_vec_scale_inplace_impl qwen_vec_scale_inplace_generic -#define qwen_vec_axpy_inplace_impl qwen_vec_axpy_inplace_generic -#define qwen_vec_scale_add_impl qwen_vec_scale_add_generic -#endif - -#endif /* QWEN_KERNELS_IMPL_H */ diff --git a/embed_build.h b/src/build.h index dc68405..dc68405 100644 --- a/embed_build.h +++ b/src/build.h @@ -1,16 +1,16 @@ /* embed_cli.c - embed command-line tool */ #include "embed.h" -#include "embed_build.h" -#include "qwen_kernels.h" -#include "qwen_tokenizer.h" -#include "wordpiece_tokenizer.h" +#include "build.h" +#include "kernels.h" +#include "tokenizer_bpe.h" +#include "tokenizer_wordpiece.h" #ifdef USE_MLX -#include "embed_mlx.h" +#include "mlx.h" #endif #ifdef USE_CUDA -#include "embed_cuda.h" +#include "cuda.h" #endif #include <stdio.h> @@ -96,8 +96,8 @@ typedef struct { #ifdef USE_CUDA embed_cuda_ctx_t *cuda_ctx; #endif - qwen_tokenizer_t *tok; - qwen_tokenizer_workspace_t *tok_ws; + embed_tokenizer_t *tok; + embed_tokenizer_workspace_t *tok_ws; /* BERT-family WordPiece tokenizer, selected by a vocab.txt probe at load. * When wp_tok is set, tokenize_text wraps the ids with [CLS]/[SEP] and the * Qwen byte-level BPE fields above stay NULL. */ @@ -111,8 +111,8 @@ typedef struct { } engine_t; static void engine_free_tokenizers(engine_t *e) { - qwen_tokenizer_workspace_free(e->tok_ws); - qwen_tokenizer_free(e->tok); + embed_tokenizer_workspace_free(e->tok_ws); + embed_tokenizer_free(e->tok); wordpiece_workspace_free(e->wp_tok_ws); wordpiece_tokenizer_free(e->wp_tok); } @@ -158,7 +158,7 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) { out->ids = ids; out->n_tokens = n + 2; } else { - out->ids = qwen_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens); + out->ids = embed_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens); if (!out->ids || out->n_tokens == 0) { fprintf(stderr, "tokenization failed: %s\n", text); free(out->ids); @@ -628,7 +628,6 @@ int main(int argc, char *argv[]) { } embed_verbose = verbose; - qwen_verbose = verbose; #ifdef USE_CUDA if (cuda_fast_gemm) { @@ -683,8 +682,8 @@ int main(int argc, char *argv[]) { #endif { if (n_threads <= 0) - n_threads = qwen_get_num_cpus(); - qwen_set_threads(n_threads); + n_threads = embed_get_num_cpus(); + embed_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } @@ -711,7 +710,7 @@ int main(int argc, char *argv[]) { } else { char vocab_path[1024]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); - e.tok = qwen_tokenizer_load(vocab_path); + e.tok = embed_tokenizer_load(vocab_path); if (!e.tok) { fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path); return 1; @@ -774,7 +773,7 @@ int main(int argc, char *argv[]) { * like the contextual separator: a vocab that defines it (test models) * wins; otherwise the released-model family constant applies. */ if (e.append_terminal_token) { - int eot = qwen_tokenizer_token_id(e.tok, "<|endoftext|>"); + int eot = embed_tokenizer_token_id(e.tok, "<|endoftext|>"); e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; } if (verbose >= 1) @@ -791,7 +790,7 @@ int main(int argc, char *argv[]) { if (e.wp_tok) e.wp_tok_ws = wordpiece_workspace_new(); else - e.tok_ws = qwen_tokenizer_workspace_new(); + e.tok_ws = embed_tokenizer_workspace_new(); if (!e.tok_ws && !e.wp_tok_ws) { fprintf(stderr, "failed to allocate tokenizer workspace\n"); if (e.workspace) diff --git a/embed_config.c b/src/config.c index 8ab81a5..f693843 100644 --- a/embed_config.c +++ b/src/config.c @@ -1,4 +1,4 @@ -#include "embed_config.h" +#include "config.h" #include <ctype.h> #include <errno.h> diff --git a/embed_config.h b/src/config.h index e8353d3..e8353d3 100644 --- a/embed_config.h +++ b/src/config.h diff --git a/embed_cuda.cu b/src/cuda.cu index 9464caa..6ce7dfb 100644 --- a/embed_cuda.cu +++ b/src/cuda.cu @@ -1,6 +1,6 @@ extern "C" { -#include "embed_cuda.h" -#include "embed_internal.h" +#include "cuda.h" +#include "internal.h" } #include <cuda_runtime.h> @@ -669,7 +669,7 @@ __global__ static void span_pool_kernel( // BERT LayerNorm: mean-subtract, biased variance (/dim), scale by gamma, shift // by beta. One block per row, two block reductions before any write, so it is -// safe in place (out == x). Matches the CPU qwen_layer_norm. +// safe in place (out == x). Matches the CPU embed_layer_norm. __global__ static void layer_norm_kernel(float *out, const float *x, const float *gamma, diff --git a/embed_cuda.h b/src/cuda.h index aad20ab..aad20ab 100644 --- a/embed_cuda.h +++ b/src/cuda.h @@ -1,8 +1,8 @@ #include "embed.h" -#include "embed_config.h" -#include "embed_internal.h" -#include "qwen_kernels.h" -#include "qwen_safetensors.h" +#include "config.h" +#include "internal.h" +#include "kernels.h" +#include "safetensors.h" #include <stdio.h> #include <stdlib.h> @@ -30,7 +30,6 @@ struct embed_late_workspace { * ======================================================================== */ int embed_verbose = 0; -int qwen_verbose = 0; #define EMBED_BF16_QKV_FUSE_MAX_SEQ 16 #define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4 @@ -306,7 +305,7 @@ static void linear_nobias_weight(embed_workspace_t *ws, int in_dim, int out_dim) { if (w->dtype == DTYPE_F32) { - qwen_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); + embed_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim); return; } if (seq_len > 16) { @@ -314,12 +313,12 @@ static void linear_nobias_weight(embed_workspace_t *ws, * BLAS-backed F32 path; below that the fused BF16 row kernels win. */ size_t count = (size_t)out_dim * (size_t)in_dim; if (ensure_bf16_widen(ws, count) == 0) { - qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count); - qwen_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); + embed_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count); + embed_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim); return; } } - qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim); + embed_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim); } static void linear_qkv_weight(embed_workspace_t *ws, @@ -336,7 +335,7 @@ static void linear_qkv_weight(embed_workspace_t *ws, int kv_dim) { if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 && seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) { - qwen_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, + embed_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, (const uint16_t *)wk->data, (const uint16_t *)wv->data, seq_len, in_dim, q_dim, kv_dim); return; @@ -369,7 +368,7 @@ static void linear_pair_weight(embed_workspace_t *ws, int b_dim) { if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) { - qwen_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, + embed_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim); return; } @@ -413,7 +412,7 @@ static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int se } static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) { - size_t bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + size_t bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); if (bytes <= ws->attn_scores_bytes) return; @@ -614,8 +613,8 @@ model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int a model->layer_start = layer_start; model->layer_end = layer_end; model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL - ? qwen_causal_gqa_attention_packed_with_scratch - : qwen_bidirectional_gqa_attention_packed_with_scratch; + ? embed_causal_gqa_attention_packed_with_scratch + : embed_bidirectional_gqa_attention_packed_with_scratch; /* Open safetensors (handles single file or multi-shard) */ multi_safetensors_t *ms = multi_safetensors_open(model_dir); @@ -955,7 +954,7 @@ float embed_late_maxsim(const float *query_vectors, float best = -FLT_MAX; for (int di = 0; di < doc_tokens; di++) { const float *d = doc_vectors + (size_t)di * dim; - float dot = qwen_dot_f32(q, d, dim); + float dot = embed_dot_f32(q, d, dim); if (dot > best) best = dot; } @@ -1009,7 +1008,7 @@ int embed_late_maxsim_batch(const float *query_vectors, doc++; int group_tokens = doc_offsets[doc] - start; - qwen_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, + embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, group_tokens); for (int i = first; i < doc; i++) { @@ -1215,7 +1214,7 @@ static int forward_packed_slice_inplace(const embed_model_t *model, for (int layer = layer_start; layer < layer_end; layer++) { const embed_layer_t *l = &w->layers[layer]; - qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); + embed_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps); linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq, hidden, q_dim, kv_dim); @@ -1224,16 +1223,16 @@ static int forward_packed_slice_inplace(const embed_model_t *model, add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim); add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim); if (cfg->qk_norm) { - qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); - qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); + embed_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps); + embed_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps); } for (int b = 0; b < batch; b++) { int start = offsets[b]; int len = offsets[b + 1] - start; - qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, + embed_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, head_dim); - qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, + embed_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, n_kv_heads, head_dim); } @@ -1241,19 +1240,19 @@ static int forward_packed_slice_inplace(const embed_model_t *model, head_dim, scale, ws->attn_scores, ws->attn_scores_bytes); linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden); - qwen_add_inplace(x, proj_out, total_seq * hidden); + embed_add_inplace(x, proj_out, total_seq * hidden); - qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); + embed_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps); linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq, hidden, inter, inter); - qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); + embed_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter); linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden); - qwen_add_inplace(x, proj_out, total_seq * hidden); + embed_add_inplace(x, proj_out, total_seq * hidden); } if (apply_final_norm) - qwen_rms_norm(x, x, w->norm, total_seq, hidden, eps); + embed_rms_norm(x, x, w->norm, total_seq, hidden, eps); return 0; } @@ -1322,11 +1321,11 @@ static int forward_packed_bert(const embed_model_t *model, } float *row = x + (size_t)(start + i) * hidden; copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden); - qwen_add_inplace(row, w->position_embeddings + (size_t)i * hidden, hidden); - qwen_add_inplace(row, w->token_type_embeddings, hidden); + embed_add_inplace(row, w->position_embeddings + (size_t)i * hidden, hidden); + embed_add_inplace(row, w->token_type_embeddings, hidden); } } - qwen_layer_norm(x, x, w->embed_ln_w, w->embed_ln_b, total_seq, hidden, eps); + embed_layer_norm(x, x, w->embed_ln_w, w->embed_ln_b, total_seq, hidden, eps); float scale = 1.0f / sqrtf((float)head_dim); @@ -1345,20 +1344,20 @@ static int forward_packed_bert(const embed_model_t *model, linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, hidden, hidden); add_bias_rows(proj_out, l->o_bias, total_seq, hidden); - qwen_add_inplace(x, proj_out, total_seq * hidden); - qwen_layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps); + embed_add_inplace(x, proj_out, total_seq * hidden); + embed_layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps); /* Feed-forward (post-norm): dense -> GeLU -> dense. */ linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter); add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter); if (cfg->ffn_act == EMBED_ACT_GELU_TANH) - qwen_gelu_tanh_inplace(ffn_up, total_seq * inter); + embed_gelu_tanh_inplace(ffn_up, total_seq * inter); else - qwen_gelu_inplace(ffn_up, total_seq * inter); + embed_gelu_inplace(ffn_up, total_seq * inter); linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden); add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden); - qwen_add_inplace(x, proj_out, total_seq * hidden); - qwen_layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps); + embed_add_inplace(x, proj_out, total_seq * hidden); + embed_layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps); } return 0; } @@ -351,6 +351,5 @@ EMBED_API int embed_late_maxsim_batch(const float *query_vectors, /* Verbose level: 0=quiet, 1=info, 2=debug */ EMBED_API extern int embed_verbose; -EMBED_API extern int qwen_verbose; #endif /* EMBED_H */ diff --git a/embed_internal.h b/src/internal.h index 39efbc4..cdc02a1 100644 --- a/embed_internal.h +++ b/src/internal.h @@ -2,7 +2,7 @@ #define EMBED_INTERNAL_H #include "embed.h" -#include "qwen_safetensors.h" +#include "safetensors.h" typedef struct { safetensor_dtype_t dtype; diff --git a/qwen_kernels.c b/src/kernels.c index a5d9778..226da45 100644 --- a/qwen_kernels.c +++ b/src/kernels.c @@ -1,5 +1,5 @@ -#include "qwen_kernels.h" -#include "qwen_kernels_impl.h" +#include "kernels.h" +#include "kernels_impl.h" #include <math.h> #include <string.h> #include <stdlib.h> @@ -38,13 +38,13 @@ void openblas_set_num_threads(int num_threads); * Thread Pool * ======================================================================== */ -#define QWEN_MAX_THREADS 16 +#define EMBED_MAX_THREADS 16 typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg); static struct { - pthread_t threads[QWEN_MAX_THREADS - 1]; - int tids[QWEN_MAX_THREADS - 1]; + pthread_t threads[EMBED_MAX_THREADS - 1]; + int tids[EMBED_MAX_THREADS - 1]; int n_threads; int shutdown; @@ -92,11 +92,11 @@ static void *worker_loop(void *arg) { } } -void qwen_set_threads(int n) { +void embed_set_threads(int n) { if (n < 1) n = 1; - if (n > QWEN_MAX_THREADS) - n = QWEN_MAX_THREADS; + if (n > EMBED_MAX_THREADS) + n = EMBED_MAX_THREADS; #if defined(USE_OPENBLAS) /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the @@ -120,7 +120,7 @@ void qwen_set_threads(int n) { tp.n_threads = n; if (n <= 1) { - if (qwen_verbose >= 2) { + if (embed_verbose >= 2) { #if defined(USE_OPENBLAS) fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads); #else @@ -135,7 +135,7 @@ void qwen_set_threads(int n) { pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]); } - if (qwen_verbose >= 2) { + if (embed_verbose >= 2) { #if defined(USE_OPENBLAS) fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads); #else @@ -189,7 +189,7 @@ static int cgroup_cpu_quota(void) { } #endif -int qwen_get_num_cpus(void) { +int embed_get_num_cpus(void) { #ifdef __APPLE__ int n = 0; size_t len = sizeof(n); @@ -236,7 +236,7 @@ static void parallel_for(parallel_fn_t fn, void *arg) { * Basic Element-wise Operations * ======================================================================== */ -void qwen_add_inplace(float *a, const float *b, int n) { +void embed_add_inplace(float *a, const float *b, int n) { for (int i = 0; i < n; i++) a[i] += b[i]; } @@ -245,7 +245,7 @@ void qwen_add_inplace(float *a, const float *b, int n) { * Matrix Operations * ======================================================================== */ -void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { +void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) { #ifdef USE_BLAS cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N); #else @@ -261,7 +261,7 @@ void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N #endif } -void qwen_linear(float *y, +void embed_linear(float *y, const float *x, const float *W, const float *b, @@ -299,13 +299,13 @@ void qwen_linear(float *y, #endif } -void qwen_linear_nobias( +void embed_linear_nobias( float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) { - qwen_linear(y, x, W, NULL, seq_len, in_dim, out_dim); + embed_linear(y, x, W, NULL, seq_len, in_dim, out_dim); } /* Convert bf16 buffer to f32 buffer */ -void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { +void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { #if defined(__ARM_NEON) || defined(__ARM_NEON__) size_t i = 0; uint32_t *d = (uint32_t *)(void *)dst; @@ -331,7 +331,7 @@ void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) { */ static void bf16_matvec_fused( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { - qwen_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); + embed_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim); } /* Threaded matvec: split output rows across threads */ @@ -476,7 +476,7 @@ static void pair_matvec_worker(int tid, int n_threads, void *arg) { } } -void qwen_linear_nobias_bf16_pair(float *a, +void embed_linear_nobias_bf16_pair(float *a, float *b, const float *x, const uint16_t *Wa_bf16, @@ -584,7 +584,7 @@ static void qkv_matvec_worker(int tid, int n_threads, void *arg) { } } -void qwen_linear_nobias_bf16_qkv(float *q, +void embed_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, @@ -625,7 +625,7 @@ void qwen_linear_nobias_bf16_qkv(float *q, parallel_for(qkv_matvec_worker, &task); } -void qwen_linear_nobias_bf16( +void embed_linear_nobias_bf16( float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) { if (seq_len == 1) { bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim); @@ -643,9 +643,9 @@ void qwen_linear_nobias_bf16( /* Rows split across the pool only when the tensor is big enough to pay * for the dispatch. */ -#define QWEN_RMS_NORM_PARALLEL_ELEMS 262144 +#define EMBED_RMS_NORM_PARALLEL_ELEMS 262144 -static void qwen_rms_norm_range( +static void embed_rms_norm_range( float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) { for (int s = start; s < end; s++) { const float *x_row = x + (size_t)s * hidden; @@ -728,16 +728,16 @@ static void rms_norm_worker(int tid, int n_threads, void *arg) { end = t->seq_len; if (start >= end) return; - qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps); + embed_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps); } -void qwen_rms_norm( +void embed_rms_norm( float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) { if (seq_len <= 0 || hidden <= 0) return; long long elems = (long long)seq_len * hidden; - if (tp.n_threads > 1 && elems >= QWEN_RMS_NORM_PARALLEL_ELEMS) { + if (tp.n_threads > 1 && elems >= EMBED_RMS_NORM_PARALLEL_ELEMS) { rms_norm_task_t task = { .out = out, .x = x, @@ -748,11 +748,11 @@ void qwen_rms_norm( }; parallel_for(rms_norm_worker, &task); } else { - qwen_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); + embed_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps); } } -void qwen_rms_norm_per_head( +void embed_rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) { /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */ int hidden = n_heads * head_dim; @@ -824,7 +824,7 @@ void qwen_rms_norm_per_head( /* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each * row's mean and variance are reduced before the row is rewritten. Scalar for * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */ -void qwen_layer_norm(float *out, +void embed_layer_norm(float *out, const float *x, const float *gamma, const float *beta, @@ -857,7 +857,7 @@ void qwen_layer_norm(float *out, * Activation Functions * ======================================================================== */ -void qwen_silu_mul_inplace(float *gate, const float *up, int n) { +void embed_silu_mul_inplace(float *gate, const float *up, int n) { #if defined(__APPLE__) && defined(USE_BLAS) enum { CHUNK = 4096 }; float exp_neg[CHUNK]; @@ -884,13 +884,13 @@ void qwen_silu_mul_inplace(float *gate, const float *up, int n) { #endif } -void qwen_gelu_inplace(float *x, int n) { +void embed_gelu_inplace(float *x, int n) { /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */ for (int i = 0; i < n; i++) x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); } -void qwen_gelu_tanh_inplace(float *x, int n) { +void embed_gelu_tanh_inplace(float *x, int n) { /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */ for (int i = 0; i < n; i++) { float v = x[i]; @@ -899,7 +899,7 @@ void qwen_gelu_tanh_inplace(float *x, int n) { } } -void qwen_softmax(float *x, int rows, int cols) { +void embed_softmax(float *x, int rows, int cols) { for (int r = 0; r < rows; r++) { float *row = x + r * cols; float max_val = row[0]; @@ -923,31 +923,31 @@ void qwen_softmax(float *x, int rows, int cols) { * Attention Operations * ======================================================================== */ -float qwen_dot_f32(const float *a, const float *b, int n) { return qwen_dot_f32_impl(a, b, n); } +float embed_dot_f32(const float *a, const float *b, int n) { return embed_dot_f32_impl(a, b, n); } -static inline float qwen_dot_f32_fast(const float *a, const float *b, int n) { - return qwen_dot_f32_impl(a, b, n); +static inline float embed_dot_f32_fast(const float *a, const float *b, int n) { + return embed_dot_f32_impl(a, b, n); } /* dst = dst * scale */ -static inline void qwen_vec_scale_inplace(float *dst, float scale, int n) { - qwen_vec_scale_inplace_impl(dst, scale, n); +static inline void embed_vec_scale_inplace(float *dst, float scale, int n) { + embed_vec_scale_inplace_impl(dst, scale, n); } /* dst += alpha * src */ -static inline void qwen_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - qwen_vec_axpy_inplace_impl(dst, src, alpha, n); +static inline void embed_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { + embed_vec_axpy_inplace_impl(dst, src, alpha, n); } /* dst = dst * correction + src */ -static inline void qwen_vec_scale_add(float *dst, const float *src, float correction, int n) { - qwen_vec_scale_add_impl(dst, src, correction, n); +static inline void embed_vec_scale_add(float *dst, const float *src, float correction, int n) { + embed_vec_scale_add_impl(dst, src, correction, n); } -#define QWEN_PACKED_ATTN_QUERY_TILE 32 -#define QWEN_PACKED_ATTN_BLAS_MIN_SEQ 128 +#define EMBED_PACKED_ATTN_QUERY_TILE 32 +#define EMBED_PACKED_ATTN_BLAS_MIN_SEQ 128 -static void qwen_bidirectional_gqa_attention_packed_online_rows(float *out, +static void embed_bidirectional_gqa_attention_packed_online_rows(float *out, const float *Q, const float *K, const float *V, @@ -978,29 +978,29 @@ static void qwen_bidirectional_gqa_attention_packed_online_rows(float *out, const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; + float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); sum_exp = sum_exp * correction + 1.0f; - qwen_vec_scale_add(o_row, v_row, correction, head_dim); + embed_vec_scale_add(o_row, v_row, correction, head_dim); max_score = score; } else { float wt = expf(score - max_score); sum_exp += wt; - qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); } } if (sum_exp > 0.0f) { float inv_sum = 1.0f / sum_exp; - qwen_vec_scale_inplace(o_row, inv_sum, head_dim); + embed_vec_scale_inplace(o_row, inv_sum, head_dim); } } } #ifdef USE_BLAS -static void qwen_bidirectional_gqa_attention_packed_blas_rows(float *out, +static void embed_bidirectional_gqa_attention_packed_blas_rows(float *out, const float *Q, const float *K, const float *V, @@ -1028,7 +1028,7 @@ static void qwen_bidirectional_gqa_attention_packed_blas_rows(float *out, cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, kv_hidden, 0.0f, scores, keys); - qwen_softmax(scores, rows, keys); + embed_softmax(scores, rows, keys); cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, 0.0f, o, q_hidden); } @@ -1053,28 +1053,28 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { int task_id = 0; #ifdef USE_BLAS float *scores = - t->scores ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; + t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; #endif for (int h = 0; h < t->n_heads; h++) { for (int b = 0; b < t->batch; b++) { int start = t->offsets[b]; int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; + for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; if (q1 > end) q1 = end; if (task_id++ % n_threads != tid) continue; #ifdef USE_BLAS - if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { - qwen_bidirectional_gqa_attention_packed_blas_rows( + if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { + embed_bidirectional_gqa_attention_packed_blas_rows( t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, scores); continue; } #endif - qwen_bidirectional_gqa_attention_packed_online_rows( + embed_bidirectional_gqa_attention_packed_online_rows( t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); } @@ -1082,7 +1082,7 @@ static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { } } -size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { +size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { #ifdef USE_BLAS if (!offsets || batch <= 0) return 0; @@ -1093,12 +1093,12 @@ size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, if (len > max_seq) max_seq = len; } - if (max_seq < QWEN_PACKED_ATTN_BLAS_MIN_SEQ) + if (max_seq < EMBED_PACKED_ATTN_BLAS_MIN_SEQ) return 0; if ((size_t)max_seq > - SIZE_MAX / (sizeof(float) * QWEN_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) + SIZE_MAX / (sizeof(float) * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)tp.n_threads)) return 0; - return (size_t)tp.n_threads * QWEN_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); + return (size_t)tp.n_threads * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); #else (void)offsets; (void)batch; @@ -1106,7 +1106,7 @@ size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, #endif } -void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out, +void embed_bidirectional_gqa_attention_packed_with_scratch(float *out, const float *Q, const float *K, const float *V, @@ -1129,7 +1129,7 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out, float *scores = NULL; #ifdef USE_BLAS - size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; #else @@ -1155,7 +1155,7 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out, } } -void qwen_bidirectional_gqa_attention_packed(float *out, +void embed_bidirectional_gqa_attention_packed(float *out, const float *Q, const float *K, const float *V, @@ -1165,14 +1165,14 @@ void qwen_bidirectional_gqa_attention_packed(float *out, int n_kv_heads, int head_dim, float scale) { - size_t scratch_bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - qwen_bidirectional_gqa_attention_packed_with_scratch( + embed_bidirectional_gqa_attention_packed_with_scratch( out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes); free(scores); } -static void qwen_causal_gqa_attention_packed_online_rows(float *out, +static void embed_causal_gqa_attention_packed_online_rows(float *out, const float *Q, const float *K, const float *V, @@ -1201,27 +1201,27 @@ static void qwen_causal_gqa_attention_packed_online_rows(float *out, for (int j = start; j <= i; j++) { const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; + float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); sum_exp = sum_exp * correction + 1.0f; - qwen_vec_scale_add(o_row, v_row, correction, head_dim); + embed_vec_scale_add(o_row, v_row, correction, head_dim); max_score = score; } else { float wt = expf(score - max_score); sum_exp += wt; - qwen_vec_axpy_inplace(o_row, v_row, wt, head_dim); + embed_vec_axpy_inplace(o_row, v_row, wt, head_dim); } } if (sum_exp > 0.0f) - qwen_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); + embed_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim); } } #ifdef USE_BLAS -static void qwen_causal_gqa_attention_packed_blas_rows(float *out, +static void embed_causal_gqa_attention_packed_blas_rows(float *out, const float *Q, const float *K, const float *V, @@ -1254,7 +1254,7 @@ static void qwen_causal_gqa_attention_packed_blas_rows(float *out, for (int j = allowed; j < keys; j++) row[j] = -INFINITY; } - qwen_softmax(scores, rows, keys); + embed_softmax(scores, rows, keys); cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, v, kv_hidden, 0.0f, o, q_hidden); } @@ -1265,28 +1265,28 @@ static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { int task_id = 0; #ifdef USE_BLAS float *scores = - t->scores ? t->scores + (size_t)tid * QWEN_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; + t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; #endif for (int h = 0; h < t->n_heads; h++) { for (int b = 0; b < t->batch; b++) { int start = t->offsets[b]; int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += QWEN_PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + QWEN_PACKED_ATTN_QUERY_TILE; + for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) { + int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE; if (q1 > end) q1 = end; if (task_id++ % n_threads != tid) continue; #ifdef USE_BLAS - if (scores && end - start >= QWEN_PACKED_ATTN_BLAS_MIN_SEQ) { - qwen_causal_gqa_attention_packed_blas_rows( + if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) { + embed_causal_gqa_attention_packed_blas_rows( t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim, t->scale, h, q0, q1, scores); continue; } #endif - qwen_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, + embed_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim, t->scale, h, q0, q1); } @@ -1294,7 +1294,7 @@ static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) { } } -void qwen_causal_gqa_attention_packed_with_scratch(float *out, +void embed_causal_gqa_attention_packed_with_scratch(float *out, const float *Q, const float *K, const float *V, @@ -1317,7 +1317,7 @@ void qwen_causal_gqa_attention_packed_with_scratch(float *out, float *scores = NULL; #ifdef USE_BLAS - size_t required = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; #else @@ -1342,7 +1342,7 @@ void qwen_causal_gqa_attention_packed_with_scratch(float *out, packed_causal_gqa_attn_worker(0, 1, &task); } -void qwen_causal_gqa_attention_packed(float *out, +void embed_causal_gqa_attention_packed(float *out, const float *Q, const float *K, const float *V, @@ -1352,9 +1352,9 @@ void qwen_causal_gqa_attention_packed(float *out, int n_kv_heads, int head_dim, float scale) { - size_t scratch_bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL; - qwen_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, + embed_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes); free(scores); } @@ -1363,7 +1363,7 @@ void qwen_causal_gqa_attention_packed(float *out, * Position Embeddings * ======================================================================== */ -void qwen_compute_rope_neox( +void embed_compute_rope_neox( float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { int half = head_dim / 2; @@ -1383,7 +1383,7 @@ void qwen_compute_rope_neox( } } -void qwen_apply_rope_neox( +void embed_apply_rope_neox( float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) { /* * NeoX split-half style: diff --git a/qwen_kernels.h b/src/kernels.h index 9abb489..6f180dd 100644 --- a/qwen_kernels.h +++ b/src/kernels.h @@ -1,11 +1,11 @@ /* - * qwen_kernels.h - Math kernels for Qwen3-architecture inference + * embed_kernels.h - Math kernels for Qwen3-architecture inference * * Low-level math operations. All operate on float32 tensors in row-major order. */ -#ifndef QWEN_KERNELS_H -#define QWEN_KERNELS_H +#ifndef EMBED_KERNELS_H +#define EMBED_KERNELS_H /* Public-API export annotation. The libraries are built with * -fvisibility=hidden, so only declarations carrying EMBED_API are exported @@ -25,30 +25,30 @@ * Basic Operations * ======================================================================== */ -void qwen_add_inplace(float *a, const float *b, int n); +void embed_add_inplace(float *a, const float *b, int n); /* ======================================================================== * Matrix Operations * ======================================================================== */ /* C = A @ B^T: A[M,K], B[N,K], C[M,N] */ -void qwen_matmul_t(float *C, const float *A, const float *B, int M, int K, int N); +void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N); /* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */ -void qwen_linear( +void embed_linear( float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim); -void qwen_linear_nobias( +void embed_linear_nobias( float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim); /* bf16 weight variants */ -void qwen_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n); -void qwen_linear_nobias_bf16( +void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n); +void embed_linear_nobias_bf16( float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim); /* Small-sequence BF16 path: compute two projections with one threaded dispatch. */ -void qwen_linear_nobias_bf16_pair(float *a, +void embed_linear_nobias_bf16_pair(float *a, float *b, const float *x, const uint16_t *Wa_bf16, @@ -59,7 +59,7 @@ void qwen_linear_nobias_bf16_pair(float *a, int b_dim); /* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */ -void qwen_linear_nobias_bf16_qkv(float *q, +void embed_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, @@ -72,26 +72,26 @@ void qwen_linear_nobias_bf16_qkv(float *q, int kv_dim); /* Dot product using the best available local SIMD implementation. */ -float qwen_dot_f32(const float *a, const float *b, int n); +float embed_dot_f32(const float *a, const float *b, int n); /* ======================================================================== * Normalization * ======================================================================== */ /* RMS Normalization: out = x / rms(x) * weight */ -void qwen_rms_norm( +void embed_rms_norm( float *out, const float *x, const float *weight, int seq_len, int hidden, float eps); /* Per-head RMS Normalization for Q/K norms in decoder * x: [seq, n_heads, head_dim], weight: [head_dim] * Normalizes each head independently */ -void qwen_rms_norm_per_head( +void embed_rms_norm_per_head( float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps); /* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps) * + beta, with mean and biased (population) variance over the hidden axis per * row. Unlike RMSNorm this subtracts the mean and adds a bias. */ -void qwen_layer_norm(float *out, +void embed_layer_norm(float *out, const float *x, const float *gamma, const float *beta, @@ -103,17 +103,17 @@ void qwen_layer_norm(float *out, * Activation Functions * ======================================================================== */ -void qwen_softmax(float *x, int rows, int cols); +void embed_softmax(float *x, int rows, int cols); /* gate = SiLU(gate) * up */ -void qwen_silu_mul_inplace(float *gate, const float *up, int n); +void embed_silu_mul_inplace(float *gate, const float *up, int n); /* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */ -void qwen_gelu_inplace(float *x, int n); +void embed_gelu_inplace(float *x, int n); /* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh): * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */ -void qwen_gelu_tanh_inplace(float *x, int n); +void embed_gelu_tanh_inplace(float *x, int n); /* ======================================================================== * Attention Operations @@ -129,7 +129,7 @@ void qwen_gelu_tanh_inplace(float *x, int n); * Attention is block-diagonal: tokens only attend within their own * offsets[b]..offsets[b+1] span. */ -void qwen_bidirectional_gqa_attention_packed(float *out, +void embed_bidirectional_gqa_attention_packed(float *out, const float *Q, const float *K, const float *V, @@ -141,8 +141,8 @@ void qwen_bidirectional_gqa_attention_packed(float *out, float scale); /* Optional reusable scratch for the BLAS-tiled packed attention path. */ -size_t qwen_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); -void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out, +size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); +void embed_bidirectional_gqa_attention_packed_with_scratch(float *out, const float *Q, const float *K, const float *V, @@ -159,7 +159,7 @@ void qwen_bidirectional_gqa_attention_packed_with_scratch(float *out, * Causal GQA attention over a packed/ragged batch. Query position i attends * only to positions at or before i within its sequence. */ -void qwen_causal_gqa_attention_packed(float *out, +void embed_causal_gqa_attention_packed(float *out, const float *Q, const float *K, const float *V, @@ -169,7 +169,7 @@ void qwen_causal_gqa_attention_packed(float *out, int n_kv_heads, int head_dim, float scale); -void qwen_causal_gqa_attention_packed_with_scratch(float *out, +void embed_causal_gqa_attention_packed_with_scratch(float *out, const float *Q, const float *K, const float *V, @@ -191,7 +191,7 @@ void qwen_causal_gqa_attention_packed_with_scratch(float *out, * cos_out, sin_out: [seq, head_dim] * cos[d] and cos[half+d] are the same (duplicated for full head_dim). */ -void qwen_compute_rope_neox( +void embed_compute_rope_neox( float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta); /* @@ -199,7 +199,7 @@ void qwen_compute_rope_neox( * x: [seq, n_heads * head_dim] (in-place) * cos_vals, sin_vals: [seq, head_dim] */ -void qwen_apply_rope_neox( +void embed_apply_rope_neox( float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim); /* ======================================================================== @@ -208,12 +208,12 @@ void qwen_apply_rope_neox( /* Set number of threads for parallel operations (default: 1). * Creates a persistent thread pool. Call before inference. */ -EMBED_API void qwen_set_threads(int n); +EMBED_API void embed_set_threads(int n); /* Get number of available CPU cores */ -EMBED_API int qwen_get_num_cpus(void); +EMBED_API int embed_get_num_cpus(void); /* Global verbose flag */ -EMBED_API extern int qwen_verbose; +EMBED_API extern int embed_verbose; -#endif /* QWEN_KERNELS_H */ +#endif /* EMBED_KERNELS_H */ diff --git a/qwen_kernels_avx.c b/src/kernels_avx.c index 42c358b..920fecf 100644 --- a/qwen_kernels_avx.c +++ b/src/kernels_avx.c @@ -1,5 +1,5 @@ /* - * qwen_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) + * embed_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available) * * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16. * Uses AVX-512F+BW for 16-wide bf16 matvec (dominant for seq_len==1), and @@ -9,7 +9,7 @@ * overhead and improve out-of-order execution on memory-bound workloads. */ -#include "qwen_kernels_impl.h" +#include "kernels_impl.h" #if defined(__AVX2__) && defined(__FMA__) @@ -22,7 +22,7 @@ #if defined(__AVX512F__) && defined(__AVX512BW__) -void qwen_bf16_matvec_fused_avx( +void embed_bf16_matvec_fused_avx( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; @@ -201,7 +201,7 @@ static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) { *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1)); } -void qwen_bf16_matvec_fused_avx( +void embed_bf16_matvec_fused_avx( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; @@ -273,7 +273,7 @@ void qwen_bf16_matvec_fused_avx( * (operates on L1-resident head vectors) * ===================================================================== */ -float qwen_dot_f32_avx(const float *a, const float *b, int n) { +float embed_dot_f32_avx(const float *a, const float *b, int n) { #if defined(__AVX512F__) int i = 0; __m512 acc0 = _mm512_setzero_ps(); @@ -320,7 +320,7 @@ float qwen_dot_f32_avx(const float *a, const float *b, int n) { #endif } -void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) { +void embed_vec_scale_inplace_avx(float *dst, float scale, int n) { #if defined(__AVX512F__) int i = 0; __m512 s = _mm512_set1_ps(scale); @@ -352,7 +352,7 @@ void qwen_vec_scale_inplace_avx(float *dst, float scale, int n) { #endif } -void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) { +void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) { #if defined(__AVX512F__) int i = 0; __m512 a = _mm512_set1_ps(alpha); @@ -394,7 +394,7 @@ void qwen_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) #endif } -void qwen_vec_scale_add_avx(float *dst, const float *src, float correction, int n) { +void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n) { #if defined(__AVX512F__) int i = 0; __m512 c = _mm512_set1_ps(correction); diff --git a/qwen_kernels_generic.c b/src/kernels_generic.c index 149dfe0..7e85298 100644 --- a/qwen_kernels_generic.c +++ b/src/kernels_generic.c @@ -1,12 +1,12 @@ /* - * qwen_kernels_generic.c - architecture-generic hot kernels + * embed_kernels_generic.c - architecture-generic hot kernels */ -#include "qwen_kernels_impl.h" +#include "kernels_impl.h" #include <string.h> -void qwen_bf16_matvec_fused_generic( +void embed_bf16_matvec_fused_generic( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { for (int o = 0; o < out_dim; o++) { const uint16_t *w_row = W_bf16 + (size_t)o * in_dim; @@ -21,24 +21,24 @@ void qwen_bf16_matvec_fused_generic( } } -float qwen_dot_f32_generic(const float *a, const float *b, int n) { +float embed_dot_f32_generic(const float *a, const float *b, int n) { float sum = 0.0f; for (int i = 0; i < n; i++) sum += a[i] * b[i]; return sum; } -void qwen_vec_scale_inplace_generic(float *dst, float scale, int n) { +void embed_vec_scale_inplace_generic(float *dst, float scale, int n) { for (int i = 0; i < n; i++) dst[i] *= scale; } -void qwen_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { +void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) { for (int i = 0; i < n; i++) dst[i] += alpha * src[i]; } -void qwen_vec_scale_add_generic(float *dst, const float *src, float correction, int n) { +void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n) { for (int i = 0; i < n; i++) dst[i] = dst[i] * correction + src[i]; } diff --git a/src/kernels_impl.h b/src/kernels_impl.h new file mode 100644 index 0000000..4ee7d74 --- /dev/null +++ b/src/kernels_impl.h @@ -0,0 +1,53 @@ +/* + * embed_kernels_impl.h - internal architecture dispatch for hot kernels + */ + +#ifndef EMBED_KERNELS_IMPL_H +#define EMBED_KERNELS_IMPL_H + +#include <stdint.h> + +void embed_bf16_matvec_fused_generic( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); +float embed_dot_f32_generic(const float *a, const float *b, int n); +void embed_vec_scale_inplace_generic(float *dst, float scale, int n); +void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n); +void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n); + +#ifdef __ARM_NEON +void embed_bf16_matvec_fused_neon( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); +float embed_dot_f32_neon(const float *a, const float *b, int n); +void embed_vec_scale_inplace_neon(float *dst, float scale, int n); +void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n); +void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n); + +#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_neon +#define embed_dot_f32_impl embed_dot_f32_neon +#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_neon +#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_neon +#define embed_vec_scale_add_impl embed_vec_scale_add_neon + +#elif defined(__AVX2__) && defined(__FMA__) +void embed_bf16_matvec_fused_avx( + float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim); +float embed_dot_f32_avx(const float *a, const float *b, int n); +void embed_vec_scale_inplace_avx(float *dst, float scale, int n); +void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n); +void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n); + +#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_avx +#define embed_dot_f32_impl embed_dot_f32_avx +#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_avx +#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_avx +#define embed_vec_scale_add_impl embed_vec_scale_add_avx + +#else +#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_generic +#define embed_dot_f32_impl embed_dot_f32_generic +#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_generic +#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_generic +#define embed_vec_scale_add_impl embed_vec_scale_add_generic +#endif + +#endif /* EMBED_KERNELS_IMPL_H */ diff --git a/qwen_kernels_neon.c b/src/kernels_neon.c index 5d51fc6..3e2a423 100644 --- a/qwen_kernels_neon.c +++ b/src/kernels_neon.c @@ -1,15 +1,15 @@ /* - * qwen_kernels_neon.c - ARM NEON hot kernels + * embed_kernels_neon.c - ARM NEON hot kernels */ -#include "qwen_kernels_impl.h" +#include "kernels_impl.h" #ifdef __ARM_NEON #include <arm_neon.h> #include <string.h> -void qwen_bf16_matvec_fused_neon( +void embed_bf16_matvec_fused_neon( float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) { int o = 0; @@ -115,7 +115,7 @@ void qwen_bf16_matvec_fused_neon( } } -float qwen_dot_f32_neon(const float *a, const float *b, int n) { +float embed_dot_f32_neon(const float *a, const float *b, int n) { int i = 0; float32x4_t acc0 = vdupq_n_f32(0.0f); float32x4_t acc1 = vdupq_n_f32(0.0f); @@ -133,7 +133,7 @@ float qwen_dot_f32_neon(const float *a, const float *b, int n) { return sum; } -void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) { +void embed_vec_scale_inplace_neon(float *dst, float scale, int n) { int i = 0; float32x4_t s = vdupq_n_f32(scale); for (; i + 8 <= n; i += 8) { @@ -146,7 +146,7 @@ void qwen_vec_scale_inplace_neon(float *dst, float scale, int n) { dst[i] *= scale; } -void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) { +void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) { int i = 0; float32x4_t a = vdupq_n_f32(alpha); for (; i + 8 <= n; i += 8) { @@ -161,7 +161,7 @@ void qwen_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n dst[i] += alpha * src[i]; } -void qwen_vec_scale_add_neon(float *dst, const float *src, float correction, int n) { +void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n) { int i = 0; float32x4_t c = vdupq_n_f32(correction); for (; i + 8 <= n; i += 8) { @@ -1,7 +1,7 @@ -#include "embed_mlx.h" -#include "embed_config.h" +#include "mlx.h" +#include "config.h" #include "embed.h" -#include "qwen_safetensors.h" +#include "safetensors.h" #include <mlx/c/mlx.h> diff --git a/qwen_safetensors.c b/src/safetensors.c index 2222a2a..f12549c 100644 --- a/qwen_safetensors.c +++ b/src/safetensors.c @@ -1,4 +1,4 @@ -#include "qwen_safetensors.h" +#include "safetensors.h" #include <stdio.h> #include <stdlib.h> #include <string.h> diff --git a/qwen_safetensors.h b/src/safetensors.h index c060847..8c9b33f 100644 --- a/qwen_safetensors.h +++ b/src/safetensors.h @@ -1,10 +1,10 @@ /* - * qwen_safetensors.h - Safetensors file format reader (multi-shard support) + * embed_safetensors.h - Safetensors file format reader (multi-shard support) * Adapted from voxtral-realtime project. */ -#ifndef QWEN_SAFETENSORS_H -#define QWEN_SAFETENSORS_H +#ifndef EMBED_SAFETENSORS_H +#define EMBED_SAFETENSORS_H #include <stddef.h> #include <stdint.h> @@ -73,4 +73,4 @@ float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t); int64_t safetensor_numel(const safetensor_t *t); -#endif /* QWEN_SAFETENSORS_H */ +#endif /* EMBED_SAFETENSORS_H */ diff --git a/embed_server.c b/src/server.c index e1f2858..c3abbac 100644 --- a/embed_server.c +++ b/src/server.c @@ -2,19 +2,19 @@ #define _GNU_SOURCE #endif -#include "embed_build.h" -#include "embed_server.h" +#include "build.h" +#include "server.h" #include "embed.h" -#include "qwen_safetensors.h" -#include "qwen_kernels.h" -#include "qwen_tokenizer.h" -#include "wordpiece_tokenizer.h" +#include "safetensors.h" +#include "kernels.h" +#include "tokenizer_bpe.h" +#include "tokenizer_wordpiece.h" #ifdef USE_MLX -#include "embed_mlx.h" +#include "mlx.h" #endif #ifdef USE_CUDA -#include "embed_cuda.h" +#include "cuda.h" #endif #include "deps/ae/ae.h" @@ -313,8 +313,8 @@ static model_slot model_slot_for_id(const char *id) { typedef struct { const model_info *info; char *path; - qwen_tokenizer_t *tok; - qwen_tokenizer_workspace_t *tok_ws; + embed_tokenizer_t *tok; + embed_tokenizer_workspace_t *tok_ws; /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model. * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the * Qwen byte-level BPE fields above stay NULL. */ @@ -1529,7 +1529,7 @@ static int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *ou out->n_tokens = n + 2; return 0; } - out->ids = qwen_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens); + out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens); if (j) j->tokenize_ns += nstime() - t0; if (!out->ids || out->n_tokens <= 0) { @@ -3041,12 +3041,12 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { } else { char vocab_path[1024]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); - m->tok = qwen_tokenizer_load(vocab_path); + m->tok = embed_tokenizer_load(vocab_path); if (!m->tok) { server_log("embed-server: failed to load tokenizer: %s", vocab_path); return -1; } - m->tok_ws = qwen_tokenizer_workspace_new(); + m->tok_ws = embed_tokenizer_workspace_new(); if (!m->tok_ws) { server_log("embed-server: failed to allocate tokenizer workspace: %s", path); return -1; @@ -3056,18 +3056,18 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { * family constant is the normal case; a vocab that does define the * token (e.g. small test models) takes precedence so the id stays * inside that model's embedding table. */ - int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>"); + int sep_id = embed_tokenizer_token_id(m->tok, "<|endoftext|>"); m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|> * suffix - the same token as the separator, not the model's chat * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */ m->terminal_token_id = m->context_separator_id; if (m->info->kind == MODEL_KIND_LATE) { - int id = qwen_tokenizer_token_id(m->tok, "[MASK]"); + int id = embed_tokenizer_token_id(m->tok, "[MASK]"); m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID; - id = qwen_tokenizer_token_id(m->tok, "[Q]"); + id = embed_tokenizer_token_id(m->tok, "[Q]"); m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID; - id = qwen_tokenizer_token_id(m->tok, "[D]"); + id = embed_tokenizer_token_id(m->tok, "[D]"); m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID; const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"; @@ -3077,7 +3077,7 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) { p++) { char text[2] = {*p, '\0'}; int n_ids = 0; - int *ids = qwen_tokenizer_encode(m->tok, text, &n_ids); + int *ids = embed_tokenizer_encode(m->tok, text, &n_ids); if (ids && n_ids == 1) m->late_skip_ids[m->n_late_skip_ids++] = ids[0]; free(ids); @@ -3142,9 +3142,9 @@ static void free_models(http_server *s) { loaded_model *m = &s->models[i]; free(m->path); if (m->tok_ws) - qwen_tokenizer_workspace_free(m->tok_ws); + embed_tokenizer_workspace_free(m->tok_ws); if (m->tok) - qwen_tokenizer_free(m->tok); + embed_tokenizer_free(m->tok); if (m->wp_tok_ws) wordpiece_workspace_free(m->wp_tok_ws); if (m->wp_tok) @@ -3626,7 +3626,6 @@ int main(int argc, char *argv[]) { } embed_verbose = verbose; - qwen_verbose = verbose; if (arg < argc) { fprintf(stderr, "unexpected argument: %s\n", argv[arg]); @@ -3692,8 +3691,8 @@ int main(int argc, char *argv[]) { #endif { if (n_threads <= 0) - n_threads = qwen_get_num_cpus(); - qwen_set_threads(n_threads); + n_threads = embed_get_num_cpus(); + embed_set_threads(n_threads); if (verbose >= 1) fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } diff --git a/embed_server.h b/src/server.h index 2a39b94..2a39b94 100644 --- a/embed_server.h +++ b/src/server.h diff --git a/qwen_tokenizer.c b/src/tokenizer_bpe.c index e813071..511b29e 100644 --- a/qwen_tokenizer.c +++ b/src/tokenizer_bpe.c @@ -1,5 +1,5 @@ -#include "qwen_tokenizer.h" -#include "qwen_kernels.h" +#include "tokenizer_bpe.h" +#include "kernels.h" #include <limits.h> #include <stdint.h> #include <stdio.h> @@ -437,7 +437,7 @@ pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int me /* Returns the rank and sets *merged_id, or INT_MAX when the pair never * merges. */ -static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, int *merged_id) { +static int pair_merge_lookup(const embed_tokenizer_t *tok, int left, int right, int *merged_id) { const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges; uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right; int mask = tok->int_merges_cap - 1; @@ -453,7 +453,7 @@ static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, i return INT_MAX; } -static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) { +static int merge_rank(const embed_tokenizer_t *tok, const char *a, const char *b) { if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX; @@ -496,7 +496,7 @@ static char *text_to_bpe_unicode(const char *text) { } static int -encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n); +encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n); static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) { @@ -828,7 +828,7 @@ static int match_contraction(const unsigned char *p, const unsigned char *end) { return 0; } -static int append_encoded_piece(const qwen_tokenizer_t *tok, +static int append_encoded_piece(const embed_tokenizer_t *tok, const char *piece, size_t piece_len, int **ids, @@ -868,7 +868,7 @@ static int append_encoded_piece(const qwen_tokenizer_t *tok, } static int -encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) { +encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) { const unsigned char *start = (const unsigned char *)text; const unsigned char *p = start; const unsigned char *end = start + strlen(text); @@ -1001,7 +1001,7 @@ encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, int **out_ids /* Encode one mapped BPE unicode string to token IDs. */ static int -encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { +encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { *out_ids = NULL; *out_n = 0; if (!mapped || !*mapped) @@ -1078,7 +1078,7 @@ typedef struct { size_t off; } bpe_sym_ref_t; -struct qwen_tokenizer_workspace { +struct embed_tokenizer_workspace { char *normalized; size_t normalized_cap; char *mapped; @@ -1101,7 +1101,7 @@ struct qwen_tokenizer_workspace { int bpe_cap; }; -static int reserve_bpe(qwen_tokenizer_workspace_t *ws, int need) { +static int reserve_bpe(embed_tokenizer_workspace_t *ws, int need) { if (ws->bpe_cap >= need) return 0; int new_cap = ws->bpe_cap ? ws->bpe_cap : 64; @@ -1160,7 +1160,7 @@ static int reserve_ints(int **buf, int *cap, int need) { return 0; } -static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) { +static int reserve_syms(embed_tokenizer_workspace_t *ws, int need) { if (ws->syms_cap >= need) return 0; int new_cap = ws->syms_cap ? ws->syms_cap : 64; @@ -1177,7 +1177,7 @@ static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) { return 0; } -static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) { +static int arena_append(embed_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) { if (ws->arena_len > SIZE_MAX - len - 1) return -1; size_t need = ws->arena_len + len + 1; @@ -1190,12 +1190,12 @@ static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, size_t le return 0; } -static const char *arena_str(const qwen_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) { +static const char *arena_str(const embed_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) { return ws->arena + sym.off; } static int -normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, const char *text, const char **out) { +normalize_nfc_latin_into(embed_tokenizer_workspace_t *ws, const char *text, const char **out) { const unsigned char *p = (const unsigned char *)text; const unsigned char *end = p + strlen(text); size_t in_len = (size_t)(end - p); @@ -1235,7 +1235,7 @@ normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, const char *text, const return 0; } -static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, +static int text_to_bpe_unicode_into(embed_tokenizer_workspace_t *ws, const char *text, size_t len, const char **out) { @@ -1267,8 +1267,8 @@ static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow return 0; } -static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static int encode_bpe_word_into_slow(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *mapped, int *out_ids, int out_cap, @@ -1349,8 +1349,8 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, * the string-based loop when the int tables are unavailable. Identical * output: initial ids are the byte-level single-char token ids and every * merge result is the vocab id of the concatenated pair. */ -static int encode_bpe_word_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static int encode_bpe_word_into(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *mapped, int *out_ids, int out_cap, @@ -1427,8 +1427,8 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok, return 0; } -static int append_encoded_piece_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static int append_encoded_piece_into(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *piece, size_t piece_len, int *out_ids, @@ -1444,8 +1444,8 @@ static int append_encoded_piece_into(const qwen_tokenizer_t *tok, return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); } -static int encode_pretokenized_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static int encode_pretokenized_into(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, int *out_ids, int out_cap, @@ -1622,7 +1622,7 @@ static int parse_merge_pair(char *line, char **a, char **b) { return 0; } -static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { +static int load_merges_map(embed_tokenizer_t *tok, const char *merges_path) { FILE *f = fopen(merges_path, "rb"); if (!f) return -1; @@ -1745,10 +1745,10 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { * Public API * ======================================================================== */ -qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { +embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { FILE *f = fopen(vocab_json_path, "rb"); if (!f) { - fprintf(stderr, "qwen_tokenizer_load: cannot open %s\n", vocab_json_path); + fprintf(stderr, "embed_tokenizer_load: cannot open %s\n", vocab_json_path); return NULL; } @@ -1803,7 +1803,7 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { } int vocab_size = max_id + 1; - qwen_tokenizer_t *tok = (qwen_tokenizer_t *)calloc(1, sizeof(qwen_tokenizer_t)); + embed_tokenizer_t *tok = (embed_tokenizer_t *)calloc(1, sizeof(embed_tokenizer_t)); if (!tok) { free(json); return NULL; @@ -1812,7 +1812,7 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *)); tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *)); if (!tok->id_to_text || !tok->id_to_bpe) { - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); free(json); return NULL; } @@ -1852,7 +1852,7 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); if (!tok->vocab_map) { - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); return NULL; } for (int i = 0; i < vocab_size; i++) { @@ -1863,7 +1863,7 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { char merges_path[1024]; if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { - if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) { + if (load_merges_map(tok, merges_path) != 0 && embed_verbose >= 2) { fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", merges_path); @@ -1873,19 +1873,19 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { return tok; } -const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) { +const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id) { if (!tok || token_id < 0 || token_id >= tok->vocab_size) return ""; return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; } -int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token) { +int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token) { if (!tok || !token || !tok->vocab_map) return -1; return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token); } -int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) { +int *embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens) { if (out_n_tokens) *out_n_tokens = 0; if (!tok || !text || text[0] == '\0') @@ -1909,11 +1909,11 @@ int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *o return ids; } -qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) { - return (qwen_tokenizer_workspace_t *)calloc(1, sizeof(qwen_tokenizer_workspace_t)); +embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void) { + return (embed_tokenizer_workspace_t *)calloc(1, sizeof(embed_tokenizer_workspace_t)); } -void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { +void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws) { if (!ws) return; free(ws->normalized); @@ -1929,8 +1929,8 @@ void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { free(ws); } -int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +int embed_tokenizer_encode_into(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, int *out_ids, int out_cap, @@ -1953,8 +1953,8 @@ int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, return rc; } -int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, int *out_n_tokens) { if (out_n_tokens) @@ -1972,11 +1972,11 @@ int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, return NULL; int n_ids = 0; - int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); + int rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); if (rc == -2) { if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) return NULL; - rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); + rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); } if (rc != 0 || n_ids <= 0) return NULL; @@ -1990,7 +1990,7 @@ int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, return ids; } -void qwen_tokenizer_free(qwen_tokenizer_t *tok) { +void embed_tokenizer_free(embed_tokenizer_t *tok) { if (!tok) return; diff --git a/qwen_tokenizer.h b/src/tokenizer_bpe.h index f7a3941..8d13d47 100644 --- a/qwen_tokenizer.h +++ b/src/tokenizer_bpe.h @@ -1,13 +1,13 @@ /* - * qwen_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) + * embed_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level) * * Supports: * - decode token ID -> UTF-8 text * - encode UTF-8 text -> token IDs (vocab.json + merges.txt) */ -#ifndef QWEN_TOKENIZER_H -#define QWEN_TOKENIZER_H +#ifndef EMBED_TOKENIZER_H +#define EMBED_TOKENIZER_H /* Public-API export annotation. The libraries are built with * -fvisibility=hidden, so only declarations carrying EMBED_API are exported @@ -38,49 +38,49 @@ typedef struct { /* Byte-level unicode codepoint -> single-character token id (512 slots), * or NULL when any byte token is missing from the vocab. */ int *cp_to_id; -} qwen_tokenizer_t; +} embed_tokenizer_t; -typedef struct qwen_tokenizer_workspace qwen_tokenizer_workspace_t; +typedef struct embed_tokenizer_workspace embed_tokenizer_workspace_t; /* Load tokenizer from vocab.json in model directory */ -EMBED_API qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path); +EMBED_API embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path); /* Decode a single token ID to text. Returns pointer to internal string. */ -EMBED_API const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id); +EMBED_API const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id); /* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>"). * Returns -1 when the token is not in the vocab. */ -EMBED_API int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token); +EMBED_API int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token); /* Encode UTF-8 text into token IDs using BPE. * Returns malloc'd array of token IDs and sets *out_n_tokens. * Returns NULL on error (and sets *out_n_tokens to 0). */ EMBED_API int * -qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens); +embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens); /* Reusable scratch for allocation-light repeated encoding. */ -EMBED_API qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void); -EMBED_API void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws); +EMBED_API embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void); +EMBED_API void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws); /* Encode into caller-provided out_ids[0..out_cap-1]. * Returns 0 on success, -1 on error, -2 if out_cap is too small. * In all cases *out_n_tokens is set to the number of tokens needed/emitted * when out_n_tokens is non-NULL. */ -EMBED_API int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +EMBED_API int embed_tokenizer_encode_into(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, int *out_ids, int out_cap, int *out_n_tokens); -/* Same tokenization as qwen_tokenizer_encode(), but reuses ws internally and +/* Same tokenization as embed_tokenizer_encode(), but reuses ws internally and * performs only the final returned int[] allocation. */ -EMBED_API int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +EMBED_API int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, int *out_n_tokens); /* Free tokenizer */ -EMBED_API void qwen_tokenizer_free(qwen_tokenizer_t *tok); +EMBED_API void embed_tokenizer_free(embed_tokenizer_t *tok); -#endif /* QWEN_TOKENIZER_H */ +#endif /* EMBED_TOKENIZER_H */ diff --git a/wordpiece_unicode.h b/src/tokenizer_unicode.h index 8ca2458..8ca2458 100644 --- a/wordpiece_unicode.h +++ b/src/tokenizer_unicode.h diff --git a/wordpiece_tokenizer.c b/src/tokenizer_wordpiece.c index 5ad0f83..d8acbbe 100644 --- a/wordpiece_tokenizer.c +++ b/src/tokenizer_wordpiece.c @@ -18,9 +18,9 @@ * Greek final sigma). */ -#include "wordpiece_tokenizer.h" +#include "tokenizer_wordpiece.h" -#include "wordpiece_unicode.h" +#include "tokenizer_unicode.h" #include <stdint.h> #include <stdio.h> diff --git a/wordpiece_tokenizer.h b/src/tokenizer_wordpiece.h index 0e1ac20..59cb2bb 100644 --- a/wordpiece_tokenizer.h +++ b/src/tokenizer_wordpiece.h @@ -7,7 +7,7 @@ * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the * byte-level BPE backend leaves terminal/separator tokens to the frontend. * - * The public surface mirrors qwen_tokenizer.h (load / encode with a reusable + * The public surface mirrors embed_tokenizer.h (load / encode with a reusable * workspace / decode / resolve a named special token / free) so both can sit * behind one tokenizer interface later. */ diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 4a9e9db..4742b70 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -2,8 +2,8 @@ * Runs via `make test` (generic and BLAS variants) and * scripts/check_kernel_golden.py. No model files required. */ -#include "qwen_kernels.h" -#include "qwen_kernels_impl.h" +#include "kernels.h" +#include "kernels_impl.h" #include <math.h> #include <stdint.h> @@ -12,7 +12,7 @@ #include <string.h> static int failures = 0; -int qwen_verbose = 0; +int embed_verbose = 0; static void expect_close(const char *name, float got, float want, float tol) { float diff = fabsf(got - want); @@ -42,7 +42,7 @@ static void test_rms_norm(void) { const float w[4] = {1.0f, 0.5f, -1.0f, 2.0f}; float out[8]; - qwen_rms_norm(out, x, w, seq, hidden, 1e-6f); + embed_rms_norm(out, x, w, seq, hidden, 1e-6f); for (int s = 0; s < seq; s++) { float ss = 0.0f; @@ -65,7 +65,7 @@ static void test_layer_norm(void) { const float b[4] = {0.1f, -0.2f, 0.3f, -0.4f}; float out[8]; - qwen_layer_norm(out, x, g, b, seq, hidden, 1e-12f); + embed_layer_norm(out, x, g, b, seq, hidden, 1e-12f); for (int s = 0; s < seq; s++) { float mean = 0.0f; @@ -88,7 +88,7 @@ static void test_layer_norm(void) { /* In-place (out aliases x) must match the out-of-place result. */ float z[8]; memcpy(z, x, sizeof(z)); - qwen_layer_norm(z, z, g, b, seq, hidden, 1e-12f); + embed_layer_norm(z, z, g, b, seq, hidden, 1e-12f); for (int i = 0; i < 8; i++) expect_close("layer_norm_inplace", z[i], out[i], 1e-6f); } @@ -97,7 +97,7 @@ static void test_gelu(void) { float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f}; float out[5]; memcpy(out, x, sizeof(out)); - qwen_gelu_inplace(out, 5); + embed_gelu_inplace(out, 5); for (int i = 0; i < 5; i++) { float want = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); expect_close("gelu", out[i], want, 1e-6f); @@ -112,7 +112,7 @@ static void test_gelu_tanh(void) { float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f}; float out[5]; memcpy(out, x, sizeof(out)); - qwen_gelu_tanh_inplace(out, 5); + embed_gelu_tanh_inplace(out, 5); for (int i = 0; i < 5; i++) { float v = x[i]; float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); @@ -134,8 +134,8 @@ static void test_rope_neox(void) { float orig[8]; memcpy(orig, x, sizeof(x)); - qwen_compute_rope_neox(cosv, sinv, positions, seq, head_dim, 10000.0f); - qwen_apply_rope_neox(x, cosv, sinv, seq, heads, head_dim); + embed_compute_rope_neox(cosv, sinv, positions, seq, head_dim, 10000.0f); + embed_apply_rope_neox(x, cosv, sinv, seq, heads, head_dim); for (int s = 0; s < seq; s++) { for (int d = 0; d < head_dim / 2; d++) { @@ -225,7 +225,7 @@ static void test_packed_gqa_attention(void) { V[i] = sinf((float)i * 0.07f) - cosf((float)i * 0.13f); } - qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, + embed_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 0); @@ -254,12 +254,12 @@ static void test_packed_gqa_attention_long(void) { V[i] = sinf((float)i * 0.007f) - cosf((float)i * 0.013f); } - qwen_set_threads(4); - qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, + embed_set_threads(4); + embed_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 0); - qwen_set_threads(1); + embed_set_threads(1); for (int i = 0; i < total * q_hidden; i++) expect_close("packed_gqa_long", got[i], want[i], 2e-5f); @@ -290,12 +290,12 @@ static void test_packed_causal_gqa_attention(void) { V[i] = sinf((float)i * 0.009f) - cosf((float)i * 0.015f); } - qwen_set_threads(4); - qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, + embed_set_threads(4); + embed_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 1); - qwen_set_threads(1); + embed_set_threads(1); for (int i = 0; i < total * q_hidden; i++) expect_close("packed_causal_gqa", got[i], want[i], 2e-5f); @@ -315,7 +315,7 @@ static void test_bf16_linear(void) { for (int i = 0; i < 6; i++) wbf16[i] = f32_to_bf16(wf32[i]); - qwen_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim); + embed_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < out_dim; o++) { @@ -351,8 +351,8 @@ static void check_bf16_qkv(int n_threads) { wv[i] = f32_to_bf16(wv_f32[i]); } - qwen_set_threads(n_threads); - qwen_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim); + embed_set_threads(n_threads); + embed_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < q_dim; o++) { @@ -376,7 +376,7 @@ static void check_bf16_qkv(int n_threads) { static void test_bf16_qkv(void) { check_bf16_qkv(1); check_bf16_qkv(3); - qwen_set_threads(1); + embed_set_threads(1); } /* seq==1 routes through bf16_matvec_threaded; a pool splits the output @@ -392,8 +392,8 @@ static void check_bf16_matvec(int n_threads) { for (int i = 0; i < out_dim * in_dim; i++) w[i] = f32_to_bf16(cosf((float)i * 0.21f) + 0.05f); - qwen_set_threads(n_threads); - qwen_linear_nobias_bf16(got, x, w, 1, in_dim, out_dim); + embed_set_threads(n_threads); + embed_linear_nobias_bf16(got, x, w, 1, in_dim, out_dim); for (int o = 0; o < out_dim; o++) { float want = 0.0f; @@ -406,10 +406,10 @@ static void check_bf16_matvec(int n_threads) { static void test_bf16_matvec(void) { check_bf16_matvec(1); check_bf16_matvec(4); - qwen_set_threads(1); + embed_set_threads(1); } -/* Above QWEN_RMS_NORM_PARALLEL_ELEMS (256k elements) rms_norm fans out to +/* Above EMBED_RMS_NORM_PARALLEL_ELEMS (256k elements) rms_norm fans out to * rms_norm_worker; row-wise math is identical, so results must match the * single-thread run exactly. */ static void test_rms_norm_threaded(void) { @@ -436,11 +436,11 @@ static void test_rms_norm_threaded(void) { for (int i = 0; i < hidden; i++) w[i] = 0.8f + 0.01f * (float)(i % 7); - qwen_set_threads(1); - qwen_rms_norm(one, x, w, seq, hidden, 1e-6f); - qwen_set_threads(4); - qwen_rms_norm(many, x, w, seq, hidden, 1e-6f); - qwen_set_threads(1); + embed_set_threads(1); + embed_rms_norm(one, x, w, seq, hidden, 1e-6f); + embed_set_threads(4); + embed_rms_norm(many, x, w, seq, hidden, 1e-6f); + embed_set_threads(1); for (size_t i = 0; i < elems; i++) { if (one[i] != many[i]) { @@ -469,7 +469,7 @@ static void test_bf16_widen_buf(void) { for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) { int n = sizes[t]; memset(got, 0, sizeof(got)); - qwen_bf16_to_f32_buf(got, src, (size_t)n); + embed_bf16_to_f32_buf(got, src, (size_t)n); for (int i = 0; i < n; i++) { float want = bf16_to_f32(src[i]); uint32_t got_bits, want_bits; @@ -503,8 +503,8 @@ static void check_bf16_pair(int n_threads) { wb[i] = f32_to_bf16(wb_f32[i]); } - qwen_set_threads(n_threads); - qwen_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim); + embed_set_threads(n_threads); + embed_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < a_dim; o++) { @@ -525,10 +525,10 @@ static void check_bf16_pair(int n_threads) { static void test_bf16_pair(void) { check_bf16_pair(1); check_bf16_pair(4); - qwen_set_threads(1); + embed_set_threads(1); } -/* The dispatch macros in qwen_kernels_impl.h pick the SIMD variant at +/* The dispatch macros in embed_kernels_impl.h pick the SIMD variant at * compile time, so on arm64/x86 the generic C kernels are linked but never * called. Run them directly against the selected implementation on * remainder-lane-hostile sizes; on a plain build impl == generic and the @@ -554,46 +554,46 @@ static void test_generic_vs_impl(void) { } float yg[OUT], yi[OUT]; - qwen_bf16_matvec_fused_generic(yg, x, W, bias, IN, OUT); - qwen_bf16_matvec_fused_impl(yi, x, W, bias, IN, OUT); + embed_bf16_matvec_fused_generic(yg, x, W, bias, IN, OUT); + embed_bf16_matvec_fused_impl(yi, x, W, bias, IN, OUT); for (int o = 0; o < OUT; o++) expect_close("matvec_fused generic-vs-impl", yg[o], yi[o], 1e-5f); - qwen_bf16_matvec_fused_generic(yg, x, W, NULL, IN, OUT); - qwen_bf16_matvec_fused_impl(yi, x, W, NULL, IN, OUT); + embed_bf16_matvec_fused_generic(yg, x, W, NULL, IN, OUT); + embed_bf16_matvec_fused_impl(yi, x, W, NULL, IN, OUT); for (int o = 0; o < OUT; o++) expect_close("matvec_fused nobias generic-vs-impl", yg[o], yi[o], 1e-5f); const int sizes[] = {1, 7, 64, NMAX}; for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) { int n = sizes[t]; - expect_close("dot generic-vs-impl", qwen_dot_f32_generic(a, b, n), - qwen_dot_f32_impl(a, b, n), 1e-5f); + expect_close("dot generic-vs-impl", embed_dot_f32_generic(a, b, n), + embed_dot_f32_impl(a, b, n), 1e-5f); memcpy(d0, a, (size_t)n * sizeof(float)); memcpy(d1, a, (size_t)n * sizeof(float)); - qwen_vec_scale_inplace_generic(d0, 1.37f, n); - qwen_vec_scale_inplace_impl(d1, 1.37f, n); + embed_vec_scale_inplace_generic(d0, 1.37f, n); + embed_vec_scale_inplace_impl(d1, 1.37f, n); for (int i = 0; i < n; i++) expect_close("scale generic-vs-impl", d0[i], d1[i], 1e-6f); memcpy(d0, a, (size_t)n * sizeof(float)); memcpy(d1, a, (size_t)n * sizeof(float)); - qwen_vec_axpy_inplace_generic(d0, b, -0.61f, n); - qwen_vec_axpy_inplace_impl(d1, b, -0.61f, n); + embed_vec_axpy_inplace_generic(d0, b, -0.61f, n); + embed_vec_axpy_inplace_impl(d1, b, -0.61f, n); for (int i = 0; i < n; i++) expect_close("axpy generic-vs-impl", d0[i], d1[i], 1e-6f); memcpy(d0, a, (size_t)n * sizeof(float)); memcpy(d1, a, (size_t)n * sizeof(float)); - qwen_vec_scale_add_generic(d0, b, 0.83f, n); - qwen_vec_scale_add_impl(d1, b, 0.83f, n); + embed_vec_scale_add_generic(d0, b, 0.83f, n); + embed_vec_scale_add_impl(d1, b, 0.83f, n); for (int i = 0; i < n; i++) expect_close("scale_add generic-vs-impl", d0[i], d1[i], 1e-6f); } } int main(void) { - qwen_set_threads(1); + embed_set_threads(1); test_rms_norm(); test_layer_norm(); test_gelu(); diff --git a/tests/test_late.c b/tests/test_late.c index 661661c..ec6c03f 100644 --- a/tests/test_late.c +++ b/tests/test_late.c @@ -3,7 +3,7 @@ * byte-complete tokenizer fixture. Runs via `make test`. */ #include "embed.h" -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tiny_model.h" #include "tok_fixture.h" @@ -63,7 +63,7 @@ int main(void) { tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE}; int hidden = dims.hidden; - qwen_tokenizer_t *tok = NULL; + embed_tokenizer_t *tok = NULL; embed_model_t *base = NULL; embed_workspace_t *base_ws = NULL; embed_late_model_t *late = NULL; @@ -145,14 +145,14 @@ int main(void) { char vocab_path[1280]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); - tok = qwen_tokenizer_load(vocab_path); + tok = embed_tokenizer_load(vocab_path); if (!tok) { fprintf(stderr, "tokenizer load failed\n"); goto fail; } int n_tokens = 0; - ids = qwen_tokenizer_encode(tok, "hello world", &n_tokens); + ids = embed_tokenizer_encode(tok, "hello world", &n_tokens); if (!ids || n_tokens <= 1) { fprintf(stderr, "tokenization failed\n"); goto fail; @@ -244,7 +244,7 @@ int main(void) { int lens[3] = {0}; int offsets[4] = {0}; for (int i = 0; i < 3; i++) { - doc_ids[i] = qwen_tokenizer_encode(tok, doc_texts[i], &lens[i]); + doc_ids[i] = embed_tokenizer_encode(tok, doc_texts[i], &lens[i]); if (!doc_ids[i] || lens[i] <= 0) { fprintf(stderr, "doc tokenization failed\n"); goto fail; @@ -401,7 +401,7 @@ fail: free(raw); free(states); free(ids); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_late_workspace_free(ws); embed_late_model_free(late); embed_workspace_free(base_ws); diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c index 36ec8c5..ccfd6fb 100644 --- a/tests/test_safetensors.c +++ b/tests/test_safetensors.c @@ -2,7 +2,7 @@ * Hermetic: synthesizes every fixture in a temp directory. Runs via * `make test`. */ -#include "qwen_safetensors.h" +#include "safetensors.h" #include <stdint.h> #include <stdio.h> diff --git a/tests/test_server.c b/tests/test_server.c index 9ebb8f2..c0c85f1 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -6,7 +6,7 @@ * path over loopback sockets. Runs via `make test`. */ #define EMBED_SERVER_TEST -#include "../embed_server.c" +#include "../src/server.c" #include "tiny_model.h" #include "tok_fixture.h" @@ -596,7 +596,7 @@ static void test_http_embeddings(int port) { "{\"model\":\"Qwen3-Embedding-0.6B\"," "\"input\":\"hello\",\"encoding_format\":\"float\"}", NULL, &body) == 200); - cJSON *qwen_full = body ? parse_embeddings_body(body, 1) : NULL; + cJSON *embed_full = body ? parse_embeddings_body(body, 1) : NULL; free(body); body = NULL; TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, @@ -604,10 +604,10 @@ static void test_http_embeddings(int port) { "\"input\":\"hello\",\"dimensions\":32," "\"encoding_format\":\"float\"}", NULL, &body) == 200); - cJSON *qwen_mrl = body ? parse_embeddings_body(body, 1) : NULL; - if (qwen_full && qwen_mrl) { - cJSON *full_data = cJSON_GetObjectItemCaseSensitive(qwen_full, "data"); - cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data"); + cJSON *embed_mrl = body ? parse_embeddings_body(body, 1) : NULL; + if (embed_full && embed_mrl) { + cJSON *full_data = cJSON_GetObjectItemCaseSensitive(embed_full, "data"); + cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(embed_mrl, "data"); cJSON *full_emb = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(full_data, 0), "embedding"); cJSON *mrl_emb = @@ -622,12 +622,12 @@ static void test_http_embeddings(int port) { prefix_diff = d; } TEST_ASSERT(prefix_diff > 0.0); - cJSON *usage = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "usage"); + cJSON *usage = cJSON_GetObjectItemCaseSensitive(embed_mrl, "usage"); cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; TEST_ASSERT(total && total->valueint == 2); } - cJSON_Delete(qwen_full); - cJSON_Delete(qwen_mrl); + cJSON_Delete(embed_full); + cJSON_Delete(embed_mrl); free(body); body = NULL; @@ -949,7 +949,7 @@ static void test_http_rerank(int port) { } static void test_http_server(void) { - char dir[1024], late_dir[1024], qwen_dir[1024], gte_dir[1024]; + char dir[1024], late_dir[1024], embed_dir[1024], gte_dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); if (!mkdtemp(dir)) { @@ -964,9 +964,9 @@ static void test_http_server(void) { test_failures++; return; } - snprintf(qwen_dir, sizeof(qwen_dir), "%s/embed-srv-qwen-test-XXXXXX", + snprintf(embed_dir, sizeof(embed_dir), "%s/embed-srv-qwen-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - if (!mkdtemp(qwen_dir)) { + if (!mkdtemp(embed_dir)) { fprintf(stderr, "FAIL: Qwen mkdtemp\n"); test_failures++; return; @@ -985,8 +985,8 @@ static void test_http_server(void) { if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 || tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 || tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 || - tf_write_vocab(qwen_dir) != 0 || - tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0 || + tf_write_vocab(embed_dir) != 0 || + tm_write_qwen3_model_dims(embed_dir, "F32", &dims, TF_EOT_ID) != 0 || tf_write_vocab(gte_dir) != 0 || tm_write_qwen2_model_dims(gte_dir, "F32", >e_dims, TF_EOT_ID, 0) != 0) { fprintf(stderr, "FAIL: fixture write\n"); @@ -1008,7 +1008,7 @@ static void test_http_server(void) { ctx.spec[2].id = "pplx-embed-v1-late-0.6b"; ctx.spec[2].path = late_dir; ctx.spec[3].id = "Qwen3-Embedding-0.6B"; - ctx.spec[3].path = qwen_dir; + ctx.spec[3].path = embed_dir; ctx.spec[4].id = "gte-Qwen2-1.5B-instruct"; ctx.spec[4].path = gte_dir; ctx.cfg.models = ctx.spec; diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index 5920977..3ef4280 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -4,7 +4,7 @@ * integer-pair fast path, and that dropping the integer tables (forcing the * string-based fallback) produces identical ids. Runs via `make test`. */ -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tok_fixture.h" #include <stdio.h> @@ -14,7 +14,7 @@ #include <unistd.h> static int failures = 0; -int qwen_verbose = 0; +int embed_verbose = 0; #define TEST_ASSERT(cond) \ do { \ @@ -24,14 +24,14 @@ int qwen_verbose = 0; } \ } while (0) -static void check_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_ids(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, const int *want, int n_want, const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL); if (!ids) return; @@ -52,19 +52,19 @@ static void check_ids(const qwen_tokenizer_t *tok, /* Concatenated decode of an encoding must reproduce the (NFC-normalized) * input - byte-level BPE is lossless. */ -static void check_roundtrip(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_roundtrip(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, const char *want, const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL && n > 0); if (!ids) return; size_t cap = 1; for (int i = 0; i < n; i++) - cap += strlen(qwen_tokenizer_decode(tok, ids[i])); + cap += strlen(embed_tokenizer_decode(tok, ids[i])); char *got = (char *)malloc(cap); if (!got) { free(ids); @@ -73,7 +73,7 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, } got[0] = '\0'; for (int i = 0; i < n; i++) - strcat(got, qwen_tokenizer_decode(tok, ids[i])); + strcat(got, embed_tokenizer_decode(tok, ids[i])); if (strcmp(got, want) != 0) { fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got, want); @@ -84,18 +84,18 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, } /* Two spellings of the same text must encode identically. */ -static void check_same_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_same_ids(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *a, const char *b, const char *label) { int na = 0, nb = 0; - int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na); + int *ia = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, a, &na); if (!ia) { TEST_ASSERT(0); return; } - int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb); + int *ib = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, b, &nb); if (!ib) { free(ia); TEST_ASSERT(0); @@ -110,7 +110,7 @@ static void check_same_ids(const qwen_tokenizer_t *tok, } static void -run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) { +run_cases(const embed_tokenizer_t *tok, embed_tokenizer_workspace_t *ws, const char *label) { /* h e l l o -> he -> he,ll -> hell -> hello */ const int hello[] = {259}; check_ids(tok, ws, "hello", hello, 1, label); @@ -212,7 +212,7 @@ static void write_file(const char *dir, const char *name, const char *body) { static void test_loader_edges(const char *root) { char dir[1152], path[2048]; - TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL); + TEST_ASSERT(embed_tokenizer_load("/nonexistent/vocab.json") == NULL); /* Not a JSON object. */ snprintf(dir, sizeof(dir), "%s/badjson", root); @@ -223,7 +223,7 @@ static void test_loader_edges(const char *root) { write_file(dir, "vocab.json", "[1, 2]\n"); write_file(dir, "merges.txt", "a b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - TEST_ASSERT(qwen_tokenizer_load(path) == NULL); + TEST_ASSERT(embed_tokenizer_load(path) == NULL); /* Missing or pair-less merges.txt is tolerated: the tokenizer loads * and encoding falls back to byte/vocab-level (no merges applied). */ @@ -234,16 +234,16 @@ static void test_loader_edges(const char *root) { } write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *nom = qwen_tokenizer_load(path); + embed_tokenizer_t *nom = embed_tokenizer_load(path); TEST_ASSERT(nom != NULL); if (nom) { - qwen_tokenizer_workspace_t *nws = qwen_tokenizer_workspace_new(); + embed_tokenizer_workspace_t *nws = embed_tokenizer_workspace_new(); int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace(nom, nws, "ab", &n); + int *ids = embed_tokenizer_encode_with_workspace(nom, nws, "ab", &n); TEST_ASSERT(ids && n == 2 && ids[0] == 0 && ids[1] == 1); free(ids); - qwen_tokenizer_workspace_free(nws); - qwen_tokenizer_free(nom); + embed_tokenizer_workspace_free(nws); + embed_tokenizer_free(nom); } snprintf(dir, sizeof(dir), "%s/emptymerges", root); @@ -254,9 +254,9 @@ static void test_loader_edges(const char *root) { write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); write_file(dir, "merges.txt", "#version: 0.2\n\n \n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *emp = qwen_tokenizer_load(path); + embed_tokenizer_t *emp = embed_tokenizer_load(path); TEST_ASSERT(emp != NULL); - qwen_tokenizer_free(emp); + embed_tokenizer_free(emp); /* Escape forms in vocab keys; a malformed merge line is skipped. */ snprintf(dir, sizeof(dir), "%s/escapes", root); @@ -270,28 +270,28 @@ static void test_loader_edges(const char *root) { " \"s\\\\t\":6, \"p\\/q\":7, \"u\\tv\":8, \"\\u0041Z\":9}\n"); write_file(dir, "merges.txt", "#version: 0.2\nmalformed-line\na b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *tok = qwen_tokenizer_load(path); + embed_tokenizer_t *tok = embed_tokenizer_load(path); TEST_ASSERT(tok != NULL); if (tok) { - TEST_ASSERT(qwen_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "x\ny") == 4); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "q\"r") == 5); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "s\\t") == 6); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "p/q") == 7); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "u\tv") == 8); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "AZ") == 9); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 2), "ab") == 0); + TEST_ASSERT(embed_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3); + TEST_ASSERT(embed_tokenizer_token_id(tok, "x\ny") == 4); + TEST_ASSERT(embed_tokenizer_token_id(tok, "q\"r") == 5); + TEST_ASSERT(embed_tokenizer_token_id(tok, "s\\t") == 6); + TEST_ASSERT(embed_tokenizer_token_id(tok, "p/q") == 7); + TEST_ASSERT(embed_tokenizer_token_id(tok, "u\tv") == 8); + TEST_ASSERT(embed_tokenizer_token_id(tok, "AZ") == 9); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 2), "ab") == 0); /* Keys outside the GPT-2 byte alphabet decode to replacement. */ - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 3), "?") == 0); - qwen_tokenizer_free(tok); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 3), "?") == 0); + embed_tokenizer_free(tok); } /* A bare filename exercises the no-directory merges-path branch. */ char saved_cwd[1024]; if (getcwd(saved_cwd, sizeof(saved_cwd)) && chdir(dir) == 0) { - qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json"); + embed_tokenizer_t *rel = embed_tokenizer_load("vocab.json"); TEST_ASSERT(rel != NULL); - qwen_tokenizer_free(rel); + embed_tokenizer_free(rel); if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); @@ -310,7 +310,7 @@ int main(void) { char vocab_path[2048]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path); TEST_ASSERT(tok != NULL); if (!tok) return 1; @@ -319,48 +319,48 @@ int main(void) { TEST_ASSERT(tok->int_merges != NULL); TEST_ASSERT(tok->cp_to_id != NULL); - qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new(); + embed_tokenizer_workspace_t *ws = embed_tokenizer_workspace_new(); TEST_ASSERT(ws != NULL); run_cases(tok, ws, "fast"); /* decode: byte tokens, merged tokens, and out-of-range ids. */ - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 'a'), "a") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 259), "hello") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, -1), "") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(NULL, 0), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 'a'), "a") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 259), "hello") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, -1), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(NULL, 0), "") == 0); /* token_id lookups. */ - TEST_ASSERT(qwen_tokenizer_token_id(tok, "hello") == 259); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "no-such-token") == -1); - TEST_ASSERT(qwen_tokenizer_token_id(NULL, "a") == -1); - TEST_ASSERT(qwen_tokenizer_token_id(tok, NULL) == -1); + TEST_ASSERT(embed_tokenizer_token_id(tok, "hello") == 259); + TEST_ASSERT(embed_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID); + TEST_ASSERT(embed_tokenizer_token_id(tok, "no-such-token") == -1); + TEST_ASSERT(embed_tokenizer_token_id(NULL, "a") == -1); + TEST_ASSERT(embed_tokenizer_token_id(tok, NULL) == -1); /* encode_into: exact fit, truncation (-2 plus the untruncated prefix), * zero capacity, empty text, and argument guards. */ { int out[8]; int n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262); /* On overflow the prefix is written and n reports the total * required count, so a caller can re-size and retry. */ n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); - TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); + TEST_ASSERT(embed_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); } - TEST_ASSERT(qwen_tokenizer_encode(tok, "", NULL) == NULL); - TEST_ASSERT(qwen_tokenizer_encode(NULL, "x", NULL) == NULL); - qwen_tokenizer_workspace_free(NULL); - qwen_tokenizer_free(NULL); + TEST_ASSERT(embed_tokenizer_encode(tok, "", NULL) == NULL); + TEST_ASSERT(embed_tokenizer_encode(NULL, "x", NULL) == NULL); + embed_tokenizer_workspace_free(NULL); + embed_tokenizer_free(NULL); /* Drop the integer tables: the string-based fallback must produce the * exact same ids. */ @@ -371,8 +371,8 @@ int main(void) { tok->cp_to_id = NULL; run_cases(tok, ws, "slow"); - qwen_tokenizer_workspace_free(ws); - qwen_tokenizer_free(tok); + embed_tokenizer_workspace_free(ws); + embed_tokenizer_free(tok); test_loader_edges(dir); diff --git a/tests/test_wordpiece.c b/tests/test_wordpiece.c index 1fd6ff8..d40b7d2 100644 --- a/tests/test_wordpiece.c +++ b/tests/test_wordpiece.c @@ -3,7 +3,7 @@ * punctuation splitting, CJK char-splitting, special-token resolution, and the * workspace encode paths. No model files or network needed. */ -#include "wordpiece_tokenizer.h" +#include "tokenizer_wordpiece.h" #include <stdio.h> #include <stdlib.h> diff --git a/tests/test_workspace.c b/tests/test_workspace.c index a906689..c8b6ee8 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -4,7 +4,7 @@ * what scripts/check_workspace_api.py does. Runs via `make test`. */ #include "embed.h" -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tiny_model.h" #include "tok_fixture.h" @@ -199,10 +199,10 @@ int main(int argc, char **argv) { embed_workspace_t *batch_ws = embed_workspace_new(model); embed_workspace_t *single_ws = embed_workspace_new(model); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path); if (!batch_ws || !single_ws || !tok) { fprintf(stderr, "failed to allocate workspace or tokenizer\n"); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); @@ -227,7 +227,7 @@ int main(int argc, char **argv) { } for (int i = 0; i < batch; i++) { - ids[i] = qwen_tokenizer_encode(tok, texts[i], &ntok[i]); + ids[i] = embed_tokenizer_encode(tok, texts[i], &ntok[i]); if (!ids[i] || ntok[i] <= 0) { fprintf(stderr, "failed to tokenize input %d\n", i); goto fail; @@ -331,7 +331,7 @@ int main(int argc, char **argv) { /* Same separator resolution as the server: the fixture vocab defines * <|endoftext|>, real snapshots use the reserved id. */ - int sep_id = qwen_tokenizer_token_id(tok, "<|endoftext|>"); + int sep_id = embed_tokenizer_token_id(tok, "<|endoftext|>"); if (sep_id < 0) sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID; if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, cfg) != 0) { @@ -349,7 +349,7 @@ int main(int argc, char **argv) { for (int i = 0; i < batch; i++) free(ids[i]); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); @@ -367,7 +367,7 @@ int main(int argc, char **argv) { fail: for (int i = 0; i < batch; i++) free(ids[i]); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); |
