diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | Makefile | 31 | ||||
| -rw-r--r-- | bench/bench_tokenizer.c | 140 | ||||
| -rw-r--r-- | src/cli.c | 71 | ||||
| -rw-r--r-- | src/config.c | 20 | ||||
| -rw-r--r-- | src/cuda.cu | 10 | ||||
| -rw-r--r-- | src/embed.h | 2 | ||||
| -rw-r--r-- | src/forward.c | 15 | ||||
| -rw-r--r-- | src/internal.h | 2 | ||||
| -rw-r--r-- | src/mlx.c | 20 | ||||
| -rw-r--r-- | src/model.c | 10 | ||||
| -rw-r--r-- | src/server_internal.h | 9 | ||||
| -rw-r--r-- | src/server_models.c | 68 | ||||
| -rw-r--r-- | src/tokenizer_sentencepiece.c | 963 | ||||
| -rw-r--r-- | src/tokenizer_sentencepiece.h | 99 | ||||
| -rw-r--r-- | tests/test_sentencepiece.c | 266 | ||||
| -rw-r--r-- | tests/test_xlm_roberta.c | 96 |
17 files changed, 1775 insertions, 51 deletions
@@ -14,6 +14,8 @@ tests/test_qwen3 tests/test_cli tests/test_cls tests/test_wordpiece +test_sentencepiece +test_xlm_roberta tests/cli_under_test bench/bench_tokenizer bench/bench_kernels @@ -28,3 +30,5 @@ __pycache__/ tmp private .vscode +Launch_*.trace/ + @@ -79,7 +79,7 @@ LDFLAGS = -lm -lpthread $(BACKEND_LDFLAGS) KERNEL_SRCS = src/kernels_gemm.c src/kernels_norm.c src/kernels_act.c src/kernels_attn.c \ src/kernels_rope.c src/threadpool.c \ src/kernels_generic.c src/kernels_neon.c src/kernels_avx.c -TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c +TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c src/tokenizer_sentencepiece.c CORE_SRCS = src/model.c src/forward.c src/late.c src/vec.c src/alloc.c \ src/config.c src/safetensors.c $(KERNEL_SRCS) @@ -203,11 +203,13 @@ TEST_CC_FLAGS = -Wall -Wextra -O2 -Isrc TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS) TEST_TOKENIZER_SRCS = tests/test_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS) TEST_WORDPIECE_SRCS = tests/test_wordpiece.c src/tokenizer_wordpiece.c +TEST_SENTENCEPIECE_SRCS = tests/test_sentencepiece.c src/tokenizer_sentencepiece.c TEST_SAFETENSORS_SRCS = tests/test_safetensors.c src/safetensors.c TEST_BF16_SRCS = tests/test_bf16_model.c $(CORE_SRCS) TEST_QWEN3_SRCS = tests/test_qwen3.c $(CORE_SRCS) TEST_QWEN2_SRCS = tests/test_qwen2.c $(CORE_SRCS) TEST_CLS_SRCS = tests/test_cls.c $(CORE_SRCS) +TEST_XLM_ROBERTA_SRCS = tests/test_xlm_roberta.c $(CORE_SRCS) TEST_WORKSPACE_SRCS = tests/test_workspace.c $(CORE_SRCS) src/tokenizer_bpe.c TEST_LATE_SRCS = tests/test_late.c $(CORE_SRCS) src/tokenizer_bpe.c # test_server.c #includes ../src/server.c, so server is not a separate object @@ -232,6 +234,8 @@ test: ./tests/test_tokenizer $(CC) $(TEST_CC_FLAGS) -o tests/test_wordpiece $(TEST_WORDPIECE_SRCS) ./tests/test_wordpiece + $(CC) $(TEST_CC_FLAGS) -o tests/test_sentencepiece $(TEST_SENTENCEPIECE_SRCS) + ./tests/test_sentencepiece $(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS) ./tests/test_safetensors $(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \ @@ -246,6 +250,9 @@ test: $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \ $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) ./tests/test_cls + $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \ + $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + ./tests/test_xlm_roberta $(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \ $(TEST_WORKSPACE_SRCS) -lm -lpthread ./tests/test_workspace @@ -280,9 +287,9 @@ COV_DIR = coverage COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \ -fcoverage-compilation-dir=. COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \ - tests/test_tokenizer tests/test_safetensors \ + tests/test_tokenizer tests/test_sentencepiece tests/test_safetensors \ tests/test_bf16_model tests/test_qwen3 tests/test_qwen2 tests/test_workspace \ - tests/test_late \ + tests/test_cls tests/test_xlm_roberta tests/test_late \ tests/cli_under_test tests/test_server # Report on project sources only - not the harnesses, vendored deps, or # system headers (cJSON lands under /opt or /usr otherwise). @@ -307,6 +314,9 @@ coverage: $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_tokenizer \ $(TEST_TOKENIZER_SRCS) -lm -lpthread LLVM_PROFILE_FILE=$(COV_DIR)/tokenizer.profraw ./tests/test_tokenizer + $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_sentencepiece \ + $(TEST_SENTENCEPIECE_SRCS) + LLVM_PROFILE_FILE=$(COV_DIR)/sentencepiece.profraw ./tests/test_sentencepiece $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS) LLVM_PROFILE_FILE=$(COV_DIR)/safetensors.profraw ./tests/test_safetensors $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_bf16_model \ @@ -318,6 +328,12 @@ coverage: $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \ $(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) LLVM_PROFILE_FILE=$(COV_DIR)/qwen2.profraw ./tests/test_qwen2 + $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \ + $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + LLVM_PROFILE_FILE=$(COV_DIR)/cls.profraw ./tests/test_cls + $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \ + $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS) + LLVM_PROFILE_FILE=$(COV_DIR)/xlm_roberta.profraw ./tests/test_xlm_roberta $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \ $(TEST_WORKSPACE_SRCS) -lm -lpthread LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace @@ -349,8 +365,8 @@ coverage: bench-tokenizer: $(CC) -Wall -Wextra -O3 -march=native -Isrc -o bench/bench_tokenizer \ - bench/bench_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS) -lm -lpthread - @echo "run: bench/bench_tokenizer <model_dir> [runs]" + bench/bench_tokenizer.c $(TOKENIZER_SRCS) $(KERNEL_SRCS) -lm -lpthread + @echo "run: bench/bench_tokenizer <model_dir-or-vocab.json> [runs] TEXT..." # ============================================================================= # Regression microbenchmarks (see bench/bench.h). Each run records a JSON @@ -394,7 +410,8 @@ clean: $(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \ tests/test_kernels_generic tests/test_kernels_blas \ tests/test_safetensors tests/test_bf16_model tests/test_server \ - tests/test_qwen3 tests/test_qwen2 \ - tests/test_tokenizer tests/test_wordpiece tests/test_workspace tests/test_cli tests/test_late \ + tests/test_qwen3 tests/test_qwen2 tests/test_cls tests/test_xlm_roberta \ + tests/test_tokenizer tests/test_wordpiece tests/test_sentencepiece \ + tests/test_workspace tests/test_cli tests/test_late \ tests/cli_under_test bench/bench_tokenizer bench/bench_kernels bench/bench_model rm -rf $(COV_DIR) diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index bf4d0b7..8175ffe 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -1,11 +1,14 @@ /* bench/bench_tokenizer.c - tokenizer encode-path benchmark. - * Build via `make bench-tokenizer`; needs a model dir (vocab.json). */ + * Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */ #include "tokenizer_bpe.h" +#include "tokenizer_sentencepiece.h" +#include "tokenizer_wordpiece.h" #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <unistd.h> #include <time.h> int embed_verbose = 0; @@ -26,6 +29,110 @@ static int same_ids(const int *a, int na, const int *b, int nb) { return 1; } +typedef enum { + TOK_BPE, + TOK_WORDPIECE, + TOK_SENTENCEPIECE, +} tok_kind_t; + +typedef struct { + tok_kind_t kind; + void *tok; + void *ws; +} bench_tok_t; + +static int path_exists(const char *path) { + return access(path, R_OK) == 0; +} + +static int load_tokenizer(bench_tok_t *bt, const char *path) { + char p[1024]; + snprintf(p, sizeof(p), "%s/vocab.txt", path); + if (path_exists(p)) { + bt->kind = TOK_WORDPIECE; + bt->tok = wordpiece_tokenizer_load(path); + bt->ws = wordpiece_workspace_new(); + return bt->tok && bt->ws ? 0 : -1; + } + snprintf(p, sizeof(p), "%s/sentencepiece.bpe.model", path); + if (!path_exists(p)) { + snprintf(p, sizeof(p), "%s/tokenizer.model", path); + if (!path_exists(p)) + snprintf(p, sizeof(p), "%s/spiece.model", path); + } + if (path_exists(p)) { + bt->kind = TOK_SENTENCEPIECE; + bt->tok = sentencepiece_tokenizer_load(path); + bt->ws = sentencepiece_workspace_new(); + return bt->tok && bt->ws ? 0 : -1; + } + bt->kind = TOK_BPE; + const char *vocab = path; + snprintf(p, sizeof(p), "%s/vocab.json", path); + if (path_exists(p)) + vocab = p; + bt->tok = embed_tokenizer_load(vocab); + bt->ws = embed_tokenizer_workspace_new(); + return bt->tok && bt->ws ? 0 : -1; +} + +static int *tok_encode(bench_tok_t *bt, const char *text, int *n) { + switch (bt->kind) { + case TOK_WORDPIECE: + return wordpiece_tokenizer_encode((wordpiece_tokenizer_t *)bt->tok, text, n); + case TOK_SENTENCEPIECE: + return sentencepiece_tokenizer_encode((sentencepiece_tokenizer_t *)bt->tok, text, n); + default: + return embed_tokenizer_encode((embed_tokenizer_t *)bt->tok, text, n); + } +} + +static int *tok_encode_ws(bench_tok_t *bt, const char *text, int *n) { + switch (bt->kind) { + case TOK_WORDPIECE: + return wordpiece_tokenizer_encode_with_workspace((wordpiece_tokenizer_t *)bt->tok, + (wordpiece_workspace_t *)bt->ws, text, n); + case TOK_SENTENCEPIECE: + return sentencepiece_tokenizer_encode_with_workspace( + (sentencepiece_tokenizer_t *)bt->tok, (sentencepiece_workspace_t *)bt->ws, text, n); + default: + return embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)bt->tok, + (embed_tokenizer_workspace_t *)bt->ws, text, n); + } +} + +static int tok_encode_into(bench_tok_t *bt, const char *text, int *out, int cap, int *n) { + switch (bt->kind) { + case TOK_WORDPIECE: + return wordpiece_tokenizer_encode_into((wordpiece_tokenizer_t *)bt->tok, + (wordpiece_workspace_t *)bt->ws, text, out, cap, n); + case TOK_SENTENCEPIECE: + return sentencepiece_tokenizer_encode_into((sentencepiece_tokenizer_t *)bt->tok, + (sentencepiece_workspace_t *)bt->ws, text, out, + cap, n); + default: + return embed_tokenizer_encode_into((embed_tokenizer_t *)bt->tok, + (embed_tokenizer_workspace_t *)bt->ws, text, out, cap, n); + } +} + +static void free_tokenizer(bench_tok_t *bt) { + switch (bt->kind) { + case TOK_WORDPIECE: + wordpiece_workspace_free((wordpiece_workspace_t *)bt->ws); + wordpiece_tokenizer_free((wordpiece_tokenizer_t *)bt->tok); + break; + case TOK_SENTENCEPIECE: + sentencepiece_workspace_free((sentencepiece_workspace_t *)bt->ws); + sentencepiece_tokenizer_free((sentencepiece_tokenizer_t *)bt->tok); + break; + default: + embed_tokenizer_workspace_free((embed_tokenizer_workspace_t *)bt->ws); + embed_tokenizer_free((embed_tokenizer_t *)bt->tok); + break; + } +} + int main(int argc, char **argv) { int status = 1; int **gold = NULL; @@ -33,8 +140,10 @@ int main(int argc, char **argv) { int **bufs = NULL; int *caps = NULL; + bench_tok_t bt = {0}; + if (argc < 4) { - fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]); + fprintf(stderr, "usage: %s MODEL_DIR_OR_VOCAB_JSON RUNS TEXT...\n", argv[0]); return 2; } @@ -45,9 +154,7 @@ int main(int argc, char **argv) { int n_texts = argc - 3; char **texts = argv + 3; - embed_tokenizer_t *tok = embed_tokenizer_load(vocab); - embed_tokenizer_workspace_t *ws = embed_tokenizer_workspace_new(); - if (!tok || !ws) { + if (load_tokenizer(&bt, vocab) != 0) { fprintf(stderr, "failed to initialize tokenizer\n"); goto cleanup; } @@ -61,14 +168,14 @@ int main(int argc, char **argv) { int tokens_per_pass = 0; for (int i = 0; i < n_texts; i++) { - gold[i] = embed_tokenizer_encode(tok, texts[i], &gold_n[i]); - if (!gold[i] || gold_n[i] <= 0) + gold[i] = tok_encode(&bt, texts[i], &gold_n[i]); + if (gold_n[i] < 0 || (!gold[i] && gold_n[i] != 0)) goto cleanup; tokens_per_pass += gold_n[i]; int n_ws = 0; - int *ids_ws = embed_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws); - if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) { + int *ids_ws = tok_encode_ws(&bt, texts[i], &n_ws); + if ((!ids_ws && n_ws != 0) || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) { free(ids_ws); goto cleanup; } @@ -81,7 +188,7 @@ int main(int argc, char **argv) { if (!bufs[i]) goto cleanup; int n_into = 0; - if (embed_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 || + if (tok_encode_into(&bt, texts[i], bufs[i], caps[i], &n_into) != 0 || !same_ids(gold[i], gold_n[i], bufs[i], n_into)) goto cleanup; } @@ -91,8 +198,8 @@ int main(int argc, char **argv) { for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int *ids = embed_tokenizer_encode(tok, texts[i], &n); - if (!ids || n != gold_n[i]) { + int *ids = tok_encode(&bt, texts[i], &n); + if ((!ids && n != 0) || n != gold_n[i]) { free(ids); goto cleanup; } @@ -107,8 +214,8 @@ int main(int argc, char **argv) { for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int *ids = embed_tokenizer_encode_with_workspace(tok, ws, texts[i], &n); - if (!ids || n != gold_n[i]) { + int *ids = tok_encode_ws(&bt, texts[i], &n); + if ((!ids && n != 0) || n != gold_n[i]) { free(ids); goto cleanup; } @@ -123,7 +230,7 @@ int main(int argc, char **argv) { for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int rc = embed_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n); + int rc = tok_encode_into(&bt, texts[i], bufs[i], caps[i], &n); if (rc != 0 || n != gold_n[i]) goto cleanup; into_tokens += n; @@ -152,7 +259,6 @@ cleanup: free(gold_n); free(bufs); free(caps); - embed_tokenizer_workspace_free(ws); - embed_tokenizer_free(tok); + free_tokenizer(&bt); return status; } @@ -4,6 +4,7 @@ #include "build.h" #include "kernels.h" #include "tokenizer_bpe.h" +#include "tokenizer_sentencepiece.h" #include "tokenizer_wordpiece.h" #ifdef USE_MLX @@ -103,6 +104,10 @@ typedef struct { * Qwen byte-level BPE fields above stay NULL. */ wordpiece_tokenizer_t *wp_tok; wordpiece_workspace_t *wp_tok_ws; + /* SentencePiece/XLM-R tokenizer. When sp_tok is set, tokenize_text wraps + * core ids with <s> ... </s>. */ + sentencepiece_tokenizer_t *sp_tok; + sentencepiece_workspace_t *sp_tok_ws; int cls_id; int sep_id; int dim; @@ -115,6 +120,8 @@ static void engine_free_tokenizers(engine_t *e) { embed_tokenizer_free(e->tok); wordpiece_workspace_free(e->wp_tok_ws); wordpiece_tokenizer_free(e->wp_tok); + sentencepiece_workspace_free(e->sp_tok_ws); + sentencepiece_tokenizer_free(e->sp_tok); } typedef struct { @@ -157,6 +164,24 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) { ids[n + 1] = e->sep_id; out->ids = ids; out->n_tokens = n + 2; + } else if (e->sp_tok) { + int n = 0; + int *core = + sentencepiece_tokenizer_encode_with_workspace(e->sp_tok, e->sp_tok_ws, text, &n); + if (n < 0 || n > INT_MAX - 2) { + free(core); + return -1; + } + int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids)); + if (!ids) { + free(core); + return -1; + } + memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); + ids[0] = e->cls_id; + ids[n + 1] = e->sep_id; + out->ids = ids; + out->n_tokens = n + 2; } else { out->ids = embed_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens); if (!out->ids || out->n_tokens == 0) { @@ -688,8 +713,8 @@ int main(int argc, char *argv[]) { fprintf(stderr, "Using %d CPU thread(s)\n", n_threads); } - /* Tokenizer: a WordPiece vocab.txt selects the BERT tokenizer (the frontend - * wraps [CLS]/[SEP]); otherwise the Qwen byte-level BPE vocab.json. */ + /* Tokenizer selection by files present: WordPiece vocab.txt for BERT, + * SentencePiece .model for XLM-R/RoBERTa, otherwise Qwen byte-level BPE. */ engine_t e = {0}; e.cls_id = -1; e.sep_id = -1; @@ -708,6 +733,27 @@ int main(int argc, char *argv[]) { return 1; } } else { + char sp_path[1024]; + snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", model_dir); + if (access(sp_path, R_OK) != 0) { + snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", model_dir); + if (access(sp_path, R_OK) != 0) + snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", model_dir); + } + if (access(sp_path, R_OK) == 0) { + e.sp_tok = sentencepiece_tokenizer_load(model_dir); + if (e.sp_tok) { + e.cls_id = sentencepiece_tokenizer_token_id(e.sp_tok, "<s>"); + e.sep_id = sentencepiece_tokenizer_token_id(e.sp_tok, "</s>"); + } + if (!e.sp_tok || e.cls_id < 0 || e.sep_id < 0) { + fprintf(stderr, "failed to load SentencePiece tokenizer: %s\n", sp_path); + sentencepiece_tokenizer_free(e.sp_tok); + return 1; + } + } + } + if (!e.wp_tok && !e.sp_tok) { char vocab_path[1024]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir); e.tok = embed_tokenizer_load(vocab_path); @@ -773,6 +819,23 @@ int main(int argc, char *argv[]) { * like the contextual separator: a vocab that defines it (test models) * wins; otherwise the released-model family constant applies. */ if (e.append_terminal_token) { + if (!e.tok) { + fprintf(stderr, "terminal-token appending requires the BPE tokenizer\n"); + if (e.workspace) + embed_workspace_free(e.workspace); + if (e.model) + embed_model_free(e.model); +#ifdef USE_MLX + if (e.mlx_ctx) + embed_mlx_free(e.mlx_ctx); +#endif +#ifdef USE_CUDA + if (e.cuda_ctx) + embed_cuda_free(e.cuda_ctx); +#endif + engine_free_tokenizers(&e); + return 1; + } int eot = embed_tokenizer_token_id(e.tok, "<|endoftext|>"); e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID; } @@ -789,9 +852,11 @@ int main(int argc, char *argv[]) { if (e.wp_tok) e.wp_tok_ws = wordpiece_workspace_new(); + else if (e.sp_tok) + e.sp_tok_ws = sentencepiece_workspace_new(); else e.tok_ws = embed_tokenizer_workspace_new(); - if (!e.tok_ws && !e.wp_tok_ws) { + if (!e.tok_ws && !e.wp_tok_ws && !e.sp_tok_ws) { fprintf(stderr, "failed to allocate tokenizer workspace\n"); if (e.workspace) embed_workspace_free(e.workspace); diff --git a/src/config.c b/src/config.c index f693843..a33a135 100644 --- a/src/config.c +++ b/src/config.c @@ -209,6 +209,8 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12); cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0); cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0); + cfg->position_id_offset = 0; + cfg->type_vocab_size = json_get_int(buf, "type_vocab_size", 2); cfg->family = EMBED_FAMILY_QWEN3; cfg->qk_norm = 1; cfg->qkv_bias = 0; @@ -226,13 +228,25 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { cfg->qk_norm = 0; cfg->qkv_bias = 1; cfg->attention_mode = EMBED_ATTENTION_CAUSAL; - } else if (json_string_equals(buf, "model_type", "bert")) { + } else if (json_string_equals(buf, "model_type", "bert") || + json_string_equals(buf, "model_type", "roberta") || + json_string_equals(buf, "model_type", "xlm-roberta")) { cfg->family = EMBED_FAMILY_BERT; cfg->qk_norm = 0; cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */ cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL; cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */ cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0; + if (json_string_equals(buf, "model_type", "roberta") || + json_string_equals(buf, "model_type", "xlm-roberta")) { + int pad_id = json_get_int(buf, "pad_token_id", 1); + if (pad_id < 0 || pad_id == INT_MAX) { + fprintf(stderr, "embed_config: invalid RoBERTa pad_token_id\n"); + free(buf); + return -1; + } + cfg->position_id_offset = pad_id + 1; + } /* hidden_act selects the feed-forward GeLU curve. gelu_new and * gelu_pytorch_tanh are the same tanh approximation; everything else * (gelu, absent) is the exact erf GeLU the released encoders use. */ @@ -240,7 +254,9 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) { json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) { cfg->ffn_act = EMBED_ACT_GELU_TANH; } - if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) || + if (cfg->max_position_embeddings <= 0 || + cfg->position_id_offset >= cfg->max_position_embeddings || + cfg->type_vocab_size <= 0 || !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n"); free(buf); diff --git a/src/cuda.cu b/src/cuda.cu index 6ce7dfb..872834b 100644 --- a/src/cuda.cu +++ b/src/cuda.cu @@ -1337,8 +1337,7 @@ static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) { const embed_weights_t *w = &cpu->weights; if (load_vector(&ctx->position_embeddings, w->position_embeddings, c->max_position_embeddings * c->hidden_size) != 0 || - load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) != - 0 || + load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) != 0 || load_vector(&ctx->embed_ln_w, w->embed_ln_w, c->hidden_size) != 0 || load_vector(&ctx->embed_ln_b, w->embed_ln_b, c->hidden_size) != 0) { embed_cuda_free(ctx); @@ -1436,6 +1435,11 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx, if (inputs[b].n_tokens > max_seq) max_seq = inputs[b].n_tokens; } + if (ctx->config.family == EMBED_FAMILY_BERT && + max_seq > ctx->config.max_position_embeddings - ctx->config.position_id_offset) { + free(h_offsets); + return -1; + } h_offsets[batch] = total; int *h_ids = (int *)malloc((size_t)total * sizeof(int)); int *h_pos = (int *)malloc((size_t)total * sizeof(int)); @@ -1449,7 +1453,7 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx, int off = h_offsets[b]; memcpy(h_ids + off, inputs[b].ids, (size_t)inputs[b].n_tokens * sizeof(int)); for (int i = 0; i < inputs[b].n_tokens; i++) - h_pos[off + i] = i; + h_pos[off + i] = ctx->config.position_id_offset + i; } if (ensure_buffers(ctx, total, batch, max_seq) != 0) { free(h_offsets); diff --git a/src/embed.h b/src/embed.h index 028b3d8..5c4f2f3 100644 --- a/src/embed.h +++ b/src/embed.h @@ -69,6 +69,8 @@ typedef struct { int intermediate_size; int vocab_size; int max_position_embeddings; /* BERT learned position table size */ + int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */ + int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */ float rms_norm_eps; float layer_norm_eps; /* BERT LayerNorm epsilon */ float rope_theta; diff --git a/src/forward.c b/src/forward.c index d5818a4..28243af 100644 --- a/src/forward.c +++ b/src/forward.c @@ -296,8 +296,9 @@ int forward_packed_inplace(const embed_model_t *model, * embedding LayerNorm, then post-norm encoder layers (bias on every projection, * GeLU feed-forward, no RoPE, no per-head norm). Bidirectional attention is * block-diagonal over the packed batch, so each sequence's positions restart at - * 0. Leaves the final hidden states in ws->x; pooling reads them as-is because - * the last layer's LayerNorm is already the model's final normalization. */ + * cfg->position_id_offset (0 for BERT, pad_id + 1 for RoBERTa/XLM-R). Leaves the + * final hidden states in ws->x; pooling reads them as-is because the last + * layer's LayerNorm is already the model's final normalization. */ static int forward_packed_bert(const embed_model_t *model, embed_workspace_t *ws, const embed_input_t *inputs, @@ -320,9 +321,10 @@ static int forward_packed_bert(const embed_model_t *model, if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings || !w->embed_ln_w || !w->embed_ln_b) return -1; - if (max_seq > cfg->max_position_embeddings) { - fprintf(stderr, "embed: sequence length %d exceeds max_position_embeddings %d\n", max_seq, - cfg->max_position_embeddings); + if (max_seq > cfg->max_position_embeddings - cfg->position_id_offset) { + fprintf(stderr, + "embed: sequence length %d plus position offset %d exceeds max_position_embeddings %d\n", + max_seq, cfg->position_id_offset, cfg->max_position_embeddings); return -1; } if (ensure_buffers(ws, cfg, total_seq) != 0) @@ -345,7 +347,8 @@ static int forward_packed_bert(const embed_model_t *model, } float *row = x + (size_t)(start + i) * hidden; copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden); - embed_add_inplace(row, w->position_embeddings + (size_t)i * hidden, hidden); + int pos = cfg->position_id_offset + i; + embed_add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); embed_add_inplace(row, w->token_type_embeddings, hidden); } } diff --git a/src/internal.h b/src/internal.h index cdc02a1..b3c2aa0 100644 --- a/src/internal.h +++ b/src/internal.h @@ -43,7 +43,7 @@ typedef struct { /* BERT family (NULL for Qwen3): learned absolute position embeddings, * token-type (segment) embeddings, and the embedding LayerNorm. */ const float *position_embeddings; /* [max_position_embeddings, hidden] */ - const float *token_type_embeddings; /* [2, hidden] */ + const float *token_type_embeddings; /* [type_vocab_size, hidden] */ const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */ const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */ } embed_weights_t; @@ -392,8 +392,10 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye p = ""; else if (multi_safetensors_find(ctx->ms, "bert.embeddings.word_embeddings.weight", NULL)) p = "bert."; + else if (multi_safetensors_find(ctx->ms, "roberta.embeddings.word_embeddings.weight", NULL)) + p = "roberta."; else { - fprintf(stderr, "mlx: BERT word embeddings not found\n"); + fprintf(stderr, "mlx: BERT-family word embeddings not found\n"); return -1; } @@ -404,7 +406,7 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye int pos_shape[] = {c->max_position_embeddings, h}; snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p); ctx->position_embeddings = load_tensor(ctx->ms, name, pos_shape, 2); - int type_shape[] = {2, h}; + int type_shape[] = {c->type_vocab_size, h}; snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p); ctx->token_type_embeddings = load_tensor(ctx->ms, name, type_shape, 2); int ln_shape[] = {h}; @@ -2046,9 +2048,19 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx, mlx_array x_normed; if (c->family == EMBED_FAMILY_BERT) { int start2[] = {0, 0}, stride2[] = {1, 1}; - int pos_stop[] = {max_seq, hidden}, type_stop[] = {1, hidden}; + int pos_start[] = {c->position_id_offset, 0}; + int pos_stop[] = {c->position_id_offset + max_seq, hidden}, type_stop[] = {1, hidden}; + if (pos_stop[0] > c->max_position_embeddings) { + mlx_array_free(x); + mlx_array_free(pool_mask); + mlx_array_free(lengths); + mlx_array_free(last_indices); + if (needs_attn_mask) + mlx_array_free(attn_mask); + return -1; + } mlx_array pos = mlx_array_new(); - mlx_slice(&pos, ctx->position_embeddings, start2, 2, pos_stop, 2, stride2, 2, S); + mlx_slice(&pos, ctx->position_embeddings, pos_start, 2, pos_stop, 2, stride2, 2, S); mlx_array xp = mlx_array_new(); mlx_add(&xp, x, pos, S); mlx_array_free(x); diff --git a/src/model.c b/src/model.c index 095ce63..2956d7f 100644 --- a/src/model.c +++ b/src/model.c @@ -173,15 +173,17 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start embed_weights_t *w = &model->weights; char name[256]; - /* sentence-transformers checkpoints store the BERT submodule unprefixed; - * raw HF checkpoints prefix every tensor with "bert.". */ + /* sentence-transformers checkpoints store encoder tensors unprefixed; raw + * HF BERT and RoBERTa/XLM-R checkpoints use "bert." or "roberta.". */ const char *p; if (multi_safetensors_find(ms, "embeddings.word_embeddings.weight", NULL)) p = ""; else if (multi_safetensors_find(ms, "bert.embeddings.word_embeddings.weight", NULL)) p = "bert."; + else if (multi_safetensors_find(ms, "roberta.embeddings.word_embeddings.weight", NULL)) + p = "roberta."; else { - fprintf(stderr, "embed_model_load: BERT word embeddings not found\n"); + fprintf(stderr, "embed_model_load: BERT-family word embeddings not found\n"); return -1; } @@ -194,7 +196,7 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start const int64_t pos_shape[2] = {cfg->max_position_embeddings, hidden}; snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p); w->position_embeddings = load_norm_f32(ms, name, pos_shape, 2); - const int64_t type_shape[2] = {2, hidden}; + const int64_t type_shape[2] = {cfg->type_vocab_size, hidden}; snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p); w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2); const int64_t ln_shape[1] = {hidden}; diff --git a/src/server_internal.h b/src/server_internal.h index e4d7992..56bf25b 100644 --- a/src/server_internal.h +++ b/src/server_internal.h @@ -10,6 +10,7 @@ #include "embed.h" #include "sbuf.h" #include "tokenizer_bpe.h" +#include "tokenizer_sentencepiece.h" #include "tokenizer_wordpiece.h" #ifdef USE_MLX @@ -54,6 +55,10 @@ #define EMBED_GTE_QWEN2_QUERY_INSTRUCT \ "Instruct: Given a web search query, retrieve relevant passages that answer the " \ "query\nQuery: " +#define EMBED_E5_QUERY_INSTRUCT \ + "Instruct: Given a web search query, retrieve relevant passages that answer the " \ + "query\nQuery: " +#define EMBED_SNOWFLAKE_QUERY_INSTRUCT "query: " /* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent * short-request throughput over 8 with no long-document penalty, while 128 * was no faster and inflated long-document tail latency. */ @@ -80,6 +85,8 @@ typedef enum { MODEL_BGE_SMALL, MODEL_BGE_BASE, MODEL_BGE_LARGE, + MODEL_E5_MULTI_LARGE_INSTRUCT, + MODEL_SNOWFLAKE_ARCTIC_L_V2, MODEL_CTX_06, MODEL_CTX_4, MODEL_LATE_06, @@ -124,6 +131,8 @@ typedef struct { * Qwen byte-level BPE fields above stay NULL. */ wordpiece_tokenizer_t *wp_tok; wordpiece_workspace_t *wp_tok_ws; + sentencepiece_tokenizer_t *sp_tok; + sentencepiece_workspace_t *sp_tok_ws; int cls_id; int sep_id; int context_separator_id; diff --git a/src/server_models.c b/src/server_models.c index ebe5811..32199d9 100644 --- a/src/server_models.c +++ b/src/server_models.c @@ -13,7 +13,7 @@ * k_models is the static table of every model the server can serve (name, * dimensions, attention/pooling mode, API family). load_one_model resolves a * served id to a slot, picks the tokenizer by the files present (WordPiece - * vocab.txt -> BERT, else byte-level BPE vocab.json), and loads the CPU model + * vocab.txt, SentencePiece .model, else byte-level BPE vocab.json), and loads the CPU model * (GPU backends load on the worker thread). The tokenize_* and model_embed_* * helpers are the single dispatch point the request handlers call, hiding the * CPU/MLX/CUDA backend choice behind one signature. @@ -43,6 +43,12 @@ const model_info k_models[] = { EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, {"bge-large-en-v1.5", MODEL_KIND_STANDARD, 1024, 1024, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL}, + {"multilingual-e5-large-instruct", MODEL_KIND_STANDARD, 1024, 1024, 0, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 1, EMBED_API_OPENAI, + EMBED_E5_QUERY_INSTRUCT}, + {"snowflake-arctic-embed-l-v2.0", MODEL_KIND_STANDARD, 1024, 256, 0, + EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI, + EMBED_SNOWFLAKE_QUERY_INSTRUCT}, {"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL}, {"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL, @@ -143,6 +149,30 @@ int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) { out->n_tokens = n + 2; return 0; } + if (m->sp_tok) { + int n = 0; + int *core = + sentencepiece_tokenizer_encode_with_workspace(m->sp_tok, m->sp_tok_ws, text, &n); + if (j) + j->tokenize_ns += nstime() - t0; + if (n < 0 || n > INT_MAX - 2) { + free(core); + memset(out, 0, sizeof(*out)); + return -1; + } + int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids)); + if (!ids) { + free(core); + memset(out, 0, sizeof(*out)); + return -1; + } + memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); + ids[0] = m->cls_id; + ids[n + 1] = m->sep_id; + out->ids = ids; + out->n_tokens = n + 2; + return 0; + } out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens); if (j) j->tokenize_ns += nstime() - t0; @@ -299,9 +329,8 @@ int load_one_model(http_server *s, model_slot slot, const char *path) { m->terminal_token_id = -1; m->cls_id = -1; m->sep_id = -1; - /* Pick the tokenizer by the files present: a WordPiece vocab.txt marks a - * BERT encoder; otherwise the Qwen byte-level BPE vocab.json. The two never - * coexist in a model directory, so the probe is unambiguous. */ + /* Pick the tokenizer by the files present: WordPiece vocab.txt, then a + * SentencePiece .model, otherwise the Qwen byte-level BPE vocab.json. */ char wp_path[1024]; snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path); if (access(wp_path, R_OK) == 0) { @@ -323,6 +352,33 @@ int load_one_model(http_server *s, model_slot slot, const char *path) { return -1; } } else { + char sp_path[1024]; + snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", path); + if (access(sp_path, R_OK) != 0) { + snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", path); + if (access(sp_path, R_OK) != 0) + snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", path); + } + if (access(sp_path, R_OK) == 0) { + m->sp_tok = sentencepiece_tokenizer_load(path); + if (!m->sp_tok) { + server_log("embed-server: failed to load SentencePiece tokenizer: %s", sp_path); + return -1; + } + m->sp_tok_ws = sentencepiece_workspace_new(); + if (!m->sp_tok_ws) { + server_log("embed-server: failed to allocate SentencePiece workspace: %s", path); + return -1; + } + m->cls_id = sentencepiece_tokenizer_token_id(m->sp_tok, "<s>"); + m->sep_id = sentencepiece_tokenizer_token_id(m->sp_tok, "</s>"); + if (m->cls_id < 0 || m->sep_id < 0) { + server_log("embed-server: SentencePiece vocab missing <s>/</s>: %s", path); + return -1; + } + } + } + if (!m->wp_tok && !m->sp_tok) { char vocab_path[1024]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path); m->tok = embed_tokenizer_load(vocab_path); @@ -433,6 +489,10 @@ void free_models(http_server *s) { wordpiece_workspace_free(m->wp_tok_ws); if (m->wp_tok) wordpiece_tokenizer_free(m->wp_tok); + if (m->sp_tok_ws) + sentencepiece_workspace_free(m->sp_tok_ws); + if (m->sp_tok) + sentencepiece_tokenizer_free(m->sp_tok); if (m->cpu_ws) embed_workspace_free(m->cpu_ws); if (m->cpu_model) diff --git a/src/tokenizer_sentencepiece.c b/src/tokenizer_sentencepiece.c new file mode 100644 index 0000000..657e8c9 --- /dev/null +++ b/src/tokenizer_sentencepiece.c @@ -0,0 +1,963 @@ +/* + * tokenizer_sentencepiece.c - SentencePiece Unigram tokenizer. + * + * The implementation reads the small subset of ModelProto needed at runtime: + * pieces, scores, piece types, trainer model_type, and normalizer options. The + * normalizer supports SentencePiece's precompiled charsmap format used by + * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program + * over UTF-8 byte offsets. + */ + +#include "tokenizer_sentencepiece.h" + +#include <errno.h> +#include <float.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +enum { + SP_MODEL_UNIGRAM = 1, + SP_PIECE_NORMAL = 1, + SP_PIECE_UNKNOWN = 2, + SP_PIECE_CONTROL = 3, + SP_PIECE_USER_DEFINED = 4, + SP_PIECE_UNUSED = 5, + SP_PIECE_BYTE = 6, +}; + +typedef struct { + const char *key; + uint16_t len; + int val; +} sp_entry_t; + +typedef struct { + int id; + int prev; + double score; +} sp_best_t; + +struct sentencepiece_workspace { + char *norm; + size_t norm_len; + size_t norm_cap; + sp_best_t *best; + int best_cap; + int *ids; + int ids_cap; +}; + +static uint64_t fnv1a_bytes(const char *s, size_t n) { + uint64_t h = 1469598103934665603ULL; + for (size_t i = 0; i < n; i++) { + h ^= (unsigned char)s[i]; + h *= 1099511628211ULL; + } + return h; +} + +static int next_pow2(int x) { + int p = 1; + while (p < x && p <= (1 << 29)) + p <<= 1; + return p; +} + +static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) { + if (len <= 0 || len > UINT16_MAX) + return -1; + uint64_t mask = (uint64_t)cap - 1; + int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); + while (map[pos].key) { + if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) { + map[pos].val = val; + return 0; + } + pos = (int)((pos + 1) & (int)mask); + } + map[pos].key = key; + map[pos].len = (uint16_t)len; + map[pos].val = val; + return 0; +} + +static int map_get(const sp_entry_t *map, int cap, const char *key, int len) { + if (!map || len <= 0) + return -1; + uint64_t mask = (uint64_t)cap - 1; + int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask); + while (map[pos].key) { + if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) + return map[pos].val; + pos = (int)((pos + 1) & (int)mask); + } + return -1; +} + +static int is_tokenizable_type(int type) { + return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED || + type == SP_PIECE_BYTE; +} + +static int reserve_chars(char **buf, size_t *cap, size_t need) { + if (need <= *cap) + return 0; + size_t nc = *cap ? *cap : 128; + while (nc < need) { + if (nc > SIZE_MAX / 2) + return -1; + nc *= 2; + } + char *p = (char *)realloc(*buf, nc); + if (!p) + return -1; + *buf = p; + *cap = nc; + return 0; +} + +static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) { + if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0) + return -1; + memcpy(*buf + *len, s, n); + *len += n; + (*buf)[*len] = '\0'; + return 0; +} + +static int reserve_best(sentencepiece_workspace_t *ws, int need) { + if (need <= ws->best_cap) + return 0; + int nc = ws->best_cap ? ws->best_cap : 128; + while (nc < need) { + if (nc > INT_MAX / 2) + return -1; + nc *= 2; + } + sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p)); + if (!p) + return -1; + ws->best = p; + ws->best_cap = nc; + return 0; +} + +static int reserve_ids(sentencepiece_workspace_t *ws, int need) { + if (need <= ws->ids_cap) + return 0; + int nc = ws->ids_cap ? ws->ids_cap : 64; + while (nc < need) { + if (nc > INT_MAX / 2) + return -1; + nc *= 2; + } + int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p)); + if (!p) + return -1; + ws->ids = p; + ws->ids_cap = nc; + return 0; +} + +static int utf8_len(unsigned char c) { + if (c < 0x80) + return 1; + if ((c & 0xE0) == 0xC0) + return 2; + if ((c & 0xF0) == 0xE0) + return 3; + if ((c & 0xF8) == 0xF0) + return 4; + return 1; +} + +static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) { + if (off >= n) + return 0; + const unsigned char *p = (const unsigned char *)s + off; + int len = utf8_len(p[0]); + if (off + (size_t)len > n) { + *len_out = 1; + return 0; + } + if (len > 1) { + for (int i = 1; i < len; i++) { + if ((p[i] & 0xC0) != 0x80) { + *len_out = 1; + return 0; + } + } + } + *len_out = (size_t)len; + return 1; +} + +static uint32_t read_le32(const unsigned char *p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) { + uint64_t v = 0; + int shift = 0; + while (*p < end && shift <= 63) { + unsigned char c = *(*p)++; + v |= (uint64_t)(c & 0x7f) << shift; + if (!(c & 0x80)) { + *out = v; + return 0; + } + shift += 7; + } + return -1; +} + +static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) { + uint64_t key = 0; + if (pb_read_varint(p, end, &key) != 0) + return -1; + *field = (int)(key >> 3); + *wire = (int)(key & 7); + return *field > 0 ? 0 : -1; +} + +static int pb_read_len(const unsigned char **p, + const unsigned char *end, + const unsigned char **data, + size_t *len) { + uint64_t n = 0; + if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p)) + return -1; + *data = *p; + *len = (size_t)n; + *p += n; + return 0; +} + +static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) { + uint64_t v = 0; + const unsigned char *data = NULL; + size_t len = 0; + switch (wire) { + case 0: + return pb_read_varint(p, end, &v); + case 1: + if ((size_t)(end - *p) < 8) + return -1; + *p += 8; + return 0; + case 2: + return pb_read_len(p, end, &data, &len); + case 5: + if ((size_t)(end - *p) < 4) + return -1; + *p += 4; + return 0; + default: + return -1; + } +} + +static char *memdup0(const unsigned char *data, size_t len) { + char *s = (char *)malloc(len + 1); + if (!s) + return NULL; + memcpy(s, data, len); + s[len] = '\0'; + return s; +} + +static int +parse_piece(sentencepiece_tokenizer_t *tok, int idx, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + char *piece = NULL; + float score = 0.0f; + int type = SP_PIECE_NORMAL; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + goto fail; + if (field == 1 && wire == 2) { + const unsigned char *s = NULL; + size_t n = 0; + if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX) + goto fail; + free(piece); + piece = memdup0(s, n); + if (!piece) + goto fail; + tok->piece_lens[idx] = (uint16_t)n; + } else if (field == 2 && wire == 5) { + if ((size_t)(end - p) < 4) + goto fail; + uint32_t bits = read_le32(p); + memcpy(&score, &bits, sizeof(score)); + p += 4; + } else if (field == 3 && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX) + goto fail; + type = (int)v; + } else if (pb_skip(&p, end, wire) != 0) { + goto fail; + } + } + if (!piece) + goto fail; + tok->id_to_piece[idx] = piece; + tok->scores[idx] = score; + tok->types[idx] = (uint8_t)type; + if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len) + tok->max_piece_len = tok->piece_lens[idx]; + return 0; +fail: + free(piece); + return -1; +} + +static int parse_trainer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + int model_type = SP_MODEL_UNIGRAM; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (field == 3 && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX) + return -1; + model_type = (int)v; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + if (model_type != SP_MODEL_UNIGRAM) { + fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n", + model_type); + return -1; + } + (void)tok; + return 0; +} + +static int parse_normalizer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if ((field == 3 || field == 4 || field == 5) && wire == 0) { + uint64_t v = 0; + if (pb_read_varint(&p, end, &v) != 0) + return -1; + if (field == 3) + tok->add_dummy_prefix = v != 0; + else if (field == 4) + tok->remove_extra_whitespaces = v != 0; + else + tok->escape_whitespaces = v != 0; + } else if (field == 2 && wire == 2) { + const unsigned char *blob = NULL; + size_t n = 0; + if (pb_read_len(&p, end, &blob, &n) != 0) + return -1; + if (n >= 4) { + uint32_t xcda_bytes = read_le32(blob); + if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) { + tok->xcda_count = xcda_bytes / 4; + tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda)); + if (!tok->xcda) + return -1; + for (size_t i = 0; i < tok->xcda_count; i++) + tok->xcda[i] = read_le32(blob + 4 + i * 4); + tok->prefix_replacements_len = n - 4 - xcda_bytes; + tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1); + if (!tok->prefix_replacements) + return -1; + memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes, + tok->prefix_replacements_len); + tok->prefix_replacements[tok->prefix_replacements_len] = '\0'; + } + } + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + return 0; +} + +static int count_pieces(const unsigned char *data, size_t len) { + const unsigned char *p = data, *end = data + len; + int n = 0; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (field == 1 && wire == 2) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0) + return -1; + n++; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + return n; +} + +static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) { + int n_pieces = count_pieces(data, len); + if (n_pieces <= 0) + return -1; + tok->piece_count = n_pieces; + tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece)); + tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens)); + tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores)); + tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types)); + if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types) + return -1; + + const unsigned char *p = data, *end = data + len; + int piece_idx = 0; + int saw_trainer = 0; + while (p < end) { + int field = 0, wire = 0; + if (pb_read_key(&p, end, &field, &wire) != 0) + return -1; + if (wire == 2 && field == 1) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces || + parse_piece(tok, piece_idx++, msg, msg_len) != 0) + return -1; + } else if (wire == 2 && field == 2) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0) + return -1; + saw_trainer = 1; + } else if (wire == 2 && field == 3) { + const unsigned char *msg = NULL; + size_t msg_len = 0; + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || + parse_normalizer(tok, msg, msg_len) != 0) + return -1; + } else if (pb_skip(&p, end, wire) != 0) { + return -1; + } + } + if (piece_idx != n_pieces || !saw_trainer) + return -1; + return 0; +} + +static int raw_piece_id(const sentencepiece_tokenizer_t *tok, const char *piece) { + if (!tok || !piece) + return -1; + return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, + (int)strlen(piece)); +} + +static int raw_to_model_id(const sentencepiece_tokenizer_t *tok, int raw_id) { + if (!tok || raw_id < 0 || raw_id >= tok->piece_count) + return -1; + if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (raw_id == 1) + return 0; /* <s> */ + if (raw_id == 2) + return 2; /* </s> */ + if (raw_id == 0) + return 3; /* <unk> */ + return raw_id + 1; + } + return raw_id; +} + +static int model_to_raw_id(const sentencepiece_tokenizer_t *tok, int id) { + if (!tok || id < 0) + return -1; + if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (id == 0) + return 1; + if (id == 2) + return 2; + if (id == 3) + return 0; + if (id >= 4 && id <= tok->piece_count) + return id - 1; + return -1; + } + return id < tok->piece_count ? id : -1; +} + +static int file_contains_xlm_roberta(const char *model_dir, const char *name) { + char path[1024]; + int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name); + if (n < 0 || (size_t)n >= sizeof(path)) + return 0; + FILE *f = fopen(path, "rb"); + if (!f) + return 0; + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return 0; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return 0; + } + char *buf = (char *)malloc((size_t)sz + 1); + if (!buf) { + fclose(f); + return 0; + } + size_t got = fread(buf, 1, (size_t)sz, f); + fclose(f); + buf[got] = '\0'; + int yes = strstr(buf, "XLMRobertaTokenizer") != NULL || + strstr(buf, "XLMRobertaTokenizerFast") != NULL || + strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL; + free(buf); + return yes; +} + +static int detect_xlm_roberta(const char *model_dir) { + return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") || + file_contains_xlm_roberta(model_dir, "config.json"); +} + +static int find_model_file(const char *model_dir, char path[1024]) { + static const char *names[] = { + "sentencepiece.bpe.model", + "tokenizer.model", + "spiece.model", + }; + for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) { + int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]); + if (n < 0 || n >= 1024) + return -1; + FILE *f = fopen(path, "rb"); + if (f) { + fclose(f); + return 0; + } + if (errno != ENOENT) + return -1; + } + return -1; +} + +static int load_file(const char *path, unsigned char **out, size_t *out_len) { + FILE *f = fopen(path, "rb"); + if (!f) + return -1; + if (fseek(f, 0, SEEK_END) != 0) { + fclose(f); + return -1; + } + long sz = ftell(f); + if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return -1; + } + unsigned char *buf = (unsigned char *)malloc((size_t)sz); + if (!buf) { + fclose(f); + return -1; + } + if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) { + fclose(f); + free(buf); + return -1; + } + fclose(f); + *out = buf; + *out_len = (size_t)sz; + return 0; +} + +static uint32_t xcda_base(const sentencepiece_tokenizer_t *tok, size_t idx) { + uint32_t v = tok->xcda[idx]; + return (v >> 10) << ((v & (1u << 9)) >> 6); +} + +static uint32_t xcda_lcheck(const sentencepiece_tokenizer_t *tok, size_t idx) { + return tok->xcda[idx] & ((1u << 31) | 0xffu); +} + +static int xcda_leaf(const sentencepiece_tokenizer_t *tok, size_t idx) { + return (tok->xcda[idx] >> 8) & 1u; +} + +static uint32_t xcda_value(const sentencepiece_tokenizer_t *tok, size_t idx) { + return tok->xcda[idx] & ((1u << 31) - 1u); +} + +typedef struct { + const char *data; + size_t len; + size_t consumed; +} norm_prefix_t; + +static norm_prefix_t normalize_prefix(const sentencepiece_tokenizer_t *tok, + const char *input, + size_t input_len, + size_t off) { + static const char repl[] = "\xef\xbf\xbd"; + if (off >= input_len) + return (norm_prefix_t){input + off, 0, 0}; + + size_t best_len = 0; + size_t best_repl = 0; + if (tok->xcda && tok->xcda_count > 0) { + uint32_t node = xcda_base(tok, 0); + for (size_t p = off; p < input_len; p++) { + unsigned char c = (unsigned char)input[p]; + if (c == 0) + break; + node ^= c; + if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c) + break; + int leaf = xcda_leaf(tok, node); + node ^= xcda_base(tok, node); + if (leaf) { + best_len = p - off + 1; + best_repl = xcda_value(tok, node); + } + } + } + if (best_len > 0 && best_repl < tok->prefix_replacements_len) { + const char *r = tok->prefix_replacements + best_repl; + size_t max = tok->prefix_replacements_len - best_repl; + size_t n = strnlen(r, max); + return (norm_prefix_t){r, n, best_len}; + } + + size_t clen = 1; + if (valid_utf8_one(input, input_len, off, &clen)) + return (norm_prefix_t){input + off, clen, clen}; + return (norm_prefix_t){repl, 3, 1}; +} + +static int normalize_into(const sentencepiece_tokenizer_t *tok, + sentencepiece_workspace_t *ws, + const char *text) { + ws->norm_len = 0; + if (!text) + return -1; + size_t input_len = strlen(text); + const char space[] = "\xe2\x96\x81"; /* U+2581 */ + const char *space_s = tok->escape_whitespaces ? space : " "; + size_t space_n = tok->escape_whitespaces ? 3 : 1; + int processing_non_ws = 0; + int space_prepended = 0; + + for (size_t off = 0; off < input_len;) { + norm_prefix_t r = normalize_prefix(tok, text, input_len, off); + for (size_t i = 0; i < r.len; i++) { + char c = r.data[i]; + if (c != ' ') { + if (!processing_non_ws) { + processing_non_ws = 1; + if ((tok->add_dummy_prefix && !space_prepended) || + tok->remove_extra_whitespaces) { + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, + space_n) != 0) + return -1; + space_prepended = 1; + } + } + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0) + return -1; + } else { + processing_non_ws = 0; + if (!tok->remove_extra_whitespaces) { + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != + 0) + return -1; + } + } + } + off += r.consumed ? r.consumed : 1; + } + if (ws->norm) + ws->norm[ws->norm_len] = '\0'; + return 0; +} + +static double compute_unknown_score(const sentencepiece_tokenizer_t *tok) { + float min_score = FLT_MAX; + for (int i = 0; i < tok->piece_count; i++) { + if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score) + min_score = tok->scores[i]; + } + if (min_score == FLT_MAX) + min_score = -10.0f; + return (double)min_score - 10.0; +} + +static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace_t *ws) { + size_t n = ws->norm_len; + if (n == 0) + return 0; + if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 || + reserve_ids(ws, (int)n + 1) != 0) + return -1; + for (size_t i = 0; i <= n; i++) + ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX}; + ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0}; + double unk_score = tok->unknown_score; + + for (size_t off = 0; off < n;) { + size_t cp_len = 1; + valid_utf8_one(ws->norm, n, off, &cp_len); + sp_best_t cur = ws->best[off]; + if (cur.score <= -DBL_MAX / 2.0) { + off += cp_len; + continue; + } + + int single_cp_found = 0; + for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) { + int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, + ws->norm + off, (int)(end - off)); + if (raw >= 0 && is_tokenizable_type(tok->types[raw])) { + if (end - off == cp_len) + single_cp_found = 1; + double sc = + cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]); + if (sc > ws->best[end].score) + ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc}; + } + if (end == n) + break; + size_t next_len = 1; + valid_utf8_one(ws->norm, n, end, &next_len); + end += next_len; + } + if (!single_cp_found) { + size_t end = off + cp_len; + double sc = cur.score + unk_score; + if (sc > ws->best[end].score) + ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc}; + } + off += cp_len; + } + + int out_n = 0; + int prev_unk = 0; + for (size_t off = n; off > 0;) { + sp_best_t b = ws->best[off]; + if (b.prev < 0 || b.prev >= (int)off) + return -1; + int is_unk = b.id == tok->unk_id; + if (!(prev_unk && is_unk)) { + if (out_n >= ws->ids_cap) + return -1; + ws->ids[out_n++] = b.id; + } + prev_unk = is_unk; + off = (size_t)b.prev; + } + for (int i = 0; i < out_n / 2; i++) { + int t = ws->ids[i]; + ws->ids[i] = ws->ids[out_n - 1 - i]; + ws->ids[out_n - 1 - i] = t; + } + return out_n; +} + +sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) { + if (!model_dir) + return NULL; + char path[1024]; + if (find_model_file(model_dir, path) != 0) { + fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir); + return NULL; + } + unsigned char *data = NULL; + size_t len = 0; + if (load_file(path, &data, &len) != 0) { + fprintf(stderr, "sentencepiece: cannot read %s\n", path); + return NULL; + } + + sentencepiece_tokenizer_t *tok = + (sentencepiece_tokenizer_t *)calloc(1, sizeof(sentencepiece_tokenizer_t)); + if (!tok) { + free(data); + return NULL; + } + tok->add_dummy_prefix = 1; + tok->remove_extra_whitespaces = 1; + tok->escape_whitespaces = 1; + tok->id_map = + detect_xlm_roberta(model_dir) ? SENTENCEPIECE_XLM_ROBERTA : SENTENCEPIECE_IDENTITY; + if (parse_model(tok, data, len) != 0) { + fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path); + free(data); + sentencepiece_tokenizer_free(tok); + return NULL; + } + free(data); + + int cap = next_pow2(tok->piece_count * 2 + 1); + tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t)); + if (!tok->piece_map) { + sentencepiece_tokenizer_free(tok); + return NULL; + } + tok->piece_map_cap = cap; + for (int i = 0; i < tok->piece_count; i++) { + if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i], + tok->piece_lens[i], i) != 0) { + sentencepiece_tokenizer_free(tok); + return NULL; + } + } + + tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>")); + tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>")); + tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>")); + tok->pad_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA + ? 1 + : raw_to_model_id(tok, raw_piece_id(tok, "<pad>")); + tok->mask_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA + ? tok->piece_count + 1 + : raw_to_model_id(tok, raw_piece_id(tok, "<mask>")); + tok->vocab_size = + tok->id_map == SENTENCEPIECE_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; + tok->unknown_score = compute_unknown_score(tok); + if (tok->unk_id < 0 || tok->max_piece_len <= 0) { + fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path); + sentencepiece_tokenizer_free(tok); + return NULL; + } + return tok; +} + +const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id) { + if (!tok || id < 0) + return NULL; + if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (id == 1) + return "<pad>"; + if (id == tok->piece_count + 1) + return "<mask>"; + } + int raw = model_to_raw_id(tok, id); + return raw >= 0 ? tok->id_to_piece[raw] : NULL; +} + +int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, const char *token) { + if (!tok || !token) + return -1; + if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) { + if (!strcmp(token, "<s>")) + return 0; + if (!strcmp(token, "<pad>")) + return 1; + if (!strcmp(token, "</s>")) + return 2; + if (!strcmp(token, "<unk>")) + return 3; + if (!strcmp(token, "<mask>")) + return tok->piece_count + 1; + } + return raw_to_model_id(tok, raw_piece_id(tok, token)); +} + +int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok, + sentencepiece_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens) { + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text) + return -1; + if (normalize_into(tok, ws, text) != 0) + return -1; + int n = segment(tok, ws); + if (n < 0) + return -1; + if (out_n_tokens) + *out_n_tokens = n; + if (n == 0) + return 0; + if (!out_ids || out_cap < n) + return -2; + memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids)); + return 0; +} + +int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok, + sentencepiece_workspace_t *ws, + const char *text, + int *out_n_tokens) { + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text) + return NULL; + int n = 0; + int rc = sentencepiece_tokenizer_encode_into(tok, ws, text, NULL, 0, &n); + if (rc < 0 && rc != -2) + return NULL; + if (out_n_tokens) + *out_n_tokens = n; + if (n == 0) + return NULL; + int *ids = (int *)malloc((size_t)n * sizeof(*ids)); + if (!ids) + return NULL; + memcpy(ids, ws->ids, (size_t)n * sizeof(*ids)); + return ids; +} + +int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok, + const char *text, + int *out_n_tokens) { + sentencepiece_workspace_t *ws = sentencepiece_workspace_new(); + if (!ws) { + if (out_n_tokens) + *out_n_tokens = 0; + return NULL; + } + int *ids = sentencepiece_tokenizer_encode_with_workspace(tok, ws, text, out_n_tokens); + sentencepiece_workspace_free(ws); + return ids; +} + +sentencepiece_workspace_t *sentencepiece_workspace_new(void) { + return (sentencepiece_workspace_t *)calloc(1, sizeof(sentencepiece_workspace_t)); +} + +void sentencepiece_workspace_free(sentencepiece_workspace_t *ws) { + if (!ws) + return; + free(ws->norm); + free(ws->best); + free(ws->ids); + free(ws); +} + +void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok) { + if (!tok) + return; + if (tok->id_to_piece) { + for (int i = 0; i < tok->piece_count; i++) + free(tok->id_to_piece[i]); + } + free(tok->id_to_piece); + free(tok->piece_lens); + free(tok->scores); + free(tok->types); + free(tok->piece_map); + free(tok->xcda); + free(tok->prefix_replacements); + free(tok); +} diff --git a/src/tokenizer_sentencepiece.h b/src/tokenizer_sentencepiece.h new file mode 100644 index 0000000..b4ce48c --- /dev/null +++ b/src/tokenizer_sentencepiece.h @@ -0,0 +1,99 @@ +/* + * tokenizer_sentencepiece.h - SentencePiece tokenizer for XLM-R style models. + * + * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`, + * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R + * and multilingual E5. It encodes text only; frontends add the model's special + * token layout, e.g. <s> ... </s> for XLM-R. + */ + +#ifndef SENTENCEPIECE_TOKENIZER_H +#define SENTENCEPIECE_TOKENIZER_H + +#include <stddef.h> +#include <stdint.h> + +#ifndef EMBED_API +#if defined(__GNUC__) +#define EMBED_API __attribute__((visibility("default"))) +#else +#define EMBED_API +#endif +#endif + +typedef enum { + SENTENCEPIECE_IDENTITY = 0, + SENTENCEPIECE_XLM_ROBERTA = 1, +} sentencepiece_id_map_t; + +typedef struct { + char **id_to_piece; /* [piece_count] raw SentencePiece ids */ + uint16_t *piece_lens; /* byte length for id_to_piece entries */ + float *scores; /* SentencePiece score per raw id */ + uint8_t *types; /* SentencePiece piece type per raw id */ + int piece_count; /* raw SentencePiece vocabulary size */ + int vocab_size; /* model embedding vocab size after id mapping */ + int unk_id; /* model-space unknown-token id */ + int bos_id; /* model-space beginning-token id, or -1 */ + int eos_id; /* model-space end-token id, or -1 */ + int pad_id; /* model-space pad-token id, or -1 */ + int mask_id; /* model-space mask-token id, or -1 */ + int max_piece_len; /* max normal/user/unused piece byte length */ + double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ + int add_dummy_prefix; + int remove_extra_whitespaces; + int escape_whitespaces; + sentencepiece_id_map_t id_map; + + void *piece_map; /* piece bytes -> raw SentencePiece id */ + int piece_map_cap; + + uint32_t *xcda; + size_t xcda_count; + char *prefix_replacements; + size_t prefix_replacements_len; +} sentencepiece_tokenizer_t; + +typedef struct sentencepiece_workspace sentencepiece_workspace_t; + +/* Load the first supported SentencePiece model found in model_dir. Returns NULL + * on missing model file, unsupported model type, malformed protobuf, or OOM. */ +EMBED_API sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir); + +/* Decode one model-space id to an internal token string, or NULL when invalid. */ +EMBED_API const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id); + +/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a + * model-space id, or -1 when absent. */ +EMBED_API int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, + const char *token); + +/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns + * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an + * error. */ +EMBED_API int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok, + const char *text, + int *out_n_tokens); + +EMBED_API sentencepiece_workspace_t *sentencepiece_workspace_new(void); +EMBED_API void sentencepiece_workspace_free(sentencepiece_workspace_t *ws); + +/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if + * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */ +EMBED_API int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok, + sentencepiece_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); + +/* Same tokenization as sentencepiece_tokenizer_encode(), reusing ws internally + * and performing only the final returned int[] allocation. */ +EMBED_API int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok, + sentencepiece_workspace_t *ws, + const char *text, + int *out_n_tokens); + +EMBED_API void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok); + +#endif /* SENTENCEPIECE_TOKENIZER_H */ diff --git a/tests/test_sentencepiece.c b/tests/test_sentencepiece.c new file mode 100644 index 0000000..b597af2 --- /dev/null +++ b/tests/test_sentencepiece.c @@ -0,0 +1,266 @@ +/* tests/test_sentencepiece.c - hermetic SentencePiece Unigram checks. */ + +#include "tokenizer_sentencepiece.h" + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +typedef struct { + unsigned char *p; + size_t n; + size_t cap; +} buf_t; + +static int reserve(buf_t *b, size_t need) { + if (need <= b->cap) + return 0; + size_t nc = b->cap ? b->cap : 128; + while (nc < need) + nc *= 2; + unsigned char *p = (unsigned char *)realloc(b->p, nc); + if (!p) + return -1; + b->p = p; + b->cap = nc; + return 0; +} + +static int put_byte(buf_t *b, int c) { + if (reserve(b, b->n + 1) != 0) + return -1; + b->p[b->n++] = (unsigned char)c; + return 0; +} + +static int put_varint(buf_t *b, uint64_t v) { + while (v >= 0x80) { + if (put_byte(b, (int)(v | 0x80)) != 0) + return -1; + v >>= 7; + } + return put_byte(b, (int)v); +} + +static int put_key(buf_t *b, int field, int wire) { + return put_varint(b, ((uint64_t)field << 3) | (uint64_t)wire); +} + +static int put_bytes(buf_t *b, const void *data, size_t n) { + if (reserve(b, b->n + n) != 0) + return -1; + memcpy(b->p + b->n, data, n); + b->n += n; + return 0; +} + +static int put_fixed32(buf_t *b, float f) { + uint32_t v; + memcpy(&v, &f, sizeof(v)); + unsigned char p[4] = { + (unsigned char)(v & 0xff), + (unsigned char)((v >> 8) & 0xff), + (unsigned char)((v >> 16) & 0xff), + (unsigned char)((v >> 24) & 0xff), + }; + return put_bytes(b, p, sizeof(p)); +} + +static int put_len_msg(buf_t *dst, int field, const buf_t *msg) { + return put_key(dst, field, 2) || put_varint(dst, msg->n) || put_bytes(dst, msg->p, msg->n); +} + +static int add_piece(buf_t *model, const char *piece, float score, int type) { + buf_t msg = {0}; + int rc = put_key(&msg, 1, 2) || put_varint(&msg, strlen(piece)) || + put_bytes(&msg, piece, strlen(piece)) || put_key(&msg, 2, 5) || + put_fixed32(&msg, score) || put_key(&msg, 3, 0) || put_varint(&msg, (uint64_t)type) || + put_len_msg(model, 1, &msg); + free(msg.p); + return rc; +} + +static int write_model(const char *dir, int xlm_roberta) { + char path[1024]; + snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", dir); + FILE *f = fopen(path, "wb"); + if (!f) + return -1; + + buf_t model = {0}; + int rc = 0; + rc |= add_piece(&model, "<unk>", 0.0f, 2); + rc |= add_piece(&model, "<s>", 0.0f, 3); + rc |= add_piece(&model, "</s>", 0.0f, 3); + rc |= add_piece(&model, ",", -3.0f, 1); + rc |= add_piece(&model, ".", -3.0f, 1); + rc |= add_piece(&model, "\xe2\x96\x81", -4.0f, 1); + rc |= add_piece(&model, "!", -3.0f, 1); + rc |= add_piece(&model, "\xe2\x96\x81Hello", -1.0f, 1); + rc |= add_piece(&model, "\xe2\x96\x81world", -1.0f, 1); + rc |= add_piece(&model, "\xe5\x8d\x97", -1.0f, 1); + rc |= add_piece(&model, "\xe7\x93\x9c", -1.0f, 1); + rc |= add_piece(&model, "\xe2\x96\x81Hi", -1.0f, 1); + + buf_t trainer = {0}; + rc |= put_key(&trainer, 3, 0) || put_varint(&trainer, 1) || put_len_msg(&model, 2, &trainer); + free(trainer.p); + + buf_t normalizer = {0}; + rc |= put_key(&normalizer, 3, 0) || put_varint(&normalizer, 1); + rc |= put_key(&normalizer, 4, 0) || put_varint(&normalizer, 1); + rc |= put_key(&normalizer, 5, 0) || put_varint(&normalizer, 1); + rc |= put_len_msg(&model, 3, &normalizer); + free(normalizer.p); + + if (rc == 0 && fwrite(model.p, 1, model.n, f) != model.n) + rc = -1; + fclose(f); + free(model.p); + if (rc != 0) + return -1; + + if (xlm_roberta) { + snprintf(path, sizeof(path), "%s/tokenizer_config.json", dir); + f = fopen(path, "w"); + if (!f) + return -1; + fputs("{\"tokenizer_class\":\"XLMRobertaTokenizer\"}", f); + fclose(f); + } + return 0; +} + +static int check_ids(const sentencepiece_tokenizer_t *tok, + sentencepiece_workspace_t *ws, + const char *text, + const int *want, + int want_n, + const char *name) { + int n = 0; + int *ids = sentencepiece_tokenizer_encode_with_workspace(tok, ws, text, &n); + int ok = n == want_n; + for (int i = 0; ok && i < n; i++) + ok = ids[i] == want[i]; + if (!ok) { + fprintf(stderr, "%s: got [", name); + for (int i = 0; i < n; i++) + fprintf(stderr, "%d%s", ids[i], i + 1 < n ? "," : ""); + fprintf(stderr, "], want ["); + for (int i = 0; i < want_n; i++) + fprintf(stderr, "%d%s", want[i], i + 1 < want_n ? "," : ""); + fprintf(stderr, "]\n"); + } + free(ids); + return ok ? 0 : -1; +} + +static int run_hermetic(void) { + const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"; + char dir[1024]; + snprintf(dir, sizeof(dir), "%s/embed-sp-test-XXXXXX", tmp); + if (!mkdtemp(dir) || write_model(dir, 1) != 0) { + fprintf(stderr, "fixture creation failed\n"); + return 2; + } + + sentencepiece_tokenizer_t *tok = sentencepiece_tokenizer_load(dir); + sentencepiece_workspace_t *ws = sentencepiece_workspace_new(); + if (!tok || !ws) { + fprintf(stderr, "load failed\n"); + return 1; + } + if (tok->piece_count != 12 || tok->vocab_size != 14 || tok->unk_id != 3 || + tok->bos_id != 0 || tok->eos_id != 2 || tok->pad_id != 1 || tok->mask_id != 13) { + fprintf(stderr, "metadata wrong\n"); + return 1; + } + if (sentencepiece_tokenizer_token_id(tok, "<s>") != 0 || + sentencepiece_tokenizer_token_id(tok, "</s>") != 2 || + sentencepiece_tokenizer_token_id(tok, "<unk>") != 3 || + sentencepiece_tokenizer_token_id(tok, ",") != 4 || + sentencepiece_tokenizer_token_id(tok, "<mask>") != 13 || + strcmp(sentencepiece_tokenizer_decode(tok, 8), "\xe2\x96\x81Hello") != 0 || + strcmp(sentencepiece_tokenizer_decode(tok, 1), "<pad>") != 0) { + fprintf(stderr, "special id/decode mapping wrong\n"); + return 1; + } + + int rc = 0; + rc |= check_ids(tok, ws, "Hello world!", (int[]){8, 9, 7}, 3, "hello"); + rc |= check_ids(tok, ws, " ", NULL, 0, "spaces"); + rc |= check_ids(tok, ws, "\xe5\x8d\x97\xe7\x93\x9c", (int[]){6, 10, 11}, 3, "cjk"); + rc |= check_ids(tok, ws, "missing", (int[]){6, 3}, 2, "unknown-collapse"); + + int tiny[1], need = 0; + if (sentencepiece_tokenizer_encode_into(tok, ws, "Hello world!", tiny, 1, &need) != -2 || + need != 3) { + fprintf(stderr, "small-buffer handling wrong\n"); + return 1; + } + sentencepiece_workspace_free(ws); + sentencepiece_tokenizer_free(tok); + + char direct[1024]; + snprintf(direct, sizeof(direct), "%s/embed-sp-direct-XXXXXX", tmp); + if (!mkdtemp(direct) || write_model(direct, 0) != 0) + return 2; + tok = sentencepiece_tokenizer_load(direct); + ws = sentencepiece_workspace_new(); + if (!tok || !ws) + return 1; + rc |= check_ids(tok, ws, "Hello world!", (int[]){7, 8, 6}, 3, "direct-map"); + sentencepiece_workspace_free(ws); + sentencepiece_tokenizer_free(tok); + if (rc) + return 1; + + puts("ok: SentencePiece Unigram normalization, Viterbi, XLM-R id mapping"); + return 0; +} + +static int run_live(const char *model_dir) { + sentencepiece_tokenizer_t *tok = sentencepiece_tokenizer_load(model_dir); + sentencepiece_workspace_t *ws = sentencepiece_workspace_new(); + if (!tok || !ws) { + fprintf(stderr, "live load failed\n"); + return 1; + } + int rc = 0; + rc |= check_ids(tok, ws, "Hello world! \xe5\x8d\x97\xe7\x93\x9c", + (int[]){35378, 8999, 38, 6, 4617, 39613}, 6, "live-mixed"); + rc |= check_ids(tok, ws, "caf\xc3\xa9 d\xc3\xa9j\xc3\xa0 vu", + (int[]){26216, 15154, 13946}, 3, "live-accents"); + rc |= check_ids(tok, ws, "\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3 \xef\xbc\x91\xef\xbc\x92\xef\xbc\x93", + (int[]){47457, 37638}, 2, "live-nfkc"); + rc |= check_ids(tok, ws, "query: what is snowflake?", + (int[]){41, 1294, 12, 2367, 83, 108203, 13034, 350, 32}, 9, + "live-query"); + rc |= check_ids(tok, ws, "\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbc\xd0\xb8\xd1\x80", + (int[]){1813, 18454, 11373}, 3, "live-cyrillic"); + rc |= check_ids(tok, ws, + "\xd9\x85\xd8\xb1\xd8\xad\xd8\xa8\xd8\xa7 \xd8\xa8\xd8\xa7\xd9\x84\xd8\xb9\xd8\xa7\xd9\x84\xd9\x85", + (int[]){665, 193478, 258, 1705, 77796}, 5, "live-arabic"); + rc |= check_ids(tok, ws, "", NULL, 0, "live-empty"); + rc |= check_ids(tok, ws, " ", NULL, 0, "live-spaces"); + if (sentencepiece_tokenizer_token_id(tok, "<s>") != 0 || + sentencepiece_tokenizer_token_id(tok, "<pad>") != 1 || + sentencepiece_tokenizer_token_id(tok, "</s>") != 2 || + sentencepiece_tokenizer_token_id(tok, "<unk>") != 3 || + sentencepiece_tokenizer_token_id(tok, "<mask>") != 250001) { + fprintf(stderr, "live special ids wrong\n"); + rc = 1; + } + sentencepiece_workspace_free(ws); + sentencepiece_tokenizer_free(tok); + return rc ? 1 : 0; +} + +int main(int argc, char **argv) { + if (argc > 1) + return run_live(argv[1]); + return run_hermetic(); +} diff --git a/tests/test_xlm_roberta.c b/tests/test_xlm_roberta.c new file mode 100644 index 0000000..29e05f7 --- /dev/null +++ b/tests/test_xlm_roberta.c @@ -0,0 +1,96 @@ +/* tests/test_xlm_roberta.c - hermetic XLM-R/RoBERTa bridge checks. */ + +#include "embed.h" +#include "tiny_model.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> + +static int write_fixture(const char *dir) { + const int hidden = 4, heads = 2, inter = 8, vocab = 16, max_pos = 8; + char path[2048]; + snprintf(path, sizeof(path), "%s/config.json", dir); + FILE *f = fopen(path, "w"); + if (!f) + return -1; + fprintf(f, + "{\"model_type\":\"xlm-roberta\",\"hidden_size\":%d," + "\"num_hidden_layers\":1,\"num_attention_heads\":%d," + "\"intermediate_size\":%d,\"vocab_size\":%d," + "\"max_position_embeddings\":%d,\"layer_norm_eps\":1e-5," + "\"pad_token_id\":1,\"type_vocab_size\":1}", + hidden, heads, inter, vocab, max_pos); + fclose(f); + + const tm_spec_t specs[] = { + {"embeddings.word_embeddings.weight", vocab, hidden}, + {"embeddings.position_embeddings.weight", max_pos, hidden}, + {"embeddings.token_type_embeddings.weight", 1, hidden}, + {"embeddings.LayerNorm.weight", hidden, 0}, + {"embeddings.LayerNorm.bias", hidden, 0}, + {"encoder.layer.0.attention.self.query.weight", hidden, hidden}, + {"encoder.layer.0.attention.self.query.bias", hidden, 0}, + {"encoder.layer.0.attention.self.key.weight", hidden, hidden}, + {"encoder.layer.0.attention.self.key.bias", hidden, 0}, + {"encoder.layer.0.attention.self.value.weight", hidden, hidden}, + {"encoder.layer.0.attention.self.value.bias", hidden, 0}, + {"encoder.layer.0.attention.output.dense.weight", hidden, hidden}, + {"encoder.layer.0.attention.output.dense.bias", hidden, 0}, + {"encoder.layer.0.attention.output.LayerNorm.weight", hidden, 0}, + {"encoder.layer.0.attention.output.LayerNorm.bias", hidden, 0}, + {"encoder.layer.0.intermediate.dense.weight", inter, hidden}, + {"encoder.layer.0.intermediate.dense.bias", inter, 0}, + {"encoder.layer.0.output.dense.weight", hidden, inter}, + {"encoder.layer.0.output.dense.bias", hidden, 0}, + {"encoder.layer.0.output.LayerNorm.weight", hidden, 0}, + {"encoder.layer.0.output.LayerNorm.bias", hidden, 0}, + }; + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + return tm_write_safetensors_with_prefix(path, "F32", specs, + (int)(sizeof(specs) / sizeof(specs[0])), + "roberta."); +} + +int main(void) { + const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"; + char dir[1024]; + snprintf(dir, sizeof(dir), "%s/embed-xlm-roberta-test-XXXXXX", tmp); + if (!mkdtemp(dir) || write_fixture(dir) != 0) { + fprintf(stderr, "fixture creation failed\n"); + return 2; + } + + embed_model_t *model = embed_model_load(dir); + embed_workspace_t *ws = embed_workspace_new(model); + const embed_config_t *cfg = embed_model_config(model); + if (!model || !ws || !cfg) { + fprintf(stderr, "model setup failed\n"); + return 1; + } + if (cfg->family != EMBED_FAMILY_BERT || cfg->position_id_offset != 2 || + cfg->type_vocab_size != 1 || cfg->pooling_mode != EMBED_POOL_MEAN || + cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL) { + fprintf(stderr, "XLM-R config bridge wrong\n"); + return 1; + } + + float out[4]; + int ids_ok[] = {0, 4, 5, 6, 7, 2}; + if (embed_model_encode_into(model, ws, ids_ok, 6, out) != 0) { + fprintf(stderr, "XLM-R forward failed at max valid position\n"); + return 1; + } + int ids_too_long[] = {0, 4, 5, 6, 7, 8, 2}; + if (embed_model_encode_into(model, ws, ids_too_long, 7, out) == 0) { + fprintf(stderr, "XLM-R position overflow was accepted\n"); + return 1; + } + + embed_workspace_free(ws); + embed_model_free(model); + puts("ok: XLM-R config, roberta prefix, type_vocab_size=1, position offset"); + return 0; +} |
