aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore4
-rw-r--r--Makefile31
-rw-r--r--bench/bench_tokenizer.c140
-rw-r--r--src/cli.c71
-rw-r--r--src/config.c20
-rw-r--r--src/cuda.cu10
-rw-r--r--src/embed.h2
-rw-r--r--src/forward.c15
-rw-r--r--src/internal.h2
-rw-r--r--src/mlx.c20
-rw-r--r--src/model.c10
-rw-r--r--src/server_internal.h9
-rw-r--r--src/server_models.c68
-rw-r--r--src/tokenizer_sentencepiece.c963
-rw-r--r--src/tokenizer_sentencepiece.h99
-rw-r--r--tests/test_sentencepiece.c266
-rw-r--r--tests/test_xlm_roberta.c96
17 files changed, 1775 insertions, 51 deletions
diff --git a/.gitignore b/.gitignore
index d51f367..4d16a0f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,8 @@ tests/test_qwen3
tests/test_cli
tests/test_cls
tests/test_wordpiece
+test_sentencepiece
+test_xlm_roberta
tests/cli_under_test
bench/bench_tokenizer
bench/bench_kernels
@@ -28,3 +30,5 @@ __pycache__/
tmp
private
.vscode
+Launch_*.trace/
+
diff --git a/Makefile b/Makefile
index 5d041ee..a33f3da 100644
--- a/Makefile
+++ b/Makefile
@@ -79,7 +79,7 @@ LDFLAGS = -lm -lpthread $(BACKEND_LDFLAGS)
KERNEL_SRCS = src/kernels_gemm.c src/kernels_norm.c src/kernels_act.c src/kernels_attn.c \
src/kernels_rope.c src/threadpool.c \
src/kernels_generic.c src/kernels_neon.c src/kernels_avx.c
-TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c
+TOKENIZER_SRCS = src/tokenizer_bpe.c src/tokenizer_wordpiece.c src/tokenizer_sentencepiece.c
CORE_SRCS = src/model.c src/forward.c src/late.c src/vec.c src/alloc.c \
src/config.c src/safetensors.c $(KERNEL_SRCS)
@@ -203,11 +203,13 @@ TEST_CC_FLAGS = -Wall -Wextra -O2 -Isrc
TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS)
TEST_TOKENIZER_SRCS = tests/test_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS)
TEST_WORDPIECE_SRCS = tests/test_wordpiece.c src/tokenizer_wordpiece.c
+TEST_SENTENCEPIECE_SRCS = tests/test_sentencepiece.c src/tokenizer_sentencepiece.c
TEST_SAFETENSORS_SRCS = tests/test_safetensors.c src/safetensors.c
TEST_BF16_SRCS = tests/test_bf16_model.c $(CORE_SRCS)
TEST_QWEN3_SRCS = tests/test_qwen3.c $(CORE_SRCS)
TEST_QWEN2_SRCS = tests/test_qwen2.c $(CORE_SRCS)
TEST_CLS_SRCS = tests/test_cls.c $(CORE_SRCS)
+TEST_XLM_ROBERTA_SRCS = tests/test_xlm_roberta.c $(CORE_SRCS)
TEST_WORKSPACE_SRCS = tests/test_workspace.c $(CORE_SRCS) src/tokenizer_bpe.c
TEST_LATE_SRCS = tests/test_late.c $(CORE_SRCS) src/tokenizer_bpe.c
# test_server.c #includes ../src/server.c, so server is not a separate object
@@ -232,6 +234,8 @@ test:
./tests/test_tokenizer
$(CC) $(TEST_CC_FLAGS) -o tests/test_wordpiece $(TEST_WORDPIECE_SRCS)
./tests/test_wordpiece
+ $(CC) $(TEST_CC_FLAGS) -o tests/test_sentencepiece $(TEST_SENTENCEPIECE_SRCS)
+ ./tests/test_sentencepiece
$(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS)
./tests/test_safetensors
$(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \
@@ -246,6 +250,9 @@ test:
$(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \
$(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
./tests/test_cls
+ $(CC) $(TEST_CC_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \
+ $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ ./tests/test_xlm_roberta
$(CC) $(TEST_CC_FLAGS) -o tests/test_workspace \
$(TEST_WORKSPACE_SRCS) -lm -lpthread
./tests/test_workspace
@@ -280,9 +287,9 @@ COV_DIR = coverage
COV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -O0 \
-fcoverage-compilation-dir=.
COV_BINS = tests/test_kernels_generic tests/test_kernels_blas \
- tests/test_tokenizer tests/test_safetensors \
+ tests/test_tokenizer tests/test_sentencepiece tests/test_safetensors \
tests/test_bf16_model tests/test_qwen3 tests/test_qwen2 tests/test_workspace \
- tests/test_late \
+ tests/test_cls tests/test_xlm_roberta tests/test_late \
tests/cli_under_test tests/test_server
# Report on project sources only - not the harnesses, vendored deps, or
# system headers (cJSON lands under /opt or /usr otherwise).
@@ -307,6 +314,9 @@ coverage:
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_tokenizer \
$(TEST_TOKENIZER_SRCS) -lm -lpthread
LLVM_PROFILE_FILE=$(COV_DIR)/tokenizer.profraw ./tests/test_tokenizer
+ $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_sentencepiece \
+ $(TEST_SENTENCEPIECE_SRCS)
+ LLVM_PROFILE_FILE=$(COV_DIR)/sentencepiece.profraw ./tests/test_sentencepiece
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS)
LLVM_PROFILE_FILE=$(COV_DIR)/safetensors.profraw ./tests/test_safetensors
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_bf16_model \
@@ -318,6 +328,12 @@ coverage:
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_qwen2 \
$(TEST_QWEN2_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
LLVM_PROFILE_FILE=$(COV_DIR)/qwen2.profraw ./tests/test_qwen2
+ $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_cls \
+ $(TEST_CLS_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ LLVM_PROFILE_FILE=$(COV_DIR)/cls.profraw ./tests/test_cls
+ $(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) $(TEST_BLAS_CFLAGS) -o tests/test_xlm_roberta \
+ $(TEST_XLM_ROBERTA_SRCS) -lm -lpthread $(TEST_BLAS_LDFLAGS)
+ LLVM_PROFILE_FILE=$(COV_DIR)/xlm_roberta.profraw ./tests/test_xlm_roberta
$(CC) $(TEST_CC_FLAGS) $(COV_FLAGS) -o tests/test_workspace \
$(TEST_WORKSPACE_SRCS) -lm -lpthread
LLVM_PROFILE_FILE=$(COV_DIR)/workspace.profraw ./tests/test_workspace
@@ -349,8 +365,8 @@ coverage:
bench-tokenizer:
$(CC) -Wall -Wextra -O3 -march=native -Isrc -o bench/bench_tokenizer \
- bench/bench_tokenizer.c src/tokenizer_bpe.c $(KERNEL_SRCS) -lm -lpthread
- @echo "run: bench/bench_tokenizer <model_dir> [runs]"
+ bench/bench_tokenizer.c $(TOKENIZER_SRCS) $(KERNEL_SRCS) -lm -lpthread
+ @echo "run: bench/bench_tokenizer <model_dir-or-vocab.json> [runs] TEXT..."
# =============================================================================
# Regression microbenchmarks (see bench/bench.h). Each run records a JSON
@@ -394,7 +410,8 @@ clean:
$(TARGET) $(SERVER_TARGET) $(LIB) libembed.dylib libembed.so \
tests/test_kernels_generic tests/test_kernels_blas \
tests/test_safetensors tests/test_bf16_model tests/test_server \
- tests/test_qwen3 tests/test_qwen2 \
- tests/test_tokenizer tests/test_wordpiece tests/test_workspace tests/test_cli tests/test_late \
+ tests/test_qwen3 tests/test_qwen2 tests/test_cls tests/test_xlm_roberta \
+ tests/test_tokenizer tests/test_wordpiece tests/test_sentencepiece \
+ tests/test_workspace tests/test_cli tests/test_late \
tests/cli_under_test bench/bench_tokenizer bench/bench_kernels bench/bench_model
rm -rf $(COV_DIR)
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index bf4d0b7..8175ffe 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -1,11 +1,14 @@
/* bench/bench_tokenizer.c - tokenizer encode-path benchmark.
- * Build via `make bench-tokenizer`; needs a model dir (vocab.json). */
+ * Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */
#include "tokenizer_bpe.h"
+#include "tokenizer_sentencepiece.h"
+#include "tokenizer_wordpiece.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <unistd.h>
#include <time.h>
int embed_verbose = 0;
@@ -26,6 +29,110 @@ static int same_ids(const int *a, int na, const int *b, int nb) {
return 1;
}
+typedef enum {
+ TOK_BPE,
+ TOK_WORDPIECE,
+ TOK_SENTENCEPIECE,
+} tok_kind_t;
+
+typedef struct {
+ tok_kind_t kind;
+ void *tok;
+ void *ws;
+} bench_tok_t;
+
+static int path_exists(const char *path) {
+ return access(path, R_OK) == 0;
+}
+
+static int load_tokenizer(bench_tok_t *bt, const char *path) {
+ char p[1024];
+ snprintf(p, sizeof(p), "%s/vocab.txt", path);
+ if (path_exists(p)) {
+ bt->kind = TOK_WORDPIECE;
+ bt->tok = wordpiece_tokenizer_load(path);
+ bt->ws = wordpiece_workspace_new();
+ return bt->tok && bt->ws ? 0 : -1;
+ }
+ snprintf(p, sizeof(p), "%s/sentencepiece.bpe.model", path);
+ if (!path_exists(p)) {
+ snprintf(p, sizeof(p), "%s/tokenizer.model", path);
+ if (!path_exists(p))
+ snprintf(p, sizeof(p), "%s/spiece.model", path);
+ }
+ if (path_exists(p)) {
+ bt->kind = TOK_SENTENCEPIECE;
+ bt->tok = sentencepiece_tokenizer_load(path);
+ bt->ws = sentencepiece_workspace_new();
+ return bt->tok && bt->ws ? 0 : -1;
+ }
+ bt->kind = TOK_BPE;
+ const char *vocab = path;
+ snprintf(p, sizeof(p), "%s/vocab.json", path);
+ if (path_exists(p))
+ vocab = p;
+ bt->tok = embed_tokenizer_load(vocab);
+ bt->ws = embed_tokenizer_workspace_new();
+ return bt->tok && bt->ws ? 0 : -1;
+}
+
+static int *tok_encode(bench_tok_t *bt, const char *text, int *n) {
+ switch (bt->kind) {
+ case TOK_WORDPIECE:
+ return wordpiece_tokenizer_encode((wordpiece_tokenizer_t *)bt->tok, text, n);
+ case TOK_SENTENCEPIECE:
+ return sentencepiece_tokenizer_encode((sentencepiece_tokenizer_t *)bt->tok, text, n);
+ default:
+ return embed_tokenizer_encode((embed_tokenizer_t *)bt->tok, text, n);
+ }
+}
+
+static int *tok_encode_ws(bench_tok_t *bt, const char *text, int *n) {
+ switch (bt->kind) {
+ case TOK_WORDPIECE:
+ return wordpiece_tokenizer_encode_with_workspace((wordpiece_tokenizer_t *)bt->tok,
+ (wordpiece_workspace_t *)bt->ws, text, n);
+ case TOK_SENTENCEPIECE:
+ return sentencepiece_tokenizer_encode_with_workspace(
+ (sentencepiece_tokenizer_t *)bt->tok, (sentencepiece_workspace_t *)bt->ws, text, n);
+ default:
+ return embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)bt->tok,
+ (embed_tokenizer_workspace_t *)bt->ws, text, n);
+ }
+}
+
+static int tok_encode_into(bench_tok_t *bt, const char *text, int *out, int cap, int *n) {
+ switch (bt->kind) {
+ case TOK_WORDPIECE:
+ return wordpiece_tokenizer_encode_into((wordpiece_tokenizer_t *)bt->tok,
+ (wordpiece_workspace_t *)bt->ws, text, out, cap, n);
+ case TOK_SENTENCEPIECE:
+ return sentencepiece_tokenizer_encode_into((sentencepiece_tokenizer_t *)bt->tok,
+ (sentencepiece_workspace_t *)bt->ws, text, out,
+ cap, n);
+ default:
+ return embed_tokenizer_encode_into((embed_tokenizer_t *)bt->tok,
+ (embed_tokenizer_workspace_t *)bt->ws, text, out, cap, n);
+ }
+}
+
+static void free_tokenizer(bench_tok_t *bt) {
+ switch (bt->kind) {
+ case TOK_WORDPIECE:
+ wordpiece_workspace_free((wordpiece_workspace_t *)bt->ws);
+ wordpiece_tokenizer_free((wordpiece_tokenizer_t *)bt->tok);
+ break;
+ case TOK_SENTENCEPIECE:
+ sentencepiece_workspace_free((sentencepiece_workspace_t *)bt->ws);
+ sentencepiece_tokenizer_free((sentencepiece_tokenizer_t *)bt->tok);
+ break;
+ default:
+ embed_tokenizer_workspace_free((embed_tokenizer_workspace_t *)bt->ws);
+ embed_tokenizer_free((embed_tokenizer_t *)bt->tok);
+ break;
+ }
+}
+
int main(int argc, char **argv) {
int status = 1;
int **gold = NULL;
@@ -33,8 +140,10 @@ int main(int argc, char **argv) {
int **bufs = NULL;
int *caps = NULL;
+ bench_tok_t bt = {0};
+
if (argc < 4) {
- fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]);
+ fprintf(stderr, "usage: %s MODEL_DIR_OR_VOCAB_JSON RUNS TEXT...\n", argv[0]);
return 2;
}
@@ -45,9 +154,7 @@ int main(int argc, char **argv) {
int n_texts = argc - 3;
char **texts = argv + 3;
- embed_tokenizer_t *tok = embed_tokenizer_load(vocab);
- embed_tokenizer_workspace_t *ws = embed_tokenizer_workspace_new();
- if (!tok || !ws) {
+ if (load_tokenizer(&bt, vocab) != 0) {
fprintf(stderr, "failed to initialize tokenizer\n");
goto cleanup;
}
@@ -61,14 +168,14 @@ int main(int argc, char **argv) {
int tokens_per_pass = 0;
for (int i = 0; i < n_texts; i++) {
- gold[i] = embed_tokenizer_encode(tok, texts[i], &gold_n[i]);
- if (!gold[i] || gold_n[i] <= 0)
+ gold[i] = tok_encode(&bt, texts[i], &gold_n[i]);
+ if (gold_n[i] < 0 || (!gold[i] && gold_n[i] != 0))
goto cleanup;
tokens_per_pass += gold_n[i];
int n_ws = 0;
- int *ids_ws = embed_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws);
- if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) {
+ int *ids_ws = tok_encode_ws(&bt, texts[i], &n_ws);
+ if ((!ids_ws && n_ws != 0) || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) {
free(ids_ws);
goto cleanup;
}
@@ -81,7 +188,7 @@ int main(int argc, char **argv) {
if (!bufs[i])
goto cleanup;
int n_into = 0;
- if (embed_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 ||
+ if (tok_encode_into(&bt, texts[i], bufs[i], caps[i], &n_into) != 0 ||
!same_ids(gold[i], gold_n[i], bufs[i], n_into))
goto cleanup;
}
@@ -91,8 +198,8 @@ int main(int argc, char **argv) {
for (int r = 0; r < runs; r++) {
for (int i = 0; i < n_texts; i++) {
int n = 0;
- int *ids = embed_tokenizer_encode(tok, texts[i], &n);
- if (!ids || n != gold_n[i]) {
+ int *ids = tok_encode(&bt, texts[i], &n);
+ if ((!ids && n != 0) || n != gold_n[i]) {
free(ids);
goto cleanup;
}
@@ -107,8 +214,8 @@ int main(int argc, char **argv) {
for (int r = 0; r < runs; r++) {
for (int i = 0; i < n_texts; i++) {
int n = 0;
- int *ids = embed_tokenizer_encode_with_workspace(tok, ws, texts[i], &n);
- if (!ids || n != gold_n[i]) {
+ int *ids = tok_encode_ws(&bt, texts[i], &n);
+ if ((!ids && n != 0) || n != gold_n[i]) {
free(ids);
goto cleanup;
}
@@ -123,7 +230,7 @@ int main(int argc, char **argv) {
for (int r = 0; r < runs; r++) {
for (int i = 0; i < n_texts; i++) {
int n = 0;
- int rc = embed_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n);
+ int rc = tok_encode_into(&bt, texts[i], bufs[i], caps[i], &n);
if (rc != 0 || n != gold_n[i])
goto cleanup;
into_tokens += n;
@@ -152,7 +259,6 @@ cleanup:
free(gold_n);
free(bufs);
free(caps);
- embed_tokenizer_workspace_free(ws);
- embed_tokenizer_free(tok);
+ free_tokenizer(&bt);
return status;
}
diff --git a/src/cli.c b/src/cli.c
index 7d951e4..2729c25 100644
--- a/src/cli.c
+++ b/src/cli.c
@@ -4,6 +4,7 @@
#include "build.h"
#include "kernels.h"
#include "tokenizer_bpe.h"
+#include "tokenizer_sentencepiece.h"
#include "tokenizer_wordpiece.h"
#ifdef USE_MLX
@@ -103,6 +104,10 @@ typedef struct {
* Qwen byte-level BPE fields above stay NULL. */
wordpiece_tokenizer_t *wp_tok;
wordpiece_workspace_t *wp_tok_ws;
+ /* SentencePiece/XLM-R tokenizer. When sp_tok is set, tokenize_text wraps
+ * core ids with <s> ... </s>. */
+ sentencepiece_tokenizer_t *sp_tok;
+ sentencepiece_workspace_t *sp_tok_ws;
int cls_id;
int sep_id;
int dim;
@@ -115,6 +120,8 @@ static void engine_free_tokenizers(engine_t *e) {
embed_tokenizer_free(e->tok);
wordpiece_workspace_free(e->wp_tok_ws);
wordpiece_tokenizer_free(e->wp_tok);
+ sentencepiece_workspace_free(e->sp_tok_ws);
+ sentencepiece_tokenizer_free(e->sp_tok);
}
typedef struct {
@@ -157,6 +164,24 @@ static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) {
ids[n + 1] = e->sep_id;
out->ids = ids;
out->n_tokens = n + 2;
+ } else if (e->sp_tok) {
+ int n = 0;
+ int *core =
+ sentencepiece_tokenizer_encode_with_workspace(e->sp_tok, e->sp_tok_ws, text, &n);
+ if (n < 0 || n > INT_MAX - 2) {
+ free(core);
+ return -1;
+ }
+ int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
+ return -1;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = e->cls_id;
+ ids[n + 1] = e->sep_id;
+ out->ids = ids;
+ out->n_tokens = n + 2;
} else {
out->ids = embed_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens);
if (!out->ids || out->n_tokens == 0) {
@@ -688,8 +713,8 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
}
- /* Tokenizer: a WordPiece vocab.txt selects the BERT tokenizer (the frontend
- * wraps [CLS]/[SEP]); otherwise the Qwen byte-level BPE vocab.json. */
+ /* Tokenizer selection by files present: WordPiece vocab.txt for BERT,
+ * SentencePiece .model for XLM-R/RoBERTa, otherwise Qwen byte-level BPE. */
engine_t e = {0};
e.cls_id = -1;
e.sep_id = -1;
@@ -708,6 +733,27 @@ int main(int argc, char *argv[]) {
return 1;
}
} else {
+ char sp_path[1024];
+ snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", model_dir);
+ if (access(sp_path, R_OK) != 0) {
+ snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", model_dir);
+ if (access(sp_path, R_OK) != 0)
+ snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", model_dir);
+ }
+ if (access(sp_path, R_OK) == 0) {
+ e.sp_tok = sentencepiece_tokenizer_load(model_dir);
+ if (e.sp_tok) {
+ e.cls_id = sentencepiece_tokenizer_token_id(e.sp_tok, "<s>");
+ e.sep_id = sentencepiece_tokenizer_token_id(e.sp_tok, "</s>");
+ }
+ if (!e.sp_tok || e.cls_id < 0 || e.sep_id < 0) {
+ fprintf(stderr, "failed to load SentencePiece tokenizer: %s\n", sp_path);
+ sentencepiece_tokenizer_free(e.sp_tok);
+ return 1;
+ }
+ }
+ }
+ if (!e.wp_tok && !e.sp_tok) {
char vocab_path[1024];
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
e.tok = embed_tokenizer_load(vocab_path);
@@ -773,6 +819,23 @@ int main(int argc, char *argv[]) {
* like the contextual separator: a vocab that defines it (test models)
* wins; otherwise the released-model family constant applies. */
if (e.append_terminal_token) {
+ if (!e.tok) {
+ fprintf(stderr, "terminal-token appending requires the BPE tokenizer\n");
+ if (e.workspace)
+ embed_workspace_free(e.workspace);
+ if (e.model)
+ embed_model_free(e.model);
+#ifdef USE_MLX
+ if (e.mlx_ctx)
+ embed_mlx_free(e.mlx_ctx);
+#endif
+#ifdef USE_CUDA
+ if (e.cuda_ctx)
+ embed_cuda_free(e.cuda_ctx);
+#endif
+ engine_free_tokenizers(&e);
+ return 1;
+ }
int eot = embed_tokenizer_token_id(e.tok, "<|endoftext|>");
e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
}
@@ -789,9 +852,11 @@ int main(int argc, char *argv[]) {
if (e.wp_tok)
e.wp_tok_ws = wordpiece_workspace_new();
+ else if (e.sp_tok)
+ e.sp_tok_ws = sentencepiece_workspace_new();
else
e.tok_ws = embed_tokenizer_workspace_new();
- if (!e.tok_ws && !e.wp_tok_ws) {
+ if (!e.tok_ws && !e.wp_tok_ws && !e.sp_tok_ws) {
fprintf(stderr, "failed to allocate tokenizer workspace\n");
if (e.workspace)
embed_workspace_free(e.workspace);
diff --git a/src/config.c b/src/config.c
index f693843..a33a135 100644
--- a/src/config.c
+++ b/src/config.c
@@ -209,6 +209,8 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12);
cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0);
+ cfg->position_id_offset = 0;
+ cfg->type_vocab_size = json_get_int(buf, "type_vocab_size", 2);
cfg->family = EMBED_FAMILY_QWEN3;
cfg->qk_norm = 1;
cfg->qkv_bias = 0;
@@ -226,13 +228,25 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
cfg->qk_norm = 0;
cfg->qkv_bias = 1;
cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
- } else if (json_string_equals(buf, "model_type", "bert")) {
+ } else if (json_string_equals(buf, "model_type", "bert") ||
+ json_string_equals(buf, "model_type", "roberta") ||
+ json_string_equals(buf, "model_type", "xlm-roberta")) {
cfg->family = EMBED_FAMILY_BERT;
cfg->qk_norm = 0;
cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */
cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */
cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
+ if (json_string_equals(buf, "model_type", "roberta") ||
+ json_string_equals(buf, "model_type", "xlm-roberta")) {
+ int pad_id = json_get_int(buf, "pad_token_id", 1);
+ if (pad_id < 0 || pad_id == INT_MAX) {
+ fprintf(stderr, "embed_config: invalid RoBERTa pad_token_id\n");
+ free(buf);
+ return -1;
+ }
+ cfg->position_id_offset = pad_id + 1;
+ }
/* hidden_act selects the feed-forward GeLU curve. gelu_new and
* gelu_pytorch_tanh are the same tanh approximation; everything else
* (gelu, absent) is the exact erf GeLU the released encoders use. */
@@ -240,7 +254,9 @@ int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) {
cfg->ffn_act = EMBED_ACT_GELU_TANH;
}
- if (cfg->max_position_embeddings <= 0 || !isfinite(cfg->layer_norm_eps) ||
+ if (cfg->max_position_embeddings <= 0 ||
+ cfg->position_id_offset >= cfg->max_position_embeddings ||
+ cfg->type_vocab_size <= 0 || !isfinite(cfg->layer_norm_eps) ||
cfg->layer_norm_eps <= 0.0f) {
fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n");
free(buf);
diff --git a/src/cuda.cu b/src/cuda.cu
index 6ce7dfb..872834b 100644
--- a/src/cuda.cu
+++ b/src/cuda.cu
@@ -1337,8 +1337,7 @@ static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) {
const embed_weights_t *w = &cpu->weights;
if (load_vector(&ctx->position_embeddings, w->position_embeddings,
c->max_position_embeddings * c->hidden_size) != 0 ||
- load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) !=
- 0 ||
+ load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) != 0 ||
load_vector(&ctx->embed_ln_w, w->embed_ln_w, c->hidden_size) != 0 ||
load_vector(&ctx->embed_ln_b, w->embed_ln_b, c->hidden_size) != 0) {
embed_cuda_free(ctx);
@@ -1436,6 +1435,11 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx,
if (inputs[b].n_tokens > max_seq)
max_seq = inputs[b].n_tokens;
}
+ if (ctx->config.family == EMBED_FAMILY_BERT &&
+ max_seq > ctx->config.max_position_embeddings - ctx->config.position_id_offset) {
+ free(h_offsets);
+ return -1;
+ }
h_offsets[batch] = total;
int *h_ids = (int *)malloc((size_t)total * sizeof(int));
int *h_pos = (int *)malloc((size_t)total * sizeof(int));
@@ -1449,7 +1453,7 @@ static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx,
int off = h_offsets[b];
memcpy(h_ids + off, inputs[b].ids, (size_t)inputs[b].n_tokens * sizeof(int));
for (int i = 0; i < inputs[b].n_tokens; i++)
- h_pos[off + i] = i;
+ h_pos[off + i] = ctx->config.position_id_offset + i;
}
if (ensure_buffers(ctx, total, batch, max_seq) != 0) {
free(h_offsets);
diff --git a/src/embed.h b/src/embed.h
index 028b3d8..5c4f2f3 100644
--- a/src/embed.h
+++ b/src/embed.h
@@ -69,6 +69,8 @@ typedef struct {
int intermediate_size;
int vocab_size;
int max_position_embeddings; /* BERT learned position table size */
+ int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */
+ int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */
float rms_norm_eps;
float layer_norm_eps; /* BERT LayerNorm epsilon */
float rope_theta;
diff --git a/src/forward.c b/src/forward.c
index d5818a4..28243af 100644
--- a/src/forward.c
+++ b/src/forward.c
@@ -296,8 +296,9 @@ int forward_packed_inplace(const embed_model_t *model,
* embedding LayerNorm, then post-norm encoder layers (bias on every projection,
* GeLU feed-forward, no RoPE, no per-head norm). Bidirectional attention is
* block-diagonal over the packed batch, so each sequence's positions restart at
- * 0. Leaves the final hidden states in ws->x; pooling reads them as-is because
- * the last layer's LayerNorm is already the model's final normalization. */
+ * cfg->position_id_offset (0 for BERT, pad_id + 1 for RoBERTa/XLM-R). Leaves the
+ * final hidden states in ws->x; pooling reads them as-is because the last
+ * layer's LayerNorm is already the model's final normalization. */
static int forward_packed_bert(const embed_model_t *model,
embed_workspace_t *ws,
const embed_input_t *inputs,
@@ -320,9 +321,10 @@ static int forward_packed_bert(const embed_model_t *model,
if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings ||
!w->embed_ln_w || !w->embed_ln_b)
return -1;
- if (max_seq > cfg->max_position_embeddings) {
- fprintf(stderr, "embed: sequence length %d exceeds max_position_embeddings %d\n", max_seq,
- cfg->max_position_embeddings);
+ if (max_seq > cfg->max_position_embeddings - cfg->position_id_offset) {
+ fprintf(stderr,
+ "embed: sequence length %d plus position offset %d exceeds max_position_embeddings %d\n",
+ max_seq, cfg->position_id_offset, cfg->max_position_embeddings);
return -1;
}
if (ensure_buffers(ws, cfg, total_seq) != 0)
@@ -345,7 +347,8 @@ static int forward_packed_bert(const embed_model_t *model,
}
float *row = x + (size_t)(start + i) * hidden;
copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden);
- embed_add_inplace(row, w->position_embeddings + (size_t)i * hidden, hidden);
+ int pos = cfg->position_id_offset + i;
+ embed_add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden);
embed_add_inplace(row, w->token_type_embeddings, hidden);
}
}
diff --git a/src/internal.h b/src/internal.h
index cdc02a1..b3c2aa0 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -43,7 +43,7 @@ typedef struct {
/* BERT family (NULL for Qwen3): learned absolute position embeddings,
* token-type (segment) embeddings, and the embedding LayerNorm. */
const float *position_embeddings; /* [max_position_embeddings, hidden] */
- const float *token_type_embeddings; /* [2, hidden] */
+ const float *token_type_embeddings; /* [type_vocab_size, hidden] */
const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */
const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */
} embed_weights_t;
diff --git a/src/mlx.c b/src/mlx.c
index bfa466c..e606b52 100644
--- a/src/mlx.c
+++ b/src/mlx.c
@@ -392,8 +392,10 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye
p = "";
else if (multi_safetensors_find(ctx->ms, "bert.embeddings.word_embeddings.weight", NULL))
p = "bert.";
+ else if (multi_safetensors_find(ctx->ms, "roberta.embeddings.word_embeddings.weight", NULL))
+ p = "roberta.";
else {
- fprintf(stderr, "mlx: BERT word embeddings not found\n");
+ fprintf(stderr, "mlx: BERT-family word embeddings not found\n");
return -1;
}
@@ -404,7 +406,7 @@ static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int laye
int pos_shape[] = {c->max_position_embeddings, h};
snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p);
ctx->position_embeddings = load_tensor(ctx->ms, name, pos_shape, 2);
- int type_shape[] = {2, h};
+ int type_shape[] = {c->type_vocab_size, h};
snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p);
ctx->token_type_embeddings = load_tensor(ctx->ms, name, type_shape, 2);
int ln_shape[] = {h};
@@ -2046,9 +2048,19 @@ static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
mlx_array x_normed;
if (c->family == EMBED_FAMILY_BERT) {
int start2[] = {0, 0}, stride2[] = {1, 1};
- int pos_stop[] = {max_seq, hidden}, type_stop[] = {1, hidden};
+ int pos_start[] = {c->position_id_offset, 0};
+ int pos_stop[] = {c->position_id_offset + max_seq, hidden}, type_stop[] = {1, hidden};
+ if (pos_stop[0] > c->max_position_embeddings) {
+ mlx_array_free(x);
+ mlx_array_free(pool_mask);
+ mlx_array_free(lengths);
+ mlx_array_free(last_indices);
+ if (needs_attn_mask)
+ mlx_array_free(attn_mask);
+ return -1;
+ }
mlx_array pos = mlx_array_new();
- mlx_slice(&pos, ctx->position_embeddings, start2, 2, pos_stop, 2, stride2, 2, S);
+ mlx_slice(&pos, ctx->position_embeddings, pos_start, 2, pos_stop, 2, stride2, 2, S);
mlx_array xp = mlx_array_new();
mlx_add(&xp, x, pos, S);
mlx_array_free(x);
diff --git a/src/model.c b/src/model.c
index 095ce63..2956d7f 100644
--- a/src/model.c
+++ b/src/model.c
@@ -173,15 +173,17 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start
embed_weights_t *w = &model->weights;
char name[256];
- /* sentence-transformers checkpoints store the BERT submodule unprefixed;
- * raw HF checkpoints prefix every tensor with "bert.". */
+ /* sentence-transformers checkpoints store encoder tensors unprefixed; raw
+ * HF BERT and RoBERTa/XLM-R checkpoints use "bert." or "roberta.". */
const char *p;
if (multi_safetensors_find(ms, "embeddings.word_embeddings.weight", NULL))
p = "";
else if (multi_safetensors_find(ms, "bert.embeddings.word_embeddings.weight", NULL))
p = "bert.";
+ else if (multi_safetensors_find(ms, "roberta.embeddings.word_embeddings.weight", NULL))
+ p = "roberta.";
else {
- fprintf(stderr, "embed_model_load: BERT word embeddings not found\n");
+ fprintf(stderr, "embed_model_load: BERT-family word embeddings not found\n");
return -1;
}
@@ -194,7 +196,7 @@ load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start
const int64_t pos_shape[2] = {cfg->max_position_embeddings, hidden};
snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p);
w->position_embeddings = load_norm_f32(ms, name, pos_shape, 2);
- const int64_t type_shape[2] = {2, hidden};
+ const int64_t type_shape[2] = {cfg->type_vocab_size, hidden};
snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p);
w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2);
const int64_t ln_shape[1] = {hidden};
diff --git a/src/server_internal.h b/src/server_internal.h
index e4d7992..56bf25b 100644
--- a/src/server_internal.h
+++ b/src/server_internal.h
@@ -10,6 +10,7 @@
#include "embed.h"
#include "sbuf.h"
#include "tokenizer_bpe.h"
+#include "tokenizer_sentencepiece.h"
#include "tokenizer_wordpiece.h"
#ifdef USE_MLX
@@ -54,6 +55,10 @@
#define EMBED_GTE_QWEN2_QUERY_INSTRUCT \
"Instruct: Given a web search query, retrieve relevant passages that answer the " \
"query\nQuery: "
+#define EMBED_E5_QUERY_INSTRUCT \
+ "Instruct: Given a web search query, retrieve relevant passages that answer the " \
+ "query\nQuery: "
+#define EMBED_SNOWFLAKE_QUERY_INSTRUCT "query: "
/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
* short-request throughput over 8 with no long-document penalty, while 128
* was no faster and inflated long-document tail latency. */
@@ -80,6 +85,8 @@ typedef enum {
MODEL_BGE_SMALL,
MODEL_BGE_BASE,
MODEL_BGE_LARGE,
+ MODEL_E5_MULTI_LARGE_INSTRUCT,
+ MODEL_SNOWFLAKE_ARCTIC_L_V2,
MODEL_CTX_06,
MODEL_CTX_4,
MODEL_LATE_06,
@@ -124,6 +131,8 @@ typedef struct {
* Qwen byte-level BPE fields above stay NULL. */
wordpiece_tokenizer_t *wp_tok;
wordpiece_workspace_t *wp_tok_ws;
+ sentencepiece_tokenizer_t *sp_tok;
+ sentencepiece_workspace_t *sp_tok_ws;
int cls_id;
int sep_id;
int context_separator_id;
diff --git a/src/server_models.c b/src/server_models.c
index ebe5811..32199d9 100644
--- a/src/server_models.c
+++ b/src/server_models.c
@@ -13,7 +13,7 @@
* k_models is the static table of every model the server can serve (name,
* dimensions, attention/pooling mode, API family). load_one_model resolves a
* served id to a slot, picks the tokenizer by the files present (WordPiece
- * vocab.txt -> BERT, else byte-level BPE vocab.json), and loads the CPU model
+ * vocab.txt, SentencePiece .model, else byte-level BPE vocab.json), and loads the CPU model
* (GPU backends load on the worker thread). The tokenize_* and model_embed_*
* helpers are the single dispatch point the request handlers call, hiding the
* CPU/MLX/CUDA backend choice behind one signature.
@@ -43,6 +43,12 @@ const model_info k_models[] = {
EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
{"bge-large-en-v1.5", MODEL_KIND_STANDARD, 1024, 1024, 0, EMBED_ATTENTION_BIDIRECTIONAL,
EMBED_POOL_CLS, 1, EMBED_API_OPENAI, NULL},
+ {"multilingual-e5-large-instruct", MODEL_KIND_STANDARD, 1024, 1024, 0,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 1, EMBED_API_OPENAI,
+ EMBED_E5_QUERY_INSTRUCT},
+ {"snowflake-arctic-embed-l-v2.0", MODEL_KIND_STANDARD, 1024, 256, 0,
+ EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_CLS, 1, EMBED_API_OPENAI,
+ EMBED_SNOWFLAKE_QUERY_INSTRUCT},
{"pplx-embed-context-v1-0.6b", MODEL_KIND_CONTEXTUAL, 1024, 128, 0,
EMBED_ATTENTION_BIDIRECTIONAL, EMBED_POOL_MEAN, 0, EMBED_API_PERPLEXITY, NULL},
{"pplx-embed-context-v1-4b", MODEL_KIND_CONTEXTUAL, 2560, 128, 0, EMBED_ATTENTION_BIDIRECTIONAL,
@@ -143,6 +149,30 @@ int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
out->n_tokens = n + 2;
return 0;
}
+ if (m->sp_tok) {
+ int n = 0;
+ int *core =
+ sentencepiece_tokenizer_encode_with_workspace(m->sp_tok, m->sp_tok_ws, text, &n);
+ if (j)
+ j->tokenize_ns += nstime() - t0;
+ if (n < 0 || n > INT_MAX - 2) {
+ free(core);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
+ memset(out, 0, sizeof(*out));
+ return -1;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = m->cls_id;
+ ids[n + 1] = m->sep_id;
+ out->ids = ids;
+ out->n_tokens = n + 2;
+ return 0;
+ }
out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens);
if (j)
j->tokenize_ns += nstime() - t0;
@@ -299,9 +329,8 @@ int load_one_model(http_server *s, model_slot slot, const char *path) {
m->terminal_token_id = -1;
m->cls_id = -1;
m->sep_id = -1;
- /* Pick the tokenizer by the files present: a WordPiece vocab.txt marks a
- * BERT encoder; otherwise the Qwen byte-level BPE vocab.json. The two never
- * coexist in a model directory, so the probe is unambiguous. */
+ /* Pick the tokenizer by the files present: WordPiece vocab.txt, then a
+ * SentencePiece .model, otherwise the Qwen byte-level BPE vocab.json. */
char wp_path[1024];
snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path);
if (access(wp_path, R_OK) == 0) {
@@ -323,6 +352,33 @@ int load_one_model(http_server *s, model_slot slot, const char *path) {
return -1;
}
} else {
+ char sp_path[1024];
+ snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", path);
+ if (access(sp_path, R_OK) != 0) {
+ snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", path);
+ if (access(sp_path, R_OK) != 0)
+ snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", path);
+ }
+ if (access(sp_path, R_OK) == 0) {
+ m->sp_tok = sentencepiece_tokenizer_load(path);
+ if (!m->sp_tok) {
+ server_log("embed-server: failed to load SentencePiece tokenizer: %s", sp_path);
+ return -1;
+ }
+ m->sp_tok_ws = sentencepiece_workspace_new();
+ if (!m->sp_tok_ws) {
+ server_log("embed-server: failed to allocate SentencePiece workspace: %s", path);
+ return -1;
+ }
+ m->cls_id = sentencepiece_tokenizer_token_id(m->sp_tok, "<s>");
+ m->sep_id = sentencepiece_tokenizer_token_id(m->sp_tok, "</s>");
+ if (m->cls_id < 0 || m->sep_id < 0) {
+ server_log("embed-server: SentencePiece vocab missing <s>/</s>: %s", path);
+ return -1;
+ }
+ }
+ }
+ if (!m->wp_tok && !m->sp_tok) {
char vocab_path[1024];
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path);
m->tok = embed_tokenizer_load(vocab_path);
@@ -433,6 +489,10 @@ void free_models(http_server *s) {
wordpiece_workspace_free(m->wp_tok_ws);
if (m->wp_tok)
wordpiece_tokenizer_free(m->wp_tok);
+ if (m->sp_tok_ws)
+ sentencepiece_workspace_free(m->sp_tok_ws);
+ if (m->sp_tok)
+ sentencepiece_tokenizer_free(m->sp_tok);
if (m->cpu_ws)
embed_workspace_free(m->cpu_ws);
if (m->cpu_model)
diff --git a/src/tokenizer_sentencepiece.c b/src/tokenizer_sentencepiece.c
new file mode 100644
index 0000000..657e8c9
--- /dev/null
+++ b/src/tokenizer_sentencepiece.c
@@ -0,0 +1,963 @@
+/*
+ * tokenizer_sentencepiece.c - SentencePiece Unigram tokenizer.
+ *
+ * The implementation reads the small subset of ModelProto needed at runtime:
+ * pieces, scores, piece types, trainer model_type, and normalizer options. The
+ * normalizer supports SentencePiece's precompiled charsmap format used by
+ * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program
+ * over UTF-8 byte offsets.
+ */
+
+#include "tokenizer_sentencepiece.h"
+
+#include <errno.h>
+#include <float.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+ SP_MODEL_UNIGRAM = 1,
+ SP_PIECE_NORMAL = 1,
+ SP_PIECE_UNKNOWN = 2,
+ SP_PIECE_CONTROL = 3,
+ SP_PIECE_USER_DEFINED = 4,
+ SP_PIECE_UNUSED = 5,
+ SP_PIECE_BYTE = 6,
+};
+
+typedef struct {
+ const char *key;
+ uint16_t len;
+ int val;
+} sp_entry_t;
+
+typedef struct {
+ int id;
+ int prev;
+ double score;
+} sp_best_t;
+
+struct sentencepiece_workspace {
+ char *norm;
+ size_t norm_len;
+ size_t norm_cap;
+ sp_best_t *best;
+ int best_cap;
+ int *ids;
+ int ids_cap;
+};
+
+static uint64_t fnv1a_bytes(const char *s, size_t n) {
+ uint64_t h = 1469598103934665603ULL;
+ for (size_t i = 0; i < n; i++) {
+ h ^= (unsigned char)s[i];
+ h *= 1099511628211ULL;
+ }
+ return h;
+}
+
+static int next_pow2(int x) {
+ int p = 1;
+ while (p < x && p <= (1 << 29))
+ p <<= 1;
+ return p;
+}
+
+static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) {
+ if (len <= 0 || len > UINT16_MAX)
+ return -1;
+ uint64_t mask = (uint64_t)cap - 1;
+ int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
+ while (map[pos].key) {
+ if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) {
+ map[pos].val = val;
+ return 0;
+ }
+ pos = (int)((pos + 1) & (int)mask);
+ }
+ map[pos].key = key;
+ map[pos].len = (uint16_t)len;
+ map[pos].val = val;
+ return 0;
+}
+
+static int map_get(const sp_entry_t *map, int cap, const char *key, int len) {
+ if (!map || len <= 0)
+ return -1;
+ uint64_t mask = (uint64_t)cap - 1;
+ int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
+ while (map[pos].key) {
+ if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0)
+ return map[pos].val;
+ pos = (int)((pos + 1) & (int)mask);
+ }
+ return -1;
+}
+
+static int is_tokenizable_type(int type) {
+ return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED ||
+ type == SP_PIECE_BYTE;
+}
+
+static int reserve_chars(char **buf, size_t *cap, size_t need) {
+ if (need <= *cap)
+ return 0;
+ size_t nc = *cap ? *cap : 128;
+ while (nc < need) {
+ if (nc > SIZE_MAX / 2)
+ return -1;
+ nc *= 2;
+ }
+ char *p = (char *)realloc(*buf, nc);
+ if (!p)
+ return -1;
+ *buf = p;
+ *cap = nc;
+ return 0;
+}
+
+static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) {
+ if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0)
+ return -1;
+ memcpy(*buf + *len, s, n);
+ *len += n;
+ (*buf)[*len] = '\0';
+ return 0;
+}
+
+static int reserve_best(sentencepiece_workspace_t *ws, int need) {
+ if (need <= ws->best_cap)
+ return 0;
+ int nc = ws->best_cap ? ws->best_cap : 128;
+ while (nc < need) {
+ if (nc > INT_MAX / 2)
+ return -1;
+ nc *= 2;
+ }
+ sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p));
+ if (!p)
+ return -1;
+ ws->best = p;
+ ws->best_cap = nc;
+ return 0;
+}
+
+static int reserve_ids(sentencepiece_workspace_t *ws, int need) {
+ if (need <= ws->ids_cap)
+ return 0;
+ int nc = ws->ids_cap ? ws->ids_cap : 64;
+ while (nc < need) {
+ if (nc > INT_MAX / 2)
+ return -1;
+ nc *= 2;
+ }
+ int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p));
+ if (!p)
+ return -1;
+ ws->ids = p;
+ ws->ids_cap = nc;
+ return 0;
+}
+
+static int utf8_len(unsigned char c) {
+ if (c < 0x80)
+ return 1;
+ if ((c & 0xE0) == 0xC0)
+ return 2;
+ if ((c & 0xF0) == 0xE0)
+ return 3;
+ if ((c & 0xF8) == 0xF0)
+ return 4;
+ return 1;
+}
+
+static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) {
+ if (off >= n)
+ return 0;
+ const unsigned char *p = (const unsigned char *)s + off;
+ int len = utf8_len(p[0]);
+ if (off + (size_t)len > n) {
+ *len_out = 1;
+ return 0;
+ }
+ if (len > 1) {
+ for (int i = 1; i < len; i++) {
+ if ((p[i] & 0xC0) != 0x80) {
+ *len_out = 1;
+ return 0;
+ }
+ }
+ }
+ *len_out = (size_t)len;
+ return 1;
+}
+
+static uint32_t read_le32(const unsigned char *p) {
+ return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
+}
+
+static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) {
+ uint64_t v = 0;
+ int shift = 0;
+ while (*p < end && shift <= 63) {
+ unsigned char c = *(*p)++;
+ v |= (uint64_t)(c & 0x7f) << shift;
+ if (!(c & 0x80)) {
+ *out = v;
+ return 0;
+ }
+ shift += 7;
+ }
+ return -1;
+}
+
+static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) {
+ uint64_t key = 0;
+ if (pb_read_varint(p, end, &key) != 0)
+ return -1;
+ *field = (int)(key >> 3);
+ *wire = (int)(key & 7);
+ return *field > 0 ? 0 : -1;
+}
+
+static int pb_read_len(const unsigned char **p,
+ const unsigned char *end,
+ const unsigned char **data,
+ size_t *len) {
+ uint64_t n = 0;
+ if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p))
+ return -1;
+ *data = *p;
+ *len = (size_t)n;
+ *p += n;
+ return 0;
+}
+
+static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) {
+ uint64_t v = 0;
+ const unsigned char *data = NULL;
+ size_t len = 0;
+ switch (wire) {
+ case 0:
+ return pb_read_varint(p, end, &v);
+ case 1:
+ if ((size_t)(end - *p) < 8)
+ return -1;
+ *p += 8;
+ return 0;
+ case 2:
+ return pb_read_len(p, end, &data, &len);
+ case 5:
+ if ((size_t)(end - *p) < 4)
+ return -1;
+ *p += 4;
+ return 0;
+ default:
+ return -1;
+ }
+}
+
+static char *memdup0(const unsigned char *data, size_t len) {
+ char *s = (char *)malloc(len + 1);
+ if (!s)
+ return NULL;
+ memcpy(s, data, len);
+ s[len] = '\0';
+ return s;
+}
+
+static int
+parse_piece(sentencepiece_tokenizer_t *tok, int idx, const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ char *piece = NULL;
+ float score = 0.0f;
+ int type = SP_PIECE_NORMAL;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ goto fail;
+ if (field == 1 && wire == 2) {
+ const unsigned char *s = NULL;
+ size_t n = 0;
+ if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX)
+ goto fail;
+ free(piece);
+ piece = memdup0(s, n);
+ if (!piece)
+ goto fail;
+ tok->piece_lens[idx] = (uint16_t)n;
+ } else if (field == 2 && wire == 5) {
+ if ((size_t)(end - p) < 4)
+ goto fail;
+ uint32_t bits = read_le32(p);
+ memcpy(&score, &bits, sizeof(score));
+ p += 4;
+ } else if (field == 3 && wire == 0) {
+ uint64_t v = 0;
+ if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX)
+ goto fail;
+ type = (int)v;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ goto fail;
+ }
+ }
+ if (!piece)
+ goto fail;
+ tok->id_to_piece[idx] = piece;
+ tok->scores[idx] = score;
+ tok->types[idx] = (uint8_t)type;
+ if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len)
+ tok->max_piece_len = tok->piece_lens[idx];
+ return 0;
+fail:
+ free(piece);
+ return -1;
+}
+
+static int parse_trainer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ int model_type = SP_MODEL_UNIGRAM;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if (field == 3 && wire == 0) {
+ uint64_t v = 0;
+ if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX)
+ return -1;
+ model_type = (int)v;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ if (model_type != SP_MODEL_UNIGRAM) {
+ fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n",
+ model_type);
+ return -1;
+ }
+ (void)tok;
+ return 0;
+}
+
+static int parse_normalizer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if ((field == 3 || field == 4 || field == 5) && wire == 0) {
+ uint64_t v = 0;
+ if (pb_read_varint(&p, end, &v) != 0)
+ return -1;
+ if (field == 3)
+ tok->add_dummy_prefix = v != 0;
+ else if (field == 4)
+ tok->remove_extra_whitespaces = v != 0;
+ else
+ tok->escape_whitespaces = v != 0;
+ } else if (field == 2 && wire == 2) {
+ const unsigned char *blob = NULL;
+ size_t n = 0;
+ if (pb_read_len(&p, end, &blob, &n) != 0)
+ return -1;
+ if (n >= 4) {
+ uint32_t xcda_bytes = read_le32(blob);
+ if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) {
+ tok->xcda_count = xcda_bytes / 4;
+ tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda));
+ if (!tok->xcda)
+ return -1;
+ for (size_t i = 0; i < tok->xcda_count; i++)
+ tok->xcda[i] = read_le32(blob + 4 + i * 4);
+ tok->prefix_replacements_len = n - 4 - xcda_bytes;
+ tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1);
+ if (!tok->prefix_replacements)
+ return -1;
+ memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes,
+ tok->prefix_replacements_len);
+ tok->prefix_replacements[tok->prefix_replacements_len] = '\0';
+ }
+ }
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static int count_pieces(const unsigned char *data, size_t len) {
+ const unsigned char *p = data, *end = data + len;
+ int n = 0;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if (field == 1 && wire == 2) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0)
+ return -1;
+ n++;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ return n;
+}
+
+static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
+ int n_pieces = count_pieces(data, len);
+ if (n_pieces <= 0)
+ return -1;
+ tok->piece_count = n_pieces;
+ tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece));
+ tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens));
+ tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores));
+ tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types));
+ if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types)
+ return -1;
+
+ const unsigned char *p = data, *end = data + len;
+ int piece_idx = 0;
+ int saw_trainer = 0;
+ while (p < end) {
+ int field = 0, wire = 0;
+ if (pb_read_key(&p, end, &field, &wire) != 0)
+ return -1;
+ if (wire == 2 && field == 1) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces ||
+ parse_piece(tok, piece_idx++, msg, msg_len) != 0)
+ return -1;
+ } else if (wire == 2 && field == 2) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0)
+ return -1;
+ saw_trainer = 1;
+ } else if (wire == 2 && field == 3) {
+ const unsigned char *msg = NULL;
+ size_t msg_len = 0;
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 ||
+ parse_normalizer(tok, msg, msg_len) != 0)
+ return -1;
+ } else if (pb_skip(&p, end, wire) != 0) {
+ return -1;
+ }
+ }
+ if (piece_idx != n_pieces || !saw_trainer)
+ return -1;
+ return 0;
+}
+
+static int raw_piece_id(const sentencepiece_tokenizer_t *tok, const char *piece) {
+ if (!tok || !piece)
+ return -1;
+ return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece,
+ (int)strlen(piece));
+}
+
+static int raw_to_model_id(const sentencepiece_tokenizer_t *tok, int raw_id) {
+ if (!tok || raw_id < 0 || raw_id >= tok->piece_count)
+ return -1;
+ if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (raw_id == 1)
+ return 0; /* <s> */
+ if (raw_id == 2)
+ return 2; /* </s> */
+ if (raw_id == 0)
+ return 3; /* <unk> */
+ return raw_id + 1;
+ }
+ return raw_id;
+}
+
+static int model_to_raw_id(const sentencepiece_tokenizer_t *tok, int id) {
+ if (!tok || id < 0)
+ return -1;
+ if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (id == 0)
+ return 1;
+ if (id == 2)
+ return 2;
+ if (id == 3)
+ return 0;
+ if (id >= 4 && id <= tok->piece_count)
+ return id - 1;
+ return -1;
+ }
+ return id < tok->piece_count ? id : -1;
+}
+
+static int file_contains_xlm_roberta(const char *model_dir, const char *name) {
+ char path[1024];
+ int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name);
+ if (n < 0 || (size_t)n >= sizeof(path))
+ return 0;
+ FILE *f = fopen(path, "rb");
+ if (!f)
+ return 0;
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return 0;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return 0;
+ }
+ char *buf = (char *)malloc((size_t)sz + 1);
+ if (!buf) {
+ fclose(f);
+ return 0;
+ }
+ size_t got = fread(buf, 1, (size_t)sz, f);
+ fclose(f);
+ buf[got] = '\0';
+ int yes = strstr(buf, "XLMRobertaTokenizer") != NULL ||
+ strstr(buf, "XLMRobertaTokenizerFast") != NULL ||
+ strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL;
+ free(buf);
+ return yes;
+}
+
+static int detect_xlm_roberta(const char *model_dir) {
+ return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") ||
+ file_contains_xlm_roberta(model_dir, "config.json");
+}
+
+static int find_model_file(const char *model_dir, char path[1024]) {
+ static const char *names[] = {
+ "sentencepiece.bpe.model",
+ "tokenizer.model",
+ "spiece.model",
+ };
+ for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) {
+ int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]);
+ if (n < 0 || n >= 1024)
+ return -1;
+ FILE *f = fopen(path, "rb");
+ if (f) {
+ fclose(f);
+ return 0;
+ }
+ if (errno != ENOENT)
+ return -1;
+ }
+ return -1;
+}
+
+static int load_file(const char *path, unsigned char **out, size_t *out_len) {
+ FILE *f = fopen(path, "rb");
+ if (!f)
+ return -1;
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return -1;
+ }
+ long sz = ftell(f);
+ if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return -1;
+ }
+ unsigned char *buf = (unsigned char *)malloc((size_t)sz);
+ if (!buf) {
+ fclose(f);
+ return -1;
+ }
+ if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
+ fclose(f);
+ free(buf);
+ return -1;
+ }
+ fclose(f);
+ *out = buf;
+ *out_len = (size_t)sz;
+ return 0;
+}
+
+static uint32_t xcda_base(const sentencepiece_tokenizer_t *tok, size_t idx) {
+ uint32_t v = tok->xcda[idx];
+ return (v >> 10) << ((v & (1u << 9)) >> 6);
+}
+
+static uint32_t xcda_lcheck(const sentencepiece_tokenizer_t *tok, size_t idx) {
+ return tok->xcda[idx] & ((1u << 31) | 0xffu);
+}
+
+static int xcda_leaf(const sentencepiece_tokenizer_t *tok, size_t idx) {
+ return (tok->xcda[idx] >> 8) & 1u;
+}
+
+static uint32_t xcda_value(const sentencepiece_tokenizer_t *tok, size_t idx) {
+ return tok->xcda[idx] & ((1u << 31) - 1u);
+}
+
+typedef struct {
+ const char *data;
+ size_t len;
+ size_t consumed;
+} norm_prefix_t;
+
+static norm_prefix_t normalize_prefix(const sentencepiece_tokenizer_t *tok,
+ const char *input,
+ size_t input_len,
+ size_t off) {
+ static const char repl[] = "\xef\xbf\xbd";
+ if (off >= input_len)
+ return (norm_prefix_t){input + off, 0, 0};
+
+ size_t best_len = 0;
+ size_t best_repl = 0;
+ if (tok->xcda && tok->xcda_count > 0) {
+ uint32_t node = xcda_base(tok, 0);
+ for (size_t p = off; p < input_len; p++) {
+ unsigned char c = (unsigned char)input[p];
+ if (c == 0)
+ break;
+ node ^= c;
+ if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c)
+ break;
+ int leaf = xcda_leaf(tok, node);
+ node ^= xcda_base(tok, node);
+ if (leaf) {
+ best_len = p - off + 1;
+ best_repl = xcda_value(tok, node);
+ }
+ }
+ }
+ if (best_len > 0 && best_repl < tok->prefix_replacements_len) {
+ const char *r = tok->prefix_replacements + best_repl;
+ size_t max = tok->prefix_replacements_len - best_repl;
+ size_t n = strnlen(r, max);
+ return (norm_prefix_t){r, n, best_len};
+ }
+
+ size_t clen = 1;
+ if (valid_utf8_one(input, input_len, off, &clen))
+ return (norm_prefix_t){input + off, clen, clen};
+ return (norm_prefix_t){repl, 3, 1};
+}
+
+static int normalize_into(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text) {
+ ws->norm_len = 0;
+ if (!text)
+ return -1;
+ size_t input_len = strlen(text);
+ const char space[] = "\xe2\x96\x81"; /* U+2581 */
+ const char *space_s = tok->escape_whitespaces ? space : " ";
+ size_t space_n = tok->escape_whitespaces ? 3 : 1;
+ int processing_non_ws = 0;
+ int space_prepended = 0;
+
+ for (size_t off = 0; off < input_len;) {
+ norm_prefix_t r = normalize_prefix(tok, text, input_len, off);
+ for (size_t i = 0; i < r.len; i++) {
+ char c = r.data[i];
+ if (c != ' ') {
+ if (!processing_non_ws) {
+ processing_non_ws = 1;
+ if ((tok->add_dummy_prefix && !space_prepended) ||
+ tok->remove_extra_whitespaces) {
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s,
+ space_n) != 0)
+ return -1;
+ space_prepended = 1;
+ }
+ }
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0)
+ return -1;
+ } else {
+ processing_non_ws = 0;
+ if (!tok->remove_extra_whitespaces) {
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
+ 0)
+ return -1;
+ }
+ }
+ }
+ off += r.consumed ? r.consumed : 1;
+ }
+ if (ws->norm)
+ ws->norm[ws->norm_len] = '\0';
+ return 0;
+}
+
+static double compute_unknown_score(const sentencepiece_tokenizer_t *tok) {
+ float min_score = FLT_MAX;
+ for (int i = 0; i < tok->piece_count; i++) {
+ if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score)
+ min_score = tok->scores[i];
+ }
+ if (min_score == FLT_MAX)
+ min_score = -10.0f;
+ return (double)min_score - 10.0;
+}
+
+static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace_t *ws) {
+ size_t n = ws->norm_len;
+ if (n == 0)
+ return 0;
+ if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 ||
+ reserve_ids(ws, (int)n + 1) != 0)
+ return -1;
+ for (size_t i = 0; i <= n; i++)
+ ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX};
+ ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0};
+ double unk_score = tok->unknown_score;
+
+ for (size_t off = 0; off < n;) {
+ size_t cp_len = 1;
+ valid_utf8_one(ws->norm, n, off, &cp_len);
+ sp_best_t cur = ws->best[off];
+ if (cur.score <= -DBL_MAX / 2.0) {
+ off += cp_len;
+ continue;
+ }
+
+ int single_cp_found = 0;
+ for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) {
+ int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap,
+ ws->norm + off, (int)(end - off));
+ if (raw >= 0 && is_tokenizable_type(tok->types[raw])) {
+ if (end - off == cp_len)
+ single_cp_found = 1;
+ double sc =
+ cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]);
+ if (sc > ws->best[end].score)
+ ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc};
+ }
+ if (end == n)
+ break;
+ size_t next_len = 1;
+ valid_utf8_one(ws->norm, n, end, &next_len);
+ end += next_len;
+ }
+ if (!single_cp_found) {
+ size_t end = off + cp_len;
+ double sc = cur.score + unk_score;
+ if (sc > ws->best[end].score)
+ ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc};
+ }
+ off += cp_len;
+ }
+
+ int out_n = 0;
+ int prev_unk = 0;
+ for (size_t off = n; off > 0;) {
+ sp_best_t b = ws->best[off];
+ if (b.prev < 0 || b.prev >= (int)off)
+ return -1;
+ int is_unk = b.id == tok->unk_id;
+ if (!(prev_unk && is_unk)) {
+ if (out_n >= ws->ids_cap)
+ return -1;
+ ws->ids[out_n++] = b.id;
+ }
+ prev_unk = is_unk;
+ off = (size_t)b.prev;
+ }
+ for (int i = 0; i < out_n / 2; i++) {
+ int t = ws->ids[i];
+ ws->ids[i] = ws->ids[out_n - 1 - i];
+ ws->ids[out_n - 1 - i] = t;
+ }
+ return out_n;
+}
+
+sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) {
+ if (!model_dir)
+ return NULL;
+ char path[1024];
+ if (find_model_file(model_dir, path) != 0) {
+ fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir);
+ return NULL;
+ }
+ unsigned char *data = NULL;
+ size_t len = 0;
+ if (load_file(path, &data, &len) != 0) {
+ fprintf(stderr, "sentencepiece: cannot read %s\n", path);
+ return NULL;
+ }
+
+ sentencepiece_tokenizer_t *tok =
+ (sentencepiece_tokenizer_t *)calloc(1, sizeof(sentencepiece_tokenizer_t));
+ if (!tok) {
+ free(data);
+ return NULL;
+ }
+ tok->add_dummy_prefix = 1;
+ tok->remove_extra_whitespaces = 1;
+ tok->escape_whitespaces = 1;
+ tok->id_map =
+ detect_xlm_roberta(model_dir) ? SENTENCEPIECE_XLM_ROBERTA : SENTENCEPIECE_IDENTITY;
+ if (parse_model(tok, data, len) != 0) {
+ fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path);
+ free(data);
+ sentencepiece_tokenizer_free(tok);
+ return NULL;
+ }
+ free(data);
+
+ int cap = next_pow2(tok->piece_count * 2 + 1);
+ tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t));
+ if (!tok->piece_map) {
+ sentencepiece_tokenizer_free(tok);
+ return NULL;
+ }
+ tok->piece_map_cap = cap;
+ for (int i = 0; i < tok->piece_count; i++) {
+ if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i],
+ tok->piece_lens[i], i) != 0) {
+ sentencepiece_tokenizer_free(tok);
+ return NULL;
+ }
+ }
+
+ tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>"));
+ tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>"));
+ tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>"));
+ tok->pad_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA
+ ? 1
+ : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
+ tok->mask_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA
+ ? tok->piece_count + 1
+ : raw_to_model_id(tok, raw_piece_id(tok, "<mask>"));
+ tok->vocab_size =
+ tok->id_map == SENTENCEPIECE_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
+ tok->unknown_score = compute_unknown_score(tok);
+ if (tok->unk_id < 0 || tok->max_piece_len <= 0) {
+ fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path);
+ sentencepiece_tokenizer_free(tok);
+ return NULL;
+ }
+ return tok;
+}
+
+const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id) {
+ if (!tok || id < 0)
+ return NULL;
+ if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (id == 1)
+ return "<pad>";
+ if (id == tok->piece_count + 1)
+ return "<mask>";
+ }
+ int raw = model_to_raw_id(tok, id);
+ return raw >= 0 ? tok->id_to_piece[raw] : NULL;
+}
+
+int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, const char *token) {
+ if (!tok || !token)
+ return -1;
+ if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
+ if (!strcmp(token, "<s>"))
+ return 0;
+ if (!strcmp(token, "<pad>"))
+ return 1;
+ if (!strcmp(token, "</s>"))
+ return 2;
+ if (!strcmp(token, "<unk>"))
+ return 3;
+ if (!strcmp(token, "<mask>"))
+ return tok->piece_count + 1;
+ }
+ return raw_to_model_id(tok, raw_piece_id(tok, token));
+}
+
+int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens) {
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text)
+ return -1;
+ if (normalize_into(tok, ws, text) != 0)
+ return -1;
+ int n = segment(tok, ws);
+ if (n < 0)
+ return -1;
+ if (out_n_tokens)
+ *out_n_tokens = n;
+ if (n == 0)
+ return 0;
+ if (!out_ids || out_cap < n)
+ return -2;
+ memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids));
+ return 0;
+}
+
+int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens) {
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text)
+ return NULL;
+ int n = 0;
+ int rc = sentencepiece_tokenizer_encode_into(tok, ws, text, NULL, 0, &n);
+ if (rc < 0 && rc != -2)
+ return NULL;
+ if (out_n_tokens)
+ *out_n_tokens = n;
+ if (n == 0)
+ return NULL;
+ int *ids = (int *)malloc((size_t)n * sizeof(*ids));
+ if (!ids)
+ return NULL;
+ memcpy(ids, ws->ids, (size_t)n * sizeof(*ids));
+ return ids;
+}
+
+int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
+ const char *text,
+ int *out_n_tokens) {
+ sentencepiece_workspace_t *ws = sentencepiece_workspace_new();
+ if (!ws) {
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ return NULL;
+ }
+ int *ids = sentencepiece_tokenizer_encode_with_workspace(tok, ws, text, out_n_tokens);
+ sentencepiece_workspace_free(ws);
+ return ids;
+}
+
+sentencepiece_workspace_t *sentencepiece_workspace_new(void) {
+ return (sentencepiece_workspace_t *)calloc(1, sizeof(sentencepiece_workspace_t));
+}
+
+void sentencepiece_workspace_free(sentencepiece_workspace_t *ws) {
+ if (!ws)
+ return;
+ free(ws->norm);
+ free(ws->best);
+ free(ws->ids);
+ free(ws);
+}
+
+void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok) {
+ if (!tok)
+ return;
+ if (tok->id_to_piece) {
+ for (int i = 0; i < tok->piece_count; i++)
+ free(tok->id_to_piece[i]);
+ }
+ free(tok->id_to_piece);
+ free(tok->piece_lens);
+ free(tok->scores);
+ free(tok->types);
+ free(tok->piece_map);
+ free(tok->xcda);
+ free(tok->prefix_replacements);
+ free(tok);
+}
diff --git a/src/tokenizer_sentencepiece.h b/src/tokenizer_sentencepiece.h
new file mode 100644
index 0000000..b4ce48c
--- /dev/null
+++ b/src/tokenizer_sentencepiece.h
@@ -0,0 +1,99 @@
+/*
+ * tokenizer_sentencepiece.h - SentencePiece tokenizer for XLM-R style models.
+ *
+ * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`,
+ * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R
+ * and multilingual E5. It encodes text only; frontends add the model's special
+ * token layout, e.g. <s> ... </s> for XLM-R.
+ */
+
+#ifndef SENTENCEPIECE_TOKENIZER_H
+#define SENTENCEPIECE_TOKENIZER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifndef EMBED_API
+#if defined(__GNUC__)
+#define EMBED_API __attribute__((visibility("default")))
+#else
+#define EMBED_API
+#endif
+#endif
+
+typedef enum {
+ SENTENCEPIECE_IDENTITY = 0,
+ SENTENCEPIECE_XLM_ROBERTA = 1,
+} sentencepiece_id_map_t;
+
+typedef struct {
+ char **id_to_piece; /* [piece_count] raw SentencePiece ids */
+ uint16_t *piece_lens; /* byte length for id_to_piece entries */
+ float *scores; /* SentencePiece score per raw id */
+ uint8_t *types; /* SentencePiece piece type per raw id */
+ int piece_count; /* raw SentencePiece vocabulary size */
+ int vocab_size; /* model embedding vocab size after id mapping */
+ int unk_id; /* model-space unknown-token id */
+ int bos_id; /* model-space beginning-token id, or -1 */
+ int eos_id; /* model-space end-token id, or -1 */
+ int pad_id; /* model-space pad-token id, or -1 */
+ int mask_id; /* model-space mask-token id, or -1 */
+ int max_piece_len; /* max normal/user/unused piece byte length */
+ double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
+ int add_dummy_prefix;
+ int remove_extra_whitespaces;
+ int escape_whitespaces;
+ sentencepiece_id_map_t id_map;
+
+ void *piece_map; /* piece bytes -> raw SentencePiece id */
+ int piece_map_cap;
+
+ uint32_t *xcda;
+ size_t xcda_count;
+ char *prefix_replacements;
+ size_t prefix_replacements_len;
+} sentencepiece_tokenizer_t;
+
+typedef struct sentencepiece_workspace sentencepiece_workspace_t;
+
+/* Load the first supported SentencePiece model found in model_dir. Returns NULL
+ * on missing model file, unsupported model type, malformed protobuf, or OOM. */
+EMBED_API sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir);
+
+/* Decode one model-space id to an internal token string, or NULL when invalid. */
+EMBED_API const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id);
+
+/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
+ * model-space id, or -1 when absent. */
+EMBED_API int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok,
+ const char *token);
+
+/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns
+ * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an
+ * error. */
+EMBED_API int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
+ const char *text,
+ int *out_n_tokens);
+
+EMBED_API sentencepiece_workspace_t *sentencepiece_workspace_new(void);
+EMBED_API void sentencepiece_workspace_free(sentencepiece_workspace_t *ws);
+
+/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if
+ * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */
+EMBED_API int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
+
+/* Same tokenization as sentencepiece_tokenizer_encode(), reusing ws internally
+ * and performing only the final returned int[] allocation. */
+EMBED_API int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
+EMBED_API void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok);
+
+#endif /* SENTENCEPIECE_TOKENIZER_H */
diff --git a/tests/test_sentencepiece.c b/tests/test_sentencepiece.c
new file mode 100644
index 0000000..b597af2
--- /dev/null
+++ b/tests/test_sentencepiece.c
@@ -0,0 +1,266 @@
+/* tests/test_sentencepiece.c - hermetic SentencePiece Unigram checks. */
+
+#include "tokenizer_sentencepiece.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+typedef struct {
+ unsigned char *p;
+ size_t n;
+ size_t cap;
+} buf_t;
+
+static int reserve(buf_t *b, size_t need) {
+ if (need <= b->cap)
+ return 0;
+ size_t nc = b->cap ? b->cap : 128;
+ while (nc < need)
+ nc *= 2;
+ unsigned char *p = (unsigned char *)realloc(b->p, nc);
+ if (!p)
+ return -1;
+ b->p = p;
+ b->cap = nc;
+ return 0;
+}
+
+static int put_byte(buf_t *b, int c) {
+ if (reserve(b, b->n + 1) != 0)
+ return -1;
+ b->p[b->n++] = (unsigned char)c;
+ return 0;
+}
+
+static int put_varint(buf_t *b, uint64_t v) {
+ while (v >= 0x80) {
+ if (put_byte(b, (int)(v | 0x80)) != 0)
+ return -1;
+ v >>= 7;
+ }
+ return put_byte(b, (int)v);
+}
+
+static int put_key(buf_t *b, int field, int wire) {
+ return put_varint(b, ((uint64_t)field << 3) | (uint64_t)wire);
+}
+
+static int put_bytes(buf_t *b, const void *data, size_t n) {
+ if (reserve(b, b->n + n) != 0)
+ return -1;
+ memcpy(b->p + b->n, data, n);
+ b->n += n;
+ return 0;
+}
+
+static int put_fixed32(buf_t *b, float f) {
+ uint32_t v;
+ memcpy(&v, &f, sizeof(v));
+ unsigned char p[4] = {
+ (unsigned char)(v & 0xff),
+ (unsigned char)((v >> 8) & 0xff),
+ (unsigned char)((v >> 16) & 0xff),
+ (unsigned char)((v >> 24) & 0xff),
+ };
+ return put_bytes(b, p, sizeof(p));
+}
+
+static int put_len_msg(buf_t *dst, int field, const buf_t *msg) {
+ return put_key(dst, field, 2) || put_varint(dst, msg->n) || put_bytes(dst, msg->p, msg->n);
+}
+
+static int add_piece(buf_t *model, const char *piece, float score, int type) {
+ buf_t msg = {0};
+ int rc = put_key(&msg, 1, 2) || put_varint(&msg, strlen(piece)) ||
+ put_bytes(&msg, piece, strlen(piece)) || put_key(&msg, 2, 5) ||
+ put_fixed32(&msg, score) || put_key(&msg, 3, 0) || put_varint(&msg, (uint64_t)type) ||
+ put_len_msg(model, 1, &msg);
+ free(msg.p);
+ return rc;
+}
+
+static int write_model(const char *dir, int xlm_roberta) {
+ char path[1024];
+ snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", dir);
+ FILE *f = fopen(path, "wb");
+ if (!f)
+ return -1;
+
+ buf_t model = {0};
+ int rc = 0;
+ rc |= add_piece(&model, "<unk>", 0.0f, 2);
+ rc |= add_piece(&model, "<s>", 0.0f, 3);
+ rc |= add_piece(&model, "</s>", 0.0f, 3);
+ rc |= add_piece(&model, ",", -3.0f, 1);
+ rc |= add_piece(&model, ".", -3.0f, 1);
+ rc |= add_piece(&model, "\xe2\x96\x81", -4.0f, 1);
+ rc |= add_piece(&model, "!", -3.0f, 1);
+ rc |= add_piece(&model, "\xe2\x96\x81Hello", -1.0f, 1);
+ rc |= add_piece(&model, "\xe2\x96\x81world", -1.0f, 1);
+ rc |= add_piece(&model, "\xe5\x8d\x97", -1.0f, 1);
+ rc |= add_piece(&model, "\xe7\x93\x9c", -1.0f, 1);
+ rc |= add_piece(&model, "\xe2\x96\x81Hi", -1.0f, 1);
+
+ buf_t trainer = {0};
+ rc |= put_key(&trainer, 3, 0) || put_varint(&trainer, 1) || put_len_msg(&model, 2, &trainer);
+ free(trainer.p);
+
+ buf_t normalizer = {0};
+ rc |= put_key(&normalizer, 3, 0) || put_varint(&normalizer, 1);
+ rc |= put_key(&normalizer, 4, 0) || put_varint(&normalizer, 1);
+ rc |= put_key(&normalizer, 5, 0) || put_varint(&normalizer, 1);
+ rc |= put_len_msg(&model, 3, &normalizer);
+ free(normalizer.p);
+
+ if (rc == 0 && fwrite(model.p, 1, model.n, f) != model.n)
+ rc = -1;
+ fclose(f);
+ free(model.p);
+ if (rc != 0)
+ return -1;
+
+ if (xlm_roberta) {
+ snprintf(path, sizeof(path), "%s/tokenizer_config.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fputs("{\"tokenizer_class\":\"XLMRobertaTokenizer\"}", f);
+ fclose(f);
+ }
+ return 0;
+}
+
+static int check_ids(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text,
+ const int *want,
+ int want_n,
+ const char *name) {
+ int n = 0;
+ int *ids = sentencepiece_tokenizer_encode_with_workspace(tok, ws, text, &n);
+ int ok = n == want_n;
+ for (int i = 0; ok && i < n; i++)
+ ok = ids[i] == want[i];
+ if (!ok) {
+ fprintf(stderr, "%s: got [", name);
+ for (int i = 0; i < n; i++)
+ fprintf(stderr, "%d%s", ids[i], i + 1 < n ? "," : "");
+ fprintf(stderr, "], want [");
+ for (int i = 0; i < want_n; i++)
+ fprintf(stderr, "%d%s", want[i], i + 1 < want_n ? "," : "");
+ fprintf(stderr, "]\n");
+ }
+ free(ids);
+ return ok ? 0 : -1;
+}
+
+static int run_hermetic(void) {
+ const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
+ char dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-sp-test-XXXXXX", tmp);
+ if (!mkdtemp(dir) || write_model(dir, 1) != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ sentencepiece_tokenizer_t *tok = sentencepiece_tokenizer_load(dir);
+ sentencepiece_workspace_t *ws = sentencepiece_workspace_new();
+ if (!tok || !ws) {
+ fprintf(stderr, "load failed\n");
+ return 1;
+ }
+ if (tok->piece_count != 12 || tok->vocab_size != 14 || tok->unk_id != 3 ||
+ tok->bos_id != 0 || tok->eos_id != 2 || tok->pad_id != 1 || tok->mask_id != 13) {
+ fprintf(stderr, "metadata wrong\n");
+ return 1;
+ }
+ if (sentencepiece_tokenizer_token_id(tok, "<s>") != 0 ||
+ sentencepiece_tokenizer_token_id(tok, "</s>") != 2 ||
+ sentencepiece_tokenizer_token_id(tok, "<unk>") != 3 ||
+ sentencepiece_tokenizer_token_id(tok, ",") != 4 ||
+ sentencepiece_tokenizer_token_id(tok, "<mask>") != 13 ||
+ strcmp(sentencepiece_tokenizer_decode(tok, 8), "\xe2\x96\x81Hello") != 0 ||
+ strcmp(sentencepiece_tokenizer_decode(tok, 1), "<pad>") != 0) {
+ fprintf(stderr, "special id/decode mapping wrong\n");
+ return 1;
+ }
+
+ int rc = 0;
+ rc |= check_ids(tok, ws, "Hello world!", (int[]){8, 9, 7}, 3, "hello");
+ rc |= check_ids(tok, ws, " ", NULL, 0, "spaces");
+ rc |= check_ids(tok, ws, "\xe5\x8d\x97\xe7\x93\x9c", (int[]){6, 10, 11}, 3, "cjk");
+ rc |= check_ids(tok, ws, "missing", (int[]){6, 3}, 2, "unknown-collapse");
+
+ int tiny[1], need = 0;
+ if (sentencepiece_tokenizer_encode_into(tok, ws, "Hello world!", tiny, 1, &need) != -2 ||
+ need != 3) {
+ fprintf(stderr, "small-buffer handling wrong\n");
+ return 1;
+ }
+ sentencepiece_workspace_free(ws);
+ sentencepiece_tokenizer_free(tok);
+
+ char direct[1024];
+ snprintf(direct, sizeof(direct), "%s/embed-sp-direct-XXXXXX", tmp);
+ if (!mkdtemp(direct) || write_model(direct, 0) != 0)
+ return 2;
+ tok = sentencepiece_tokenizer_load(direct);
+ ws = sentencepiece_workspace_new();
+ if (!tok || !ws)
+ return 1;
+ rc |= check_ids(tok, ws, "Hello world!", (int[]){7, 8, 6}, 3, "direct-map");
+ sentencepiece_workspace_free(ws);
+ sentencepiece_tokenizer_free(tok);
+ if (rc)
+ return 1;
+
+ puts("ok: SentencePiece Unigram normalization, Viterbi, XLM-R id mapping");
+ return 0;
+}
+
+static int run_live(const char *model_dir) {
+ sentencepiece_tokenizer_t *tok = sentencepiece_tokenizer_load(model_dir);
+ sentencepiece_workspace_t *ws = sentencepiece_workspace_new();
+ if (!tok || !ws) {
+ fprintf(stderr, "live load failed\n");
+ return 1;
+ }
+ int rc = 0;
+ rc |= check_ids(tok, ws, "Hello world! \xe5\x8d\x97\xe7\x93\x9c",
+ (int[]){35378, 8999, 38, 6, 4617, 39613}, 6, "live-mixed");
+ rc |= check_ids(tok, ws, "caf\xc3\xa9 d\xc3\xa9j\xc3\xa0 vu",
+ (int[]){26216, 15154, 13946}, 3, "live-accents");
+ rc |= check_ids(tok, ws, "\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3 \xef\xbc\x91\xef\xbc\x92\xef\xbc\x93",
+ (int[]){47457, 37638}, 2, "live-nfkc");
+ rc |= check_ids(tok, ws, "query: what is snowflake?",
+ (int[]){41, 1294, 12, 2367, 83, 108203, 13034, 350, 32}, 9,
+ "live-query");
+ rc |= check_ids(tok, ws, "\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbc\xd0\xb8\xd1\x80",
+ (int[]){1813, 18454, 11373}, 3, "live-cyrillic");
+ rc |= check_ids(tok, ws,
+ "\xd9\x85\xd8\xb1\xd8\xad\xd8\xa8\xd8\xa7 \xd8\xa8\xd8\xa7\xd9\x84\xd8\xb9\xd8\xa7\xd9\x84\xd9\x85",
+ (int[]){665, 193478, 258, 1705, 77796}, 5, "live-arabic");
+ rc |= check_ids(tok, ws, "", NULL, 0, "live-empty");
+ rc |= check_ids(tok, ws, " ", NULL, 0, "live-spaces");
+ if (sentencepiece_tokenizer_token_id(tok, "<s>") != 0 ||
+ sentencepiece_tokenizer_token_id(tok, "<pad>") != 1 ||
+ sentencepiece_tokenizer_token_id(tok, "</s>") != 2 ||
+ sentencepiece_tokenizer_token_id(tok, "<unk>") != 3 ||
+ sentencepiece_tokenizer_token_id(tok, "<mask>") != 250001) {
+ fprintf(stderr, "live special ids wrong\n");
+ rc = 1;
+ }
+ sentencepiece_workspace_free(ws);
+ sentencepiece_tokenizer_free(tok);
+ return rc ? 1 : 0;
+}
+
+int main(int argc, char **argv) {
+ if (argc > 1)
+ return run_live(argv[1]);
+ return run_hermetic();
+}
diff --git a/tests/test_xlm_roberta.c b/tests/test_xlm_roberta.c
new file mode 100644
index 0000000..29e05f7
--- /dev/null
+++ b/tests/test_xlm_roberta.c
@@ -0,0 +1,96 @@
+/* tests/test_xlm_roberta.c - hermetic XLM-R/RoBERTa bridge checks. */
+
+#include "embed.h"
+#include "tiny_model.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+static int write_fixture(const char *dir) {
+ const int hidden = 4, heads = 2, inter = 8, vocab = 16, max_pos = 8;
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fprintf(f,
+ "{\"model_type\":\"xlm-roberta\",\"hidden_size\":%d,"
+ "\"num_hidden_layers\":1,\"num_attention_heads\":%d,"
+ "\"intermediate_size\":%d,\"vocab_size\":%d,"
+ "\"max_position_embeddings\":%d,\"layer_norm_eps\":1e-5,"
+ "\"pad_token_id\":1,\"type_vocab_size\":1}",
+ hidden, heads, inter, vocab, max_pos);
+ fclose(f);
+
+ const tm_spec_t specs[] = {
+ {"embeddings.word_embeddings.weight", vocab, hidden},
+ {"embeddings.position_embeddings.weight", max_pos, hidden},
+ {"embeddings.token_type_embeddings.weight", 1, hidden},
+ {"embeddings.LayerNorm.weight", hidden, 0},
+ {"embeddings.LayerNorm.bias", hidden, 0},
+ {"encoder.layer.0.attention.self.query.weight", hidden, hidden},
+ {"encoder.layer.0.attention.self.query.bias", hidden, 0},
+ {"encoder.layer.0.attention.self.key.weight", hidden, hidden},
+ {"encoder.layer.0.attention.self.key.bias", hidden, 0},
+ {"encoder.layer.0.attention.self.value.weight", hidden, hidden},
+ {"encoder.layer.0.attention.self.value.bias", hidden, 0},
+ {"encoder.layer.0.attention.output.dense.weight", hidden, hidden},
+ {"encoder.layer.0.attention.output.dense.bias", hidden, 0},
+ {"encoder.layer.0.attention.output.LayerNorm.weight", hidden, 0},
+ {"encoder.layer.0.attention.output.LayerNorm.bias", hidden, 0},
+ {"encoder.layer.0.intermediate.dense.weight", inter, hidden},
+ {"encoder.layer.0.intermediate.dense.bias", inter, 0},
+ {"encoder.layer.0.output.dense.weight", hidden, inter},
+ {"encoder.layer.0.output.dense.bias", hidden, 0},
+ {"encoder.layer.0.output.LayerNorm.weight", hidden, 0},
+ {"encoder.layer.0.output.LayerNorm.bias", hidden, 0},
+ };
+ snprintf(path, sizeof(path), "%s/model.safetensors", dir);
+ return tm_write_safetensors_with_prefix(path, "F32", specs,
+ (int)(sizeof(specs) / sizeof(specs[0])),
+ "roberta.");
+}
+
+int main(void) {
+ const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
+ char dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-xlm-roberta-test-XXXXXX", tmp);
+ if (!mkdtemp(dir) || write_fixture(dir) != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ embed_model_t *model = embed_model_load(dir);
+ embed_workspace_t *ws = embed_workspace_new(model);
+ const embed_config_t *cfg = embed_model_config(model);
+ if (!model || !ws || !cfg) {
+ fprintf(stderr, "model setup failed\n");
+ return 1;
+ }
+ if (cfg->family != EMBED_FAMILY_BERT || cfg->position_id_offset != 2 ||
+ cfg->type_vocab_size != 1 || cfg->pooling_mode != EMBED_POOL_MEAN ||
+ cfg->attention_mode != EMBED_ATTENTION_BIDIRECTIONAL) {
+ fprintf(stderr, "XLM-R config bridge wrong\n");
+ return 1;
+ }
+
+ float out[4];
+ int ids_ok[] = {0, 4, 5, 6, 7, 2};
+ if (embed_model_encode_into(model, ws, ids_ok, 6, out) != 0) {
+ fprintf(stderr, "XLM-R forward failed at max valid position\n");
+ return 1;
+ }
+ int ids_too_long[] = {0, 4, 5, 6, 7, 8, 2};
+ if (embed_model_encode_into(model, ws, ids_too_long, 7, out) == 0) {
+ fprintf(stderr, "XLM-R position overflow was accepted\n");
+ return 1;
+ }
+
+ embed_workspace_free(ws);
+ embed_model_free(model);
+ puts("ok: XLM-R config, roberta prefix, type_vocab_size=1, position offset");
+ return 0;
+}