aboutsummaryrefslogtreecommitdiff
path: root/src/tokenizer_sentencepiece.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 20:30:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 23:38:17 +0200
commit1ea03a65ed2a836005ea7a0061db100e470cc69d (patch)
tree9274145ea702cdc5d9574e5dc3b2dcaa0bd726ae /src/tokenizer_sentencepiece.h
parent7500042ac2c33e0501e3d2ebae5a423e61cc39d1 (diff)
downloadffwd-1ea03a65ed2a836005ea7a0061db100e470cc69d.tar.gz
ffwd-1ea03a65ed2a836005ea7a0061db100e470cc69d.zip
all: add SentencePiece XLM-R support
Diffstat (limited to 'src/tokenizer_sentencepiece.h')
-rw-r--r--src/tokenizer_sentencepiece.h99
1 files changed, 99 insertions, 0 deletions
diff --git a/src/tokenizer_sentencepiece.h b/src/tokenizer_sentencepiece.h
new file mode 100644
index 0000000..b4ce48c
--- /dev/null
+++ b/src/tokenizer_sentencepiece.h
@@ -0,0 +1,99 @@
+/*
+ * tokenizer_sentencepiece.h - SentencePiece tokenizer for XLM-R style models.
+ *
+ * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`,
+ * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R
+ * and multilingual E5. It encodes text only; frontends add the model's special
+ * token layout, e.g. <s> ... </s> for XLM-R.
+ */
+
+#ifndef SENTENCEPIECE_TOKENIZER_H
+#define SENTENCEPIECE_TOKENIZER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifndef EMBED_API
+#if defined(__GNUC__)
+#define EMBED_API __attribute__((visibility("default")))
+#else
+#define EMBED_API
+#endif
+#endif
+
+typedef enum {
+ SENTENCEPIECE_IDENTITY = 0,
+ SENTENCEPIECE_XLM_ROBERTA = 1,
+} sentencepiece_id_map_t;
+
+typedef struct {
+ char **id_to_piece; /* [piece_count] raw SentencePiece ids */
+ uint16_t *piece_lens; /* byte length for id_to_piece entries */
+ float *scores; /* SentencePiece score per raw id */
+ uint8_t *types; /* SentencePiece piece type per raw id */
+ int piece_count; /* raw SentencePiece vocabulary size */
+ int vocab_size; /* model embedding vocab size after id mapping */
+ int unk_id; /* model-space unknown-token id */
+ int bos_id; /* model-space beginning-token id, or -1 */
+ int eos_id; /* model-space end-token id, or -1 */
+ int pad_id; /* model-space pad-token id, or -1 */
+ int mask_id; /* model-space mask-token id, or -1 */
+ int max_piece_len; /* max normal/user/unused piece byte length */
+ double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
+ int add_dummy_prefix;
+ int remove_extra_whitespaces;
+ int escape_whitespaces;
+ sentencepiece_id_map_t id_map;
+
+ void *piece_map; /* piece bytes -> raw SentencePiece id */
+ int piece_map_cap;
+
+ uint32_t *xcda;
+ size_t xcda_count;
+ char *prefix_replacements;
+ size_t prefix_replacements_len;
+} sentencepiece_tokenizer_t;
+
+typedef struct sentencepiece_workspace sentencepiece_workspace_t;
+
+/* Load the first supported SentencePiece model found in model_dir. Returns NULL
+ * on missing model file, unsupported model type, malformed protobuf, or OOM. */
+EMBED_API sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir);
+
+/* Decode one model-space id to an internal token string, or NULL when invalid. */
+EMBED_API const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id);
+
+/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
+ * model-space id, or -1 when absent. */
+EMBED_API int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok,
+ const char *token);
+
+/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns
+ * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an
+ * error. */
+EMBED_API int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
+ const char *text,
+ int *out_n_tokens);
+
+EMBED_API sentencepiece_workspace_t *sentencepiece_workspace_new(void);
+EMBED_API void sentencepiece_workspace_free(sentencepiece_workspace_t *ws);
+
+/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if
+ * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */
+EMBED_API int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
+
+/* Same tokenization as sentencepiece_tokenizer_encode(), reusing ws internally
+ * and performing only the final returned int[] allocation. */
+EMBED_API int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
+ sentencepiece_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
+EMBED_API void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok);
+
+#endif /* SENTENCEPIECE_TOKENIZER_H */