aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 14:46:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 14:46:04 +0200
commit8e8836796e7dd22820c820c0916e83ecdbf38368 (patch)
tree4ed9213bd6335d64ec050e27749a6f98047db7a0
parent458563c163a885e336ef3365922af4d17e578f56 (diff)
downloadffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.tar.gz
ffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.zip
tokenizer: add WordPiece backend for BERT-family models
-rw-r--r--Makefile3
-rw-r--r--tests/test_wordpiece.c165
-rw-r--r--wordpiece_tokenizer.c722
-rw-r--r--wordpiece_tokenizer.h79
4 files changed, 969 insertions, 0 deletions
diff --git a/Makefile b/Makefile
index 76b9f57..3dfb8dd 100644
--- a/Makefile
+++ b/Makefile
@@ -164,6 +164,7 @@ endif
TEST_CC_FLAGS = -Wall -Wextra -O2 -I.
TEST_KERNELS_SRCS = tests/test_kernels.c $(KERNEL_SRCS)
TEST_TOKENIZER_SRCS = tests/test_tokenizer.c qwen_tokenizer.c $(KERNEL_SRCS)
+TEST_WORDPIECE_SRCS = tests/test_wordpiece.c wordpiece_tokenizer.c
TEST_SAFETENSORS_SRCS = tests/test_safetensors.c qwen_safetensors.c
TEST_BF16_SRCS = tests/test_bf16_model.c embed.c embed_config.c qwen_safetensors.c \
$(KERNEL_SRCS)
@@ -194,6 +195,8 @@ test:
$(CC) $(TEST_CC_FLAGS) -o tests/test_tokenizer \
$(TEST_TOKENIZER_SRCS) -lm -lpthread
./tests/test_tokenizer
+ $(CC) $(TEST_CC_FLAGS) -o tests/test_wordpiece $(TEST_WORDPIECE_SRCS)
+ ./tests/test_wordpiece
$(CC) $(TEST_CC_FLAGS) -o tests/test_safetensors $(TEST_SAFETENSORS_SRCS)
./tests/test_safetensors
$(CC) $(TEST_CC_FLAGS) -o tests/test_bf16_model \
diff --git a/tests/test_wordpiece.c b/tests/test_wordpiece.c
new file mode 100644
index 0000000..1fd6ff8
--- /dev/null
+++ b/tests/test_wordpiece.c
@@ -0,0 +1,165 @@
+/* tests/test_wordpiece.c - hermetic WordPiece tokenizer checks. A synthetic
+ * vocab exercises greedy ## matching, [UNK], casing, accent stripping,
+ * punctuation splitting, CJK char-splitting, special-token resolution, and the
+ * workspace encode paths. No model files or network needed. */
+
+#include "wordpiece_tokenizer.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* id order = line order. */
+static const char *VOCAB[] = {
+ "[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "the", "capital", "of",
+ "france", "is", "paris", ",", ".", "un", "##aff", "##able",
+ "cafe", "hello", "world", "!", "2023", "token", "##ization", "\xe4\xb8\xad",
+};
+enum { VOCAB_N = (int)(sizeof(VOCAB) / sizeof(VOCAB[0])) };
+
+static int write_fixture(const char *dir, int lower_case) {
+ char path[1024];
+ snprintf(path, sizeof(path), "%s/vocab.txt", dir);
+ FILE *f = fopen(path, "w");
+ if (!f)
+ return -1;
+ for (int i = 0; i < VOCAB_N; i++)
+ fprintf(f, "%s\n", VOCAB[i]);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/tokenizer_config.json", dir);
+ f = fopen(path, "w");
+ if (!f)
+ return -1;
+ fprintf(f, "{\"do_lower_case\":%s}", lower_case ? "true" : "false");
+ fclose(f);
+ return 0;
+}
+
+static int check(const wordpiece_tokenizer_t *tok,
+ const char *text,
+ const int *want,
+ int want_n,
+ const char *name) {
+ int n = 0;
+ int *ids = wordpiece_tokenizer_encode(tok, text, &n);
+ if (!ids) {
+ fprintf(stderr, "%s: encode returned NULL\n", name);
+ return -1;
+ }
+ int ok = (n == want_n);
+ for (int i = 0; ok && i < n; i++)
+ ok = ids[i] == want[i];
+ if (!ok) {
+ fprintf(stderr, "%s: got [", name);
+ for (int i = 0; i < n; i++)
+ fprintf(stderr, "%d%s", ids[i], i + 1 < n ? "," : "");
+ fprintf(stderr, "], want [");
+ for (int i = 0; i < want_n; i++)
+ fprintf(stderr, "%d%s", want[i], i + 1 < want_n ? "," : "");
+ fprintf(stderr, "]\n");
+ }
+ free(ids);
+ return ok ? 0 : -1;
+}
+
+int main(void) {
+ const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
+ char dir[1024];
+ snprintf(dir, sizeof(dir), "%s/embed-wp-test-XXXXXX", tmp);
+ if (!mkdtemp(dir) || write_fixture(dir, 1) != 0) {
+ fprintf(stderr, "fixture creation failed\n");
+ return 2;
+ }
+
+ wordpiece_tokenizer_t *tok = wordpiece_tokenizer_load(dir);
+ if (!tok) {
+ fprintf(stderr, "load failed\n");
+ return 1;
+ }
+ if (tok->vocab_size != VOCAB_N || !tok->do_lower_case || tok->unk_id != 1) {
+ fprintf(stderr, "load metadata wrong: vocab=%d lower=%d unk=%d\n", tok->vocab_size,
+ tok->do_lower_case, tok->unk_id);
+ return 1;
+ }
+
+ int rc = 0;
+ /* whole words */
+ rc |= check(tok, "the capital of france is paris", (int[]){5, 6, 7, 8, 9, 10}, 6, "sentence");
+ /* greedy ## subword matching */
+ rc |= check(tok, "unaffable", (int[]){13, 14, 15}, 3, "unaffable");
+ rc |= check(tok, "tokenization", (int[]){21, 22}, 2, "tokenization");
+ /* lowercasing + punctuation split */
+ rc |= check(tok, "The Capital.", (int[]){5, 6, 12}, 3, "casing+punct");
+ rc |= check(tok, "hello, world!", (int[]){17, 11, 18, 19}, 4, "punct");
+ /* accent stripping: café (combining) and café (precomposed) */
+ rc |= check(tok, "caf\xc3\xa9", (int[]){16}, 1, "precomposed-accent");
+ rc |= check(tok, "cafe\xcc\x81", (int[]){16}, 1, "combining-accent");
+ /* digits as a whole token */
+ rc |= check(tok, "2023", (int[]){20}, 1, "digits");
+ /* unknown word -> single [UNK] */
+ rc |= check(tok, "xyzzy", (int[]){1}, 1, "unknown");
+ /* CJK ideographs split per character; the second is unknown */
+ rc |= check(tok, "\xe4\xb8\xad\xe6\x96\x87", (int[]){23, 1}, 2, "cjk-split");
+ if (rc)
+ return 1;
+
+ /* named special-token resolution */
+ if (wordpiece_tokenizer_token_id(tok, "[CLS]") != 2 ||
+ wordpiece_tokenizer_token_id(tok, "[SEP]") != 3 ||
+ wordpiece_tokenizer_token_id(tok, "[UNK]") != 1 ||
+ wordpiece_tokenizer_token_id(tok, "[MISSING]") != -1) {
+ fprintf(stderr, "special-token resolution wrong\n");
+ return 1;
+ }
+ if (strcmp(wordpiece_tokenizer_decode(tok, 5), "the") != 0 ||
+ wordpiece_tokenizer_decode(tok, -1) != NULL) {
+ fprintf(stderr, "decode wrong\n");
+ return 1;
+ }
+
+ /* the three encode entry points must agree */
+ wordpiece_workspace_t *ws = wordpiece_workspace_new();
+ int n_alloc = 0;
+ int *alloc = wordpiece_tokenizer_encode_with_workspace(tok, ws, "the unaffable cafe", &n_alloc);
+ int into[16], n_into = 0;
+ int rc2 = wordpiece_tokenizer_encode_into(tok, ws, "the unaffable cafe", into, 16, &n_into);
+ if (!alloc || rc2 != 0 || n_alloc != n_into || n_alloc <= 0) {
+ fprintf(stderr, "encode-path counts disagree (%d vs %d, rc=%d)\n", n_alloc, n_into, rc2);
+ return 1;
+ }
+ for (int i = 0; i < n_alloc; i++) {
+ if (alloc[i] != into[i]) {
+ fprintf(stderr, "encode-path ids disagree at %d\n", i);
+ return 1;
+ }
+ }
+ /* a too-small buffer reports the needed count and -2 */
+ int tiny[1], n_need = 0;
+ if (wordpiece_tokenizer_encode_into(tok, ws, "the unaffable cafe", tiny, 1, &n_need) != -2 ||
+ n_need != n_alloc) {
+ fprintf(stderr, "small-buffer handling wrong\n");
+ return 1;
+ }
+ free(alloc);
+ wordpiece_workspace_free(ws);
+
+ /* cased mode keeps capitals distinct (here -> [UNK] since vocab is lower). */
+ char cdir[1024];
+ snprintf(cdir, sizeof(cdir), "%s/embed-wp-cased-XXXXXX", tmp);
+ if (!mkdtemp(cdir) || write_fixture(cdir, 0) != 0)
+ return 2;
+ wordpiece_tokenizer_t *cased = wordpiece_tokenizer_load(cdir);
+ if (!cased || cased->do_lower_case)
+ return 1;
+ rc |= check(cased, "The", (int[]){1}, 1, "cased-unknown");
+ rc |= check(cased, "the", (int[]){5}, 1, "cased-known");
+ wordpiece_tokenizer_free(cased);
+ if (rc)
+ return 1;
+
+ wordpiece_tokenizer_free(tok);
+ puts("ok: WordPiece basic+subword tokenization, casing, accents, CJK, specials");
+ return 0;
+}
diff --git a/wordpiece_tokenizer.c b/wordpiece_tokenizer.c
new file mode 100644
index 0000000..6119c0d
--- /dev/null
+++ b/wordpiece_tokenizer.c
@@ -0,0 +1,722 @@
+/*
+ * wordpiece_tokenizer.c - BERT-family WordPiece tokenizer.
+ *
+ * Pipeline (matching transformers' BertTokenizer):
+ * BasicTokenizer: clean control/whitespace -> space CJK ideographs ->
+ * whitespace split -> (lowercase + strip accents) -> split on punctuation
+ * WordPiece: greedy longest-match-first against vocab.txt, continuation
+ * pieces marked "##", whole token -> [UNK] when any piece fails to match.
+ *
+ * Unicode coverage is scoped to the English/Western-European + CJK-split path
+ * that the first target encoders (bge-large-en, all-MiniLM-L6-v2) need: ASCII
+ * and Latin-1 Supplement casing/accents, the BERT CJK ranges, and the common
+ * punctuation/whitespace/control classes. Latin Extended beyond Latin-1, full
+ * Unicode category tables, and full NFD normalization are deliberate follow-ups
+ * (they matter for multilingual models, which mostly use SentencePiece anyway).
+ */
+
+#include "wordpiece_tokenizer.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* ===================== string -> int open-addressing map ===================== */
+
+typedef struct {
+ const char *key; /* points into tok->id_to_token; not owned */
+ int val;
+} wp_entry_t;
+
+static uint64_t fnv1a_hash(const char *s) {
+ uint64_t h = 1469598103934665603ULL;
+ for (const unsigned char *p = (const unsigned char *)s; *p; p++) {
+ h ^= (uint64_t)*p;
+ h *= 1099511628211ULL;
+ }
+ return h;
+}
+
+static int next_pow2(int x) {
+ int p = 1;
+ while (p < x)
+ p <<= 1;
+ return p;
+}
+
+static void map_put(wp_entry_t *map, int cap, const char *key, int val) {
+ uint64_t mask = (uint64_t)cap - 1;
+ int pos = (int)(fnv1a_hash(key) & mask);
+ while (map[pos].key) {
+ if (!strcmp(map[pos].key, key)) {
+ map[pos].val = val; /* keep the first (lowest) id, like a dict insert */
+ return;
+ }
+ pos = (int)((pos + 1) & (int)mask);
+ }
+ map[pos].key = key;
+ map[pos].val = val;
+}
+
+static int map_get(const wp_entry_t *map, int cap, const char *key) {
+ if (!map)
+ return -1;
+ uint64_t mask = (uint64_t)cap - 1;
+ int pos = (int)(fnv1a_hash(key) & mask);
+ while (map[pos].key) {
+ if (!strcmp(map[pos].key, key))
+ return map[pos].val;
+ pos = (int)((pos + 1) & (int)mask);
+ }
+ return -1;
+}
+
+/* ============================== UTF-8 helpers ================================ */
+
+/* Decode one codepoint. Returns bytes consumed (>=1); on a malformed lead byte
+ * emits U+FFFD and consumes one byte so iteration always advances. */
+static int utf8_decode(const unsigned char *p, const unsigned char *end, uint32_t *cp) {
+ unsigned char c = p[0];
+ if (c < 0x80) {
+ *cp = c;
+ return 1;
+ }
+ int n;
+ uint32_t v;
+ if ((c & 0xE0) == 0xC0) {
+ n = 2;
+ v = c & 0x1F;
+ } else if ((c & 0xF0) == 0xE0) {
+ n = 3;
+ v = c & 0x0F;
+ } else if ((c & 0xF8) == 0xF0) {
+ n = 4;
+ v = c & 0x07;
+ } else {
+ *cp = 0xFFFD;
+ return 1;
+ }
+ if (p + n > end) {
+ *cp = 0xFFFD;
+ return 1;
+ }
+ for (int i = 1; i < n; i++) {
+ if ((p[i] & 0xC0) != 0x80) {
+ *cp = 0xFFFD;
+ return 1;
+ }
+ v = (v << 6) | (p[i] & 0x3F);
+ }
+ *cp = v;
+ return n;
+}
+
+static int utf8_encode(uint32_t cp, char out[4]) {
+ if (cp < 0x80) {
+ out[0] = (char)cp;
+ return 1;
+ }
+ if (cp < 0x800) {
+ out[0] = (char)(0xC0 | (cp >> 6));
+ out[1] = (char)(0x80 | (cp & 0x3F));
+ return 2;
+ }
+ if (cp < 0x10000) {
+ out[0] = (char)(0xE0 | (cp >> 12));
+ out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
+ out[2] = (char)(0x80 | (cp & 0x3F));
+ return 3;
+ }
+ out[0] = (char)(0xF0 | (cp >> 18));
+ out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
+ out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
+ out[3] = (char)(0x80 | (cp & 0x3F));
+ return 4;
+}
+
+/* ========================== Unicode classification ========================== */
+
+static int wp_is_whitespace(uint32_t cp) {
+ if (cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r')
+ return 1;
+ return cp == 0x00A0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || cp == 0x2028 ||
+ cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000;
+}
+
+static int wp_is_control(uint32_t cp) {
+ if (cp == '\t' || cp == '\n' || cp == '\r')
+ return 0;
+ if (cp < 0x20 || cp == 0x7F || (cp >= 0x80 && cp <= 0x9F))
+ return 1; /* category Cc */
+ return cp == 0x00AD || (cp >= 0x200B && cp <= 0x200F) || (cp >= 0x202A && cp <= 0x202E) ||
+ (cp >= 0x2060 && cp <= 0x2064) || cp == 0xFEFF; /* common Cf */
+}
+
+static int wp_is_punct(uint32_t cp) {
+ if ((cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64) || (cp >= 91 && cp <= 96) ||
+ (cp >= 123 && cp <= 126))
+ return 1; /* all ASCII non-alnum, matching BERT */
+ switch (cp) {
+ case 0x00A1:
+ case 0x00BF:
+ case 0x2010:
+ case 0x2011:
+ case 0x2012:
+ case 0x2013:
+ case 0x2014:
+ case 0x2015:
+ case 0x2018:
+ case 0x2019:
+ case 0x201A:
+ case 0x201B:
+ case 0x201C:
+ case 0x201D:
+ case 0x201E:
+ case 0x201F:
+ case 0x2026:
+ case 0x2032:
+ case 0x2033:
+ case 0x3001:
+ case 0x3002:
+ case 0xFF01:
+ case 0xFF0C:
+ case 0xFF1A:
+ case 0xFF1B:
+ case 0xFF1F:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+static int wp_is_cjk(uint32_t cp) {
+ return (cp >= 0x4E00 && cp <= 0x9FFF) || (cp >= 0x3400 && cp <= 0x4DBF) ||
+ (cp >= 0x20000 && cp <= 0x2A6DF) || (cp >= 0x2A700 && cp <= 0x2B73F) ||
+ (cp >= 0x2B740 && cp <= 0x2B81F) || (cp >= 0x2B820 && cp <= 0x2CEAF) ||
+ (cp >= 0xF900 && cp <= 0xFAFF) || (cp >= 0x2F800 && cp <= 0x2FA1F);
+}
+
+/* Lowercase + strip accents, BERT do_lower_case style. Returns the mapped
+ * codepoint, or 0 to drop it (combining diacritical marks, like NFD + drop Mn).
+ * ASCII and Latin-1 Supplement letters fold to a base ASCII letter where one
+ * exists; letters with no ASCII base (æ ð ø þ) only lowercase. */
+static uint32_t wp_lower_deaccent(uint32_t cp) {
+ if (cp >= 'A' && cp <= 'Z')
+ return cp + 32;
+ if (cp >= 0x0300 && cp <= 0x036F)
+ return 0;
+ if (cp < 0x00C0)
+ return cp;
+ switch (cp) {
+ case 0xC0:
+ case 0xC1:
+ case 0xC2:
+ case 0xC3:
+ case 0xC4:
+ case 0xC5:
+ case 0xE0:
+ case 0xE1:
+ case 0xE2:
+ case 0xE3:
+ case 0xE4:
+ case 0xE5:
+ return 'a';
+ case 0xC7:
+ case 0xE7:
+ return 'c';
+ case 0xC8:
+ case 0xC9:
+ case 0xCA:
+ case 0xCB:
+ case 0xE8:
+ case 0xE9:
+ case 0xEA:
+ case 0xEB:
+ return 'e';
+ case 0xCC:
+ case 0xCD:
+ case 0xCE:
+ case 0xCF:
+ case 0xEC:
+ case 0xED:
+ case 0xEE:
+ case 0xEF:
+ return 'i';
+ case 0xD1:
+ case 0xF1:
+ return 'n';
+ case 0xD2:
+ case 0xD3:
+ case 0xD4:
+ case 0xD5:
+ case 0xD6:
+ case 0xF2:
+ case 0xF3:
+ case 0xF4:
+ case 0xF5:
+ case 0xF6:
+ return 'o';
+ case 0xD9:
+ case 0xDA:
+ case 0xDB:
+ case 0xDC:
+ case 0xF9:
+ case 0xFA:
+ case 0xFB:
+ case 0xFC:
+ return 'u';
+ case 0xDD:
+ case 0xFD:
+ case 0xFF:
+ return 'y';
+ case 0xC6:
+ return 0xE6; /* AE -> ae (no ASCII base) */
+ case 0xD0:
+ return 0xF0; /* ETH -> eth */
+ case 0xD8:
+ return 0xF8; /* O/ -> o/ */
+ case 0xDE:
+ return 0xFE; /* THORN -> thorn */
+ default:
+ return cp; /* incl. 0xD7 x, 0xF7 /, 0xDF ss, already-lowercase letters */
+ }
+}
+
+/* =============================== workspace ================================== */
+
+struct wordpiece_workspace {
+ uint32_t *cps; /* cleaned + CJK-spaced codepoints of the whole input */
+ int cps_cap;
+ uint32_t *norm; /* one word's normalized codepoints */
+ int norm_cap;
+ char *wbuf; /* one sub-word's UTF-8 bytes */
+ int wbuf_cap;
+ int *boff; /* byte offset of each codepoint within wbuf (+1 sentinel) */
+ int boff_cap;
+ char *look; /* "##" + candidate substring, for vocab lookup */
+ int look_cap;
+ int *ids; /* output ids */
+ int ids_cap;
+};
+
+static int grow_u32(uint32_t **buf, int *cap, int need) {
+ if (*cap >= need)
+ return 0;
+ int n = *cap ? *cap : 64;
+ while (n < need)
+ n *= 2;
+ uint32_t *p = (uint32_t *)realloc(*buf, (size_t)n * sizeof(uint32_t));
+ if (!p)
+ return -1;
+ *buf = p;
+ *cap = n;
+ return 0;
+}
+
+static int grow_int(int **buf, int *cap, int need) {
+ if (*cap >= need)
+ return 0;
+ int n = *cap ? *cap : 64;
+ while (n < need)
+ n *= 2;
+ int *p = (int *)realloc(*buf, (size_t)n * sizeof(int));
+ if (!p)
+ return -1;
+ *buf = p;
+ *cap = n;
+ return 0;
+}
+
+static int grow_char(char **buf, int *cap, int need) {
+ if (*cap >= need)
+ return 0;
+ int n = *cap ? *cap : 256;
+ while (n < need)
+ n *= 2;
+ char *p = (char *)realloc(*buf, (size_t)n);
+ if (!p)
+ return -1;
+ *buf = p;
+ *cap = n;
+ return 0;
+}
+
+wordpiece_workspace_t *wordpiece_workspace_new(void) {
+ return (wordpiece_workspace_t *)calloc(1, sizeof(wordpiece_workspace_t));
+}
+
+void wordpiece_workspace_free(wordpiece_workspace_t *ws) {
+ if (!ws)
+ return;
+ free(ws->cps);
+ free(ws->norm);
+ free(ws->wbuf);
+ free(ws->boff);
+ free(ws->look);
+ free(ws->ids);
+ free(ws);
+}
+
+/* ============================== WordPiece core ============================== */
+
+/* Append the WordPiece ids of one BasicTokenizer sub-word (codepoints
+ * word[0..n)) to ws->ids[*n_out]. The whole sub-word maps to [UNK] when it is
+ * too long or any greedy piece misses the vocab. Returns 0 / -1 on OOM. */
+static int wordpiece_subword(const wordpiece_tokenizer_t *tok,
+ wordpiece_workspace_t *ws,
+ const uint32_t *word,
+ int n,
+ int *n_out) {
+ if (n == 0)
+ return 0;
+ if (n > tok->max_chars_per_word) {
+ if (grow_int(&ws->ids, &ws->ids_cap, *n_out + 1) != 0)
+ return -1;
+ ws->ids[(*n_out)++] = tok->unk_id;
+ return 0;
+ }
+
+ /* Materialize the sub-word as UTF-8 with per-codepoint byte offsets. */
+ if (grow_int(&ws->boff, &ws->boff_cap, n + 1) != 0 ||
+ grow_char(&ws->wbuf, &ws->wbuf_cap, n * 4 + 1) != 0)
+ return -1;
+ int bytes = 0;
+ for (int i = 0; i < n; i++) {
+ ws->boff[i] = bytes;
+ bytes += utf8_encode(word[i], ws->wbuf + bytes);
+ }
+ ws->boff[n] = bytes;
+
+ int start_count = *n_out;
+ int start = 0;
+ int bad = 0;
+ while (start < n) {
+ int end = n;
+ int matched = -1;
+ int matched_end = start;
+ while (start < end) {
+ int seg = ws->boff[end] - ws->boff[start];
+ int lp = 0;
+ if (grow_char(&ws->look, &ws->look_cap, seg + 3) != 0)
+ return -1;
+ if (start > 0) {
+ ws->look[lp++] = '#';
+ ws->look[lp++] = '#';
+ }
+ memcpy(ws->look + lp, ws->wbuf + ws->boff[start], (size_t)seg);
+ ws->look[lp + seg] = '\0';
+ int id = map_get((const wp_entry_t *)tok->vocab_map, tok->vocab_map_cap, ws->look);
+ if (id >= 0) {
+ matched = id;
+ matched_end = end;
+ break;
+ }
+ end--;
+ }
+ if (matched < 0) {
+ bad = 1;
+ break;
+ }
+ if (grow_int(&ws->ids, &ws->ids_cap, *n_out + 1) != 0)
+ return -1;
+ ws->ids[(*n_out)++] = matched;
+ start = matched_end;
+ }
+ if (bad) {
+ *n_out = start_count; /* discard partial pieces; emit one [UNK] */
+ if (grow_int(&ws->ids, &ws->ids_cap, *n_out + 1) != 0)
+ return -1;
+ ws->ids[(*n_out)++] = tok->unk_id;
+ }
+ return 0;
+}
+
+/* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. Returns the
+ * token count, or -1 on allocation failure. */
+static int
+wordpiece_run(const wordpiece_tokenizer_t *tok, wordpiece_workspace_t *ws, const char *text) {
+ /* clean_text + tokenize_chinese_chars, in one pass over the input. */
+ const unsigned char *p = (const unsigned char *)text;
+ const unsigned char *end = p + strlen(text);
+ int n_cps = 0;
+ while (p < end) {
+ uint32_t cp;
+ p += utf8_decode(p, end, &cp);
+ if (cp == 0 || cp == 0xFFFD || wp_is_control(cp))
+ continue;
+ if (wp_is_whitespace(cp)) {
+ if (grow_u32(&ws->cps, &ws->cps_cap, n_cps + 1) != 0)
+ return -1;
+ ws->cps[n_cps++] = ' ';
+ continue;
+ }
+ int extra = wp_is_cjk(cp) ? 3 : 1;
+ if (grow_u32(&ws->cps, &ws->cps_cap, n_cps + extra) != 0)
+ return -1;
+ if (extra == 3) {
+ ws->cps[n_cps++] = ' ';
+ ws->cps[n_cps++] = cp;
+ ws->cps[n_cps++] = ' ';
+ } else {
+ ws->cps[n_cps++] = cp;
+ }
+ }
+
+ int n_out = 0;
+ int i = 0;
+ while (i < n_cps) {
+ if (ws->cps[i] == ' ') {
+ i++;
+ continue;
+ }
+ int wstart = i;
+ while (i < n_cps && ws->cps[i] != ' ')
+ i++;
+ int wlen = i - wstart;
+
+ /* lowercase + strip accents (may drop codepoints). */
+ int m = 0;
+ if (grow_u32(&ws->norm, &ws->norm_cap, wlen) != 0)
+ return -1;
+ for (int k = 0; k < wlen; k++) {
+ uint32_t cp = ws->cps[wstart + k];
+ if (tok->do_lower_case) {
+ cp = wp_lower_deaccent(cp);
+ if (cp == 0)
+ continue;
+ }
+ ws->norm[m++] = cp;
+ }
+
+ /* split_on_punc, then WordPiece each piece. */
+ int run = 0;
+ for (int k = 0; k < m; k++) {
+ if (wp_is_punct(ws->norm[k])) {
+ if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (k - run), run, &n_out) != 0)
+ return -1;
+ run = 0;
+ if (wordpiece_subword(tok, ws, ws->norm + k, 1, &n_out) != 0)
+ return -1;
+ } else {
+ run++;
+ }
+ }
+ if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (m - run), run, &n_out) != 0)
+ return -1;
+ }
+ return n_out;
+}
+
+/* ================================ loading =================================== */
+
+static char *read_file(const char *path, long *out_len) {
+ FILE *f = fopen(path, "rb");
+ if (!f)
+ return NULL;
+ if (fseek(f, 0, SEEK_END) != 0) {
+ fclose(f);
+ return NULL;
+ }
+ long len = ftell(f);
+ if (len < 0 || fseek(f, 0, SEEK_SET) != 0) {
+ fclose(f);
+ return NULL;
+ }
+ char *buf = (char *)malloc((size_t)len + 1);
+ if (!buf) {
+ fclose(f);
+ return NULL;
+ }
+ size_t rd = fread(buf, 1, (size_t)len, f);
+ fclose(f);
+ if (rd != (size_t)len) {
+ free(buf);
+ return NULL;
+ }
+ buf[len] = '\0';
+ if (out_len)
+ *out_len = len;
+ return buf;
+}
+
+/* do_lower_case from tokenizer_config.json: default true (BERT uncased). A
+ * lightweight scan is enough - we only need one boolean and avoid a JSON dep
+ * here, mirroring how the config layer reads small flags. */
+static int read_do_lower_case(const char *model_dir) {
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/tokenizer_config.json", model_dir);
+ char *buf = read_file(path, NULL);
+ if (!buf)
+ return 1;
+ int result = 1;
+ const char *k = strstr(buf, "\"do_lower_case\"");
+ if (k) {
+ const char *c = strchr(k, ':');
+ if (c) {
+ c++;
+ while (*c == ' ' || *c == '\t' || *c == '\n' || *c == '\r')
+ c++;
+ if (!strncmp(c, "false", 5))
+ result = 0;
+ }
+ }
+ free(buf);
+ return result;
+}
+
+wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) {
+ if (!model_dir)
+ return NULL;
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/vocab.txt", model_dir);
+ long len = 0;
+ char *buf = read_file(path, &len);
+ if (!buf) {
+ fprintf(stderr, "wordpiece: cannot read %s\n", path);
+ return NULL;
+ }
+
+ /* Count lines (one token each); a single trailing newline adds no token. */
+ int count = 0;
+ for (long i = 0; i < len; i++)
+ if (buf[i] == '\n')
+ count++;
+ if (len > 0 && buf[len - 1] != '\n')
+ count++;
+
+ wordpiece_tokenizer_t *tok = (wordpiece_tokenizer_t *)calloc(1, sizeof(*tok));
+ char **id_to_token = (char **)calloc((size_t)(count > 0 ? count : 1), sizeof(char *));
+ if (!tok || !id_to_token) {
+ free(buf);
+ free(tok);
+ free(id_to_token);
+ return NULL;
+ }
+
+ int n = 0;
+ char *line = buf;
+ for (long i = 0; i <= len && n < count; i++) {
+ if (i == len || buf[i] == '\n') {
+ char *e = buf + i;
+ if (e > line && e[-1] == '\r')
+ e[-1] = '\0';
+ else
+ *e = '\0';
+ id_to_token[n] = strdup(line);
+ if (!id_to_token[n]) {
+ for (int j = 0; j < n; j++)
+ free(id_to_token[j]);
+ free(id_to_token);
+ free(tok);
+ free(buf);
+ return NULL;
+ }
+ n++;
+ line = buf + i + 1;
+ }
+ }
+ free(buf);
+
+ tok->id_to_token = id_to_token;
+ tok->vocab_size = n;
+ tok->do_lower_case = read_do_lower_case(model_dir);
+ tok->max_chars_per_word = 100;
+
+ int cap = next_pow2(n * 2 < 16 ? 16 : n * 2);
+ wp_entry_t *map = (wp_entry_t *)calloc((size_t)cap, sizeof(wp_entry_t));
+ if (!map) {
+ wordpiece_tokenizer_free(tok);
+ return NULL;
+ }
+ for (int i = 0; i < n; i++)
+ map_put(map, cap, id_to_token[i], i);
+ tok->vocab_map = map;
+ tok->vocab_map_cap = cap;
+
+ tok->unk_id = map_get(map, cap, "[UNK]");
+ if (tok->unk_id < 0) {
+ fprintf(stderr, "wordpiece: vocab.txt has no [UNK] token\n");
+ wordpiece_tokenizer_free(tok);
+ return NULL;
+ }
+ return tok;
+}
+
+/* ============================== public API ================================= */
+
+const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id) {
+ if (!tok || id < 0 || id >= tok->vocab_size)
+ return NULL;
+ return tok->id_to_token[id];
+}
+
+int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token) {
+ if (!tok || !token)
+ return -1;
+ return map_get((const wp_entry_t *)tok->vocab_map, tok->vocab_map_cap, token);
+}
+
+int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
+ wordpiece_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens) {
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text)
+ return -1;
+ int n = wordpiece_run(tok, ws, text);
+ if (n < 0)
+ return -1;
+ if (out_n_tokens)
+ *out_n_tokens = n;
+ if (!out_ids || out_cap < n)
+ return out_ids ? -2 : -1;
+ memcpy(out_ids, ws->ids, (size_t)n * sizeof(int));
+ return 0;
+}
+
+int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
+ wordpiece_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens) {
+ if (out_n_tokens)
+ *out_n_tokens = 0;
+ if (!tok || !ws || !text)
+ return NULL;
+ int n = wordpiece_run(tok, ws, text);
+ if (n < 0)
+ return NULL;
+ int *ids = (int *)malloc((size_t)(n > 0 ? n : 1) * sizeof(int));
+ if (!ids)
+ return NULL;
+ memcpy(ids, ws->ids, (size_t)n * sizeof(int));
+ if (out_n_tokens)
+ *out_n_tokens = n;
+ return ids;
+}
+
+int *wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n) {
+ if (out_n)
+ *out_n = 0;
+ wordpiece_workspace_t *ws = wordpiece_workspace_new();
+ if (!ws)
+ return NULL;
+ int *ids = wordpiece_tokenizer_encode_with_workspace(tok, ws, text, out_n);
+ wordpiece_workspace_free(ws);
+ return ids;
+}
+
+void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok) {
+ if (!tok)
+ return;
+ if (tok->id_to_token) {
+ for (int i = 0; i < tok->vocab_size; i++)
+ free(tok->id_to_token[i]);
+ free(tok->id_to_token);
+ }
+ free(tok->vocab_map);
+ free(tok);
+}
diff --git a/wordpiece_tokenizer.h b/wordpiece_tokenizer.h
new file mode 100644
index 0000000..0e1ac20
--- /dev/null
+++ b/wordpiece_tokenizer.h
@@ -0,0 +1,79 @@
+/*
+ * wordpiece_tokenizer.h - BERT-family WordPiece tokenizer.
+ *
+ * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional
+ * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting)
+ * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes
+ * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the
+ * byte-level BPE backend leaves terminal/separator tokens to the frontend.
+ *
+ * The public surface mirrors qwen_tokenizer.h (load / encode with a reusable
+ * workspace / decode / resolve a named special token / free) so both can sit
+ * behind one tokenizer interface later.
+ */
+
+#ifndef WORDPIECE_TOKENIZER_H
+#define WORDPIECE_TOKENIZER_H
+
+#ifndef EMBED_API
+#if defined(__GNUC__)
+#define EMBED_API __attribute__((visibility("default")))
+#else
+#define EMBED_API
+#endif
+#endif
+
+typedef struct {
+ char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */
+ int vocab_size;
+ int unk_id; /* [UNK] id, resolved at load */
+ int do_lower_case; /* lowercase + strip accents before WordPiece */
+ int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */
+
+ /* Opaque token-string -> id hash map. */
+ void *vocab_map;
+ int vocab_map_cap;
+} wordpiece_tokenizer_t;
+
+typedef struct wordpiece_workspace wordpiece_workspace_t;
+
+/* Load from <model_dir>/vocab.txt, reading do_lower_case from
+ * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on
+ * error (missing vocab.txt, missing [UNK], allocation failure). */
+EMBED_API wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir);
+
+/* Decode a single id to its vocab token string (internal pointer), or NULL. */
+EMBED_API const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id);
+
+/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */
+EMBED_API int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token);
+
+/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on
+ * error (and *out_n_tokens = 0). */
+EMBED_API int *
+wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n_tokens);
+
+/* Reusable scratch for allocation-light repeated encoding. */
+EMBED_API wordpiece_workspace_t *wordpiece_workspace_new(void);
+EMBED_API void wordpiece_workspace_free(wordpiece_workspace_t *ws);
+
+/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1
+ * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to
+ * the number of tokens needed/emitted in all cases. */
+EMBED_API int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
+ wordpiece_workspace_t *ws,
+ const char *text,
+ int *out_ids,
+ int out_cap,
+ int *out_n_tokens);
+
+/* Same tokenization as wordpiece_tokenizer_encode(), reusing ws internally and
+ * performing only the final returned int[] allocation. */
+EMBED_API int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
+ wordpiece_workspace_t *ws,
+ const char *text,
+ int *out_n_tokens);
+
+EMBED_API void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok);
+
+#endif /* WORDPIECE_TOKENIZER_H */