diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 14:46:04 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 14:46:04 +0200 |
| commit | 8e8836796e7dd22820c820c0916e83ecdbf38368 (patch) | |
| tree | 4ed9213bd6335d64ec050e27749a6f98047db7a0 /tests | |
| parent | 458563c163a885e336ef3365922af4d17e578f56 (diff) | |
| download | ffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.tar.gz ffwd-8e8836796e7dd22820c820c0916e83ecdbf38368.zip | |
tokenizer: add WordPiece backend for BERT-family models
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/test_wordpiece.c | 165 |
1 files changed, 165 insertions, 0 deletions
diff --git a/tests/test_wordpiece.c b/tests/test_wordpiece.c new file mode 100644 index 0000000..1fd6ff8 --- /dev/null +++ b/tests/test_wordpiece.c @@ -0,0 +1,165 @@ +/* tests/test_wordpiece.c - hermetic WordPiece tokenizer checks. A synthetic + * vocab exercises greedy ## matching, [UNK], casing, accent stripping, + * punctuation splitting, CJK char-splitting, special-token resolution, and the + * workspace encode paths. No model files or network needed. */ + +#include "wordpiece_tokenizer.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +/* id order = line order. */ +static const char *VOCAB[] = { + "[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]", "the", "capital", "of", + "france", "is", "paris", ",", ".", "un", "##aff", "##able", + "cafe", "hello", "world", "!", "2023", "token", "##ization", "\xe4\xb8\xad", +}; +enum { VOCAB_N = (int)(sizeof(VOCAB) / sizeof(VOCAB[0])) }; + +static int write_fixture(const char *dir, int lower_case) { + char path[1024]; + snprintf(path, sizeof(path), "%s/vocab.txt", dir); + FILE *f = fopen(path, "w"); + if (!f) + return -1; + for (int i = 0; i < VOCAB_N; i++) + fprintf(f, "%s\n", VOCAB[i]); + fclose(f); + + snprintf(path, sizeof(path), "%s/tokenizer_config.json", dir); + f = fopen(path, "w"); + if (!f) + return -1; + fprintf(f, "{\"do_lower_case\":%s}", lower_case ? "true" : "false"); + fclose(f); + return 0; +} + +static int check(const wordpiece_tokenizer_t *tok, + const char *text, + const int *want, + int want_n, + const char *name) { + int n = 0; + int *ids = wordpiece_tokenizer_encode(tok, text, &n); + if (!ids) { + fprintf(stderr, "%s: encode returned NULL\n", name); + return -1; + } + int ok = (n == want_n); + for (int i = 0; ok && i < n; i++) + ok = ids[i] == want[i]; + if (!ok) { + fprintf(stderr, "%s: got [", name); + for (int i = 0; i < n; i++) + fprintf(stderr, "%d%s", ids[i], i + 1 < n ? "," : ""); + fprintf(stderr, "], want ["); + for (int i = 0; i < want_n; i++) + fprintf(stderr, "%d%s", want[i], i + 1 < want_n ? "," : ""); + fprintf(stderr, "]\n"); + } + free(ids); + return ok ? 0 : -1; +} + +int main(void) { + const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"; + char dir[1024]; + snprintf(dir, sizeof(dir), "%s/embed-wp-test-XXXXXX", tmp); + if (!mkdtemp(dir) || write_fixture(dir, 1) != 0) { + fprintf(stderr, "fixture creation failed\n"); + return 2; + } + + wordpiece_tokenizer_t *tok = wordpiece_tokenizer_load(dir); + if (!tok) { + fprintf(stderr, "load failed\n"); + return 1; + } + if (tok->vocab_size != VOCAB_N || !tok->do_lower_case || tok->unk_id != 1) { + fprintf(stderr, "load metadata wrong: vocab=%d lower=%d unk=%d\n", tok->vocab_size, + tok->do_lower_case, tok->unk_id); + return 1; + } + + int rc = 0; + /* whole words */ + rc |= check(tok, "the capital of france is paris", (int[]){5, 6, 7, 8, 9, 10}, 6, "sentence"); + /* greedy ## subword matching */ + rc |= check(tok, "unaffable", (int[]){13, 14, 15}, 3, "unaffable"); + rc |= check(tok, "tokenization", (int[]){21, 22}, 2, "tokenization"); + /* lowercasing + punctuation split */ + rc |= check(tok, "The Capital.", (int[]){5, 6, 12}, 3, "casing+punct"); + rc |= check(tok, "hello, world!", (int[]){17, 11, 18, 19}, 4, "punct"); + /* accent stripping: café (combining) and café (precomposed) */ + rc |= check(tok, "caf\xc3\xa9", (int[]){16}, 1, "precomposed-accent"); + rc |= check(tok, "cafe\xcc\x81", (int[]){16}, 1, "combining-accent"); + /* digits as a whole token */ + rc |= check(tok, "2023", (int[]){20}, 1, "digits"); + /* unknown word -> single [UNK] */ + rc |= check(tok, "xyzzy", (int[]){1}, 1, "unknown"); + /* CJK ideographs split per character; the second is unknown */ + rc |= check(tok, "\xe4\xb8\xad\xe6\x96\x87", (int[]){23, 1}, 2, "cjk-split"); + if (rc) + return 1; + + /* named special-token resolution */ + if (wordpiece_tokenizer_token_id(tok, "[CLS]") != 2 || + wordpiece_tokenizer_token_id(tok, "[SEP]") != 3 || + wordpiece_tokenizer_token_id(tok, "[UNK]") != 1 || + wordpiece_tokenizer_token_id(tok, "[MISSING]") != -1) { + fprintf(stderr, "special-token resolution wrong\n"); + return 1; + } + if (strcmp(wordpiece_tokenizer_decode(tok, 5), "the") != 0 || + wordpiece_tokenizer_decode(tok, -1) != NULL) { + fprintf(stderr, "decode wrong\n"); + return 1; + } + + /* the three encode entry points must agree */ + wordpiece_workspace_t *ws = wordpiece_workspace_new(); + int n_alloc = 0; + int *alloc = wordpiece_tokenizer_encode_with_workspace(tok, ws, "the unaffable cafe", &n_alloc); + int into[16], n_into = 0; + int rc2 = wordpiece_tokenizer_encode_into(tok, ws, "the unaffable cafe", into, 16, &n_into); + if (!alloc || rc2 != 0 || n_alloc != n_into || n_alloc <= 0) { + fprintf(stderr, "encode-path counts disagree (%d vs %d, rc=%d)\n", n_alloc, n_into, rc2); + return 1; + } + for (int i = 0; i < n_alloc; i++) { + if (alloc[i] != into[i]) { + fprintf(stderr, "encode-path ids disagree at %d\n", i); + return 1; + } + } + /* a too-small buffer reports the needed count and -2 */ + int tiny[1], n_need = 0; + if (wordpiece_tokenizer_encode_into(tok, ws, "the unaffable cafe", tiny, 1, &n_need) != -2 || + n_need != n_alloc) { + fprintf(stderr, "small-buffer handling wrong\n"); + return 1; + } + free(alloc); + wordpiece_workspace_free(ws); + + /* cased mode keeps capitals distinct (here -> [UNK] since vocab is lower). */ + char cdir[1024]; + snprintf(cdir, sizeof(cdir), "%s/embed-wp-cased-XXXXXX", tmp); + if (!mkdtemp(cdir) || write_fixture(cdir, 0) != 0) + return 2; + wordpiece_tokenizer_t *cased = wordpiece_tokenizer_load(cdir); + if (!cased || cased->do_lower_case) + return 1; + rc |= check(cased, "The", (int[]){1}, 1, "cased-unknown"); + rc |= check(cased, "the", (int[]){5}, 1, "cased-known"); + wordpiece_tokenizer_free(cased); + if (rc) + return 1; + + wordpiece_tokenizer_free(tok); + puts("ok: WordPiece basic+subword tokenization, casing, accents, CJK, specials"); + return 0; +} |
