diff options
| -rw-r--r-- | qwen_tokenizer.c | 15 | ||||
| -rw-r--r-- | tests/test_tokenizer.c | 213 |
2 files changed, 215 insertions, 13 deletions
diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c index ee95326..59f8784 100644 --- a/qwen_tokenizer.c +++ b/qwen_tokenizer.c @@ -55,17 +55,6 @@ static int utf8_encode_cp(int cp, char out[4]) { return 3; } -static int utf8_encode_full_cp(int cp, char out[4]) { - if (cp < 0x80) return utf8_encode_cp(cp, out); - if (cp < 0x800) return utf8_encode_cp(cp, out); - if (cp < 0x10000) return utf8_encode_cp(cp, out); - out[0] = (char)(0xF0 | (cp >> 18)); - out[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); - out[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); - out[3] = (char)(0x80 | (cp & 0x3F)); - return 4; -} - /* * Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text. * Returns allocated string (caller must free). @@ -635,7 +624,7 @@ static char *normalize_nfc_latin(const char *text) { int composed = compose_latin_mark(cp, mark); if (composed) { char tmp[4]; - int n = utf8_encode_full_cp(composed, tmp); + int n = utf8_encode_cp(composed, tmp); for (int i = 0; i < n; i++) out[w++] = tmp[i]; p += mark_len; continue; @@ -1039,7 +1028,7 @@ static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, int composed = compose_latin_mark(cp, mark); if (composed) { char tmp[4]; - int n = utf8_encode_full_cp(composed, tmp); + int n = utf8_encode_cp(composed, tmp); memcpy(ws->normalized + w, tmp, (size_t)n); w += (size_t)n; p += mark_len; diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index 1de61b6..b0c694e 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -8,6 +8,7 @@ #include "tok_fixture.h" #include <stdio.h> +#include <sys/stat.h> #include <stdlib.h> #include <string.h> #include <unistd.h> @@ -47,6 +48,54 @@ static void check_ids(const qwen_tokenizer_t *tok, free(ids); } +/* Concatenated decode of an encoding must reproduce the (NFC-normalized) + * input - byte-level BPE is lossless. */ +static void check_roundtrip(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, const char *text, + const char *want, const char *label) +{ + int n = 0; + int *ids = qwen_tokenizer_encode_with_workspace( + (qwen_tokenizer_t *)tok, ws, text, &n); + TEST_ASSERT(ids != NULL && n > 0); + if (!ids) return; + size_t cap = 1; + for (int i = 0; i < n; i++) + cap += strlen(qwen_tokenizer_decode(tok, ids[i])); + char *got = (char *)malloc(cap); + if (!got) { free(ids); TEST_ASSERT(0); return; } + got[0] = '\0'; + for (int i = 0; i < n; i++) + strcat(got, qwen_tokenizer_decode(tok, ids[i])); + if (strcmp(got, want) != 0) { + fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", + label, text, got, want); + failures++; + } + free(got); + free(ids); +} + +/* Two spellings of the same text must encode identically. */ +static void check_same_ids(const qwen_tokenizer_t *tok, + qwen_tokenizer_workspace_t *ws, + const char *a, const char *b, const char *label) +{ + int na = 0, nb = 0; + int *ia = qwen_tokenizer_encode_with_workspace( + (qwen_tokenizer_t *)tok, ws, a, &na); + if (!ia) { TEST_ASSERT(0); return; } + int *ib = qwen_tokenizer_encode_with_workspace( + (qwen_tokenizer_t *)tok, ws, b, &nb); + if (!ib) { free(ia); TEST_ASSERT(0); return; } + if (na != nb || memcmp(ia, ib, (size_t)na * sizeof(int)) != 0) { + fprintf(stderr, "FAIL %s same-ids \"%s\" vs \"%s\"\n", label, a, b); + failures++; + } + free(ib); + free(ia); +} + static void run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) { @@ -95,10 +144,130 @@ static void run_cases(const qwen_tokenizer_t *tok, * U+2026 is symbolish, so it prefixes the following letter run. */ const int eacute[] = {0xC3, 0xA9}; check_ids(tok, ws, "\xC3\xA9", eacute, 2, label); + + /* NFC: decomposed base+combining-mark spellings must match the + * precomposed codepoint (one pair per mark in the composition table). */ + check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */ + check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */ + check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */ + check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */ + check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */ + check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */ + check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */ + check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */ + check_roundtrip(tok, ws, "cafe\xCC\x81", "caf\xC3\xA9", label); + /* 'x' + acute has no composition: bytes pass through untouched. */ + check_roundtrip(tok, ws, "x\xCC\x81", "x\xCC\x81", label); + + /* Contraction splitting (ASCII apostrophe forms, both cases). */ + check_roundtrip(tok, ws, "it's", "it's", label); + check_roundtrip(tok, ws, "don't", "don't", label); + check_roundtrip(tok, ws, "I'm", "I'm", label); + check_roundtrip(tok, ws, "he'd", "he'd", label); + check_roundtrip(tok, ws, "we're", "we're", label); + check_roundtrip(tok, ws, "you've", "you've", label); + check_roundtrip(tok, ws, "she'll", "she'll", label); + check_roundtrip(tok, ws, "IT'S", "IT'S", label); + check_roundtrip(tok, ws, "rock '", "rock '", label); + + /* Arbitrary and broken byte sequences survive byte-level fallback. */ + check_roundtrip(tok, ws, "\x80", "\x80", label); + check_roundtrip(tok, ws, "\xFF\xFE", "\xFF\xFE", label); + check_roundtrip(tok, ws, "a\xE2\x82", "a\xE2\x82", label); + check_roundtrip(tok, ws, "\xF0\x9F\x98\x80", "\xF0\x9F\x98\x80", label); + check_roundtrip(tok, ws, "tab\tand\nnl 42!", "tab\tand\nnl 42!", label); const int ellipsis[] = {'a', 0xE2, 0x80, 0xA6, 'b'}; check_ids(tok, ws, "a\xE2\x80\xA6" "b", ellipsis, 5, label); } +static void write_file(const char *dir, const char *name, const char *body) +{ + char path[2048]; + snprintf(path, sizeof(path), "%s/%s", dir, name); + FILE *f = fopen(path, "w"); + if (!f) { perror(path); exit(2); } + fputs(body, f); + fclose(f); +} + +/* Loader rejection paths and vocab.json escape handling, in throwaway + * subdirectories of the main fixture dir. */ +static void test_loader_edges(const char *root) +{ + char dir[1152], path[2048]; + + TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL); + + /* Not a JSON object. */ + snprintf(dir, sizeof(dir), "%s/badjson", root); + if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + write_file(dir, "vocab.json", "[1, 2]\n"); + write_file(dir, "merges.txt", "a b\n"); + snprintf(path, sizeof(path), "%s/vocab.json", dir); + TEST_ASSERT(qwen_tokenizer_load(path) == NULL); + + /* Missing or pair-less merges.txt is tolerated: the tokenizer loads + * and encoding falls back to byte/vocab-level (no merges applied). */ + snprintf(dir, sizeof(dir), "%s/nomerges", root); + if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); + snprintf(path, sizeof(path), "%s/vocab.json", dir); + qwen_tokenizer_t *nom = qwen_tokenizer_load(path); + TEST_ASSERT(nom != NULL); + if (nom) { + qwen_tokenizer_workspace_t *nws = qwen_tokenizer_workspace_new(); + int n = 0; + int *ids = qwen_tokenizer_encode_with_workspace(nom, nws, "ab", &n); + TEST_ASSERT(ids && n == 2 && ids[0] == 0 && ids[1] == 1); + free(ids); + qwen_tokenizer_workspace_free(nws); + qwen_tokenizer_free(nom); + } + + snprintf(dir, sizeof(dir), "%s/emptymerges", root); + if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); + write_file(dir, "merges.txt", "#version: 0.2\n\n \n"); + snprintf(path, sizeof(path), "%s/vocab.json", dir); + qwen_tokenizer_t *emp = qwen_tokenizer_load(path); + TEST_ASSERT(emp != NULL); + qwen_tokenizer_free(emp); + + /* Escape forms in vocab keys; a malformed merge line is skipped. */ + snprintf(dir, sizeof(dir), "%s/escapes", root); + if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + write_file(dir, "vocab.json", + "{\"a\":0, \"b\":1, \"ab\":2,\n" + " \"\\u20AC\":3, \"x\\ny\":4, \"q\\\"r\":5,\n" + " \"s\\\\t\":6, \"p\\/q\":7, \"u\\tv\":8, \"\\u0041Z\":9}\n"); + write_file(dir, "merges.txt", "#version: 0.2\nmalformed-line\na b\n"); + snprintf(path, sizeof(path), "%s/vocab.json", dir); + qwen_tokenizer_t *tok = qwen_tokenizer_load(path); + TEST_ASSERT(tok != NULL); + if (tok) { + TEST_ASSERT(qwen_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "x\ny") == 4); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "q\"r") == 5); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "s\\t") == 6); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "p/q") == 7); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "u\tv") == 8); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "AZ") == 9); + TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 2), "ab") == 0); + /* Keys outside the GPT-2 byte alphabet decode to replacement. */ + TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 3), "?") == 0); + qwen_tokenizer_free(tok); + } + + /* A bare filename exercises the no-directory merges-path branch. */ + char saved_cwd[1024]; + if (getcwd(saved_cwd, sizeof(saved_cwd)) && chdir(dir) == 0) { + qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json"); + TEST_ASSERT(rel != NULL); + qwen_tokenizer_free(rel); + if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); } + } +} + int main(void) { char dir[1024]; @@ -124,6 +293,48 @@ int main(void) run_cases(tok, ws, "fast"); + /* decode: byte tokens, merged tokens, and out-of-range ids. */ + TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 'a'), "a") == 0); + TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 259), "hello") == 0); + TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, -1), "") == 0); + TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0); + TEST_ASSERT(strcmp(qwen_tokenizer_decode(NULL, 0), "") == 0); + + /* token_id lookups. */ + TEST_ASSERT(qwen_tokenizer_token_id(tok, "hello") == 259); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID); + TEST_ASSERT(qwen_tokenizer_token_id(tok, "no-such-token") == -1); + TEST_ASSERT(qwen_tokenizer_token_id(NULL, "a") == -1); + TEST_ASSERT(qwen_tokenizer_token_id(tok, NULL) == -1); + + /* encode_into: exact fit, truncation (-2 plus the untruncated prefix), + * zero capacity, empty text, and argument guards. */ + { + int out[8]; + int n = -1; + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, + &n) == 0 && n == 3); + TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262); + /* On overflow the prefix is written and n reports the total + * required count, so a caller can re-size and retry. */ + n = -1; + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, + &n) == -2 && n == 3); + TEST_ASSERT(out[0] == 259 && out[1] == 263); + n = -1; + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, + &n) == -2 && n == 3); + n = -1; + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && + n == 0); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); + TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); + } + TEST_ASSERT(qwen_tokenizer_encode(tok, "", NULL) == NULL); + TEST_ASSERT(qwen_tokenizer_encode(NULL, "x", NULL) == NULL); + qwen_tokenizer_workspace_free(NULL); + qwen_tokenizer_free(NULL); + /* Drop the integer tables: the string-based fallback must produce the * exact same ids. */ free(tok->int_merges); @@ -136,6 +347,8 @@ int main(void) qwen_tokenizer_workspace_free(ws); qwen_tokenizer_free(tok); + test_loader_edges(dir); + if (failures) { fprintf(stderr, "tokenizer tests: %d failure(s)\n", failures); return 1; |
