aboutsummaryrefslogtreecommitdiff
path: root/tests/test_tokenizer.c
diff options
context:
space:
mode:
Diffstat (limited to 'tests/test_tokenizer.c')
-rw-r--r--tests/test_tokenizer.c159
1 files changed, 93 insertions, 66 deletions
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c
index bed5b7d..5920977 100644
--- a/tests/test_tokenizer.c
+++ b/tests/test_tokenizer.c
@@ -16,31 +16,33 @@
static int failures = 0;
int qwen_verbose = 0;
-#define TEST_ASSERT(cond) do { \
- if (!(cond)) { \
- fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
- failures++; \
- } \
-} while (0)
+#define TEST_ASSERT(cond) \
+ do { \
+ if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+ failures++; \
+ } \
+ } while (0)
static void check_ids(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws, const char *text,
- const int *want, int n_want, const char *label)
-{
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ const int *want,
+ int n_want,
+ const char *label) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, text, &n);
+ int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n);
TEST_ASSERT(ids != NULL);
- if (!ids) return;
+ if (!ids)
+ return;
if (n != n_want) {
- fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n",
- label, text, n, n_want);
+ fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n", label, text, n, n_want);
failures++;
} else {
for (int i = 0; i < n; i++) {
if (ids[i] != want[i]) {
- fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n",
- label, text, i, ids[i], want[i]);
+ fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n", label, text, i, ids[i],
+ want[i]);
failures++;
}
}
@@ -51,25 +53,30 @@ static void check_ids(const qwen_tokenizer_t *tok,
/* Concatenated decode of an encoding must reproduce the (NFC-normalized)
* input - byte-level BPE is lossless. */
static void check_roundtrip(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws, const char *text,
- const char *want, const char *label)
-{
+ qwen_tokenizer_workspace_t *ws,
+ const char *text,
+ const char *want,
+ const char *label) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, text, &n);
+ int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n);
TEST_ASSERT(ids != NULL && n > 0);
- if (!ids) return;
+ if (!ids)
+ return;
size_t cap = 1;
for (int i = 0; i < n; i++)
cap += strlen(qwen_tokenizer_decode(tok, ids[i]));
char *got = (char *)malloc(cap);
- if (!got) { free(ids); TEST_ASSERT(0); return; }
+ if (!got) {
+ free(ids);
+ TEST_ASSERT(0);
+ return;
+ }
got[0] = '\0';
for (int i = 0; i < n; i++)
strcat(got, qwen_tokenizer_decode(tok, ids[i]));
if (strcmp(got, want) != 0) {
- fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n",
- label, text, got, want);
+ fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got,
+ want);
failures++;
}
free(got);
@@ -79,15 +86,21 @@ static void check_roundtrip(const qwen_tokenizer_t *tok,
/* Two spellings of the same text must encode identically. */
static void check_same_ids(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws,
- const char *a, const char *b, const char *label)
-{
+ const char *a,
+ const char *b,
+ const char *label) {
int na = 0, nb = 0;
- int *ia = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, a, &na);
- if (!ia) { TEST_ASSERT(0); return; }
- int *ib = qwen_tokenizer_encode_with_workspace(
- (qwen_tokenizer_t *)tok, ws, b, &nb);
- if (!ib) { free(ia); TEST_ASSERT(0); return; }
+ int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na);
+ if (!ia) {
+ TEST_ASSERT(0);
+ return;
+ }
+ int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb);
+ if (!ib) {
+ free(ia);
+ TEST_ASSERT(0);
+ return;
+ }
if (na != nb || memcmp(ia, ib, (size_t)na * sizeof(int)) != 0) {
fprintf(stderr, "FAIL %s same-ids \"%s\" vs \"%s\"\n", label, a, b);
failures++;
@@ -96,9 +109,8 @@ static void check_same_ids(const qwen_tokenizer_t *tok,
free(ia);
}
-static void run_cases(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws, const char *label)
-{
+static void
+run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) {
/* h e l l o -> he -> he,ll -> hell -> hello */
const int hello[] = {259};
check_ids(tok, ws, "hello", hello, 1, label);
@@ -147,14 +159,14 @@ static void run_cases(const qwen_tokenizer_t *tok,
/* NFC: decomposed base+combining-mark spellings must match the
* precomposed codepoint (one pair per mark in the composition table). */
- check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */
- check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */
- check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */
- check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */
- check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */
- check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */
- check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */
- check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */
+ check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */
+ check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */
+ check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */
+ check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */
+ check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */
+ check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */
+ check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */
+ check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */
check_roundtrip(tok, ws, "cafe\xCC\x81", "caf\xC3\xA9", label);
/* 'x' + acute has no composition: bytes pass through untouched. */
check_roundtrip(tok, ws, "x\xCC\x81", "x\xCC\x81", label);
@@ -177,30 +189,37 @@ static void run_cases(const qwen_tokenizer_t *tok,
check_roundtrip(tok, ws, "\xF0\x9F\x98\x80", "\xF0\x9F\x98\x80", label);
check_roundtrip(tok, ws, "tab\tand\nnl 42!", "tab\tand\nnl 42!", label);
const int ellipsis[] = {'a', 0xE2, 0x80, 0xA6, 'b'};
- check_ids(tok, ws, "a\xE2\x80\xA6" "b", ellipsis, 5, label);
+ check_ids(tok, ws,
+ "a\xE2\x80\xA6"
+ "b",
+ ellipsis, 5, label);
}
-static void write_file(const char *dir, const char *name, const char *body)
-{
+static void write_file(const char *dir, const char *name, const char *body) {
char path[2048];
snprintf(path, sizeof(path), "%s/%s", dir, name);
FILE *f = fopen(path, "w");
- if (!f) { perror(path); exit(2); }
+ if (!f) {
+ perror(path);
+ exit(2);
+ }
fputs(body, f);
fclose(f);
}
/* Loader rejection paths and vocab.json escape handling, in throwaway
* subdirectories of the main fixture dir. */
-static void test_loader_edges(const char *root)
-{
+static void test_loader_edges(const char *root) {
char dir[1152], path[2048];
TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL);
/* Not a JSON object. */
snprintf(dir, sizeof(dir), "%s/badjson", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json", "[1, 2]\n");
write_file(dir, "merges.txt", "a b\n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
@@ -209,7 +228,10 @@ static void test_loader_edges(const char *root)
/* Missing or pair-less merges.txt is tolerated: the tokenizer loads
* and encoding falls back to byte/vocab-level (no merges applied). */
snprintf(dir, sizeof(dir), "%s/nomerges", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
qwen_tokenizer_t *nom = qwen_tokenizer_load(path);
@@ -225,7 +247,10 @@ static void test_loader_edges(const char *root)
}
snprintf(dir, sizeof(dir), "%s/emptymerges", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n");
write_file(dir, "merges.txt", "#version: 0.2\n\n \n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
@@ -235,7 +260,10 @@ static void test_loader_edges(const char *root)
/* Escape forms in vocab keys; a malformed merge line is skipped. */
snprintf(dir, sizeof(dir), "%s/escapes", root);
- if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); }
+ if (mkdir(dir, 0755) != 0) {
+ perror(dir);
+ exit(2);
+ }
write_file(dir, "vocab.json",
"{\"a\":0, \"b\":1, \"ab\":2,\n"
" \"\\u20AC\":3, \"x\\ny\":4, \"q\\\"r\":5,\n"
@@ -264,12 +292,14 @@ static void test_loader_edges(const char *root)
qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json");
TEST_ASSERT(rel != NULL);
qwen_tokenizer_free(rel);
- if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); }
+ if (chdir(saved_cwd) != 0) {
+ perror("chdir back");
+ exit(2);
+ }
}
}
-int main(void)
-{
+int main(void) {
char dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-tok-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
@@ -282,7 +312,8 @@ int main(void)
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir);
qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
TEST_ASSERT(tok != NULL);
- if (!tok) return 1;
+ if (!tok)
+ return 1;
/* The complete byte alphabet means the integer-pair tables must exist. */
TEST_ASSERT(tok->int_merges != NULL);
@@ -312,21 +343,17 @@ int main(void)
{
int out[8];
int n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8,
- &n) == 0 && n == 3);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3);
TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262);
/* On overflow the prefix is written and n reports the total
* required count, so a caller can re-size and retry. */
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2,
- &n) == -2 && n == 3);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3);
TEST_ASSERT(out[0] == 259 && out[1] == 263);
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0,
- &n) == -2 && n == 3);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3);
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 &&
- n == 0);
+ TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0);
TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1);
TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1);
}