From c4a1c97a09da6d8d88c2ac834bf6234506fc41bc Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Mon, 15 Jun 2026 08:40:11 +0200 Subject: chore: run clang-format --- tests/test_tokenizer.c | 159 +++++++++++++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 66 deletions(-) (limited to 'tests/test_tokenizer.c') diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index bed5b7d..5920977 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -16,31 +16,33 @@ static int failures = 0; int qwen_verbose = 0; -#define TEST_ASSERT(cond) do { \ - if (!(cond)) { \ - fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ - failures++; \ - } \ -} while (0) +#define TEST_ASSERT(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + failures++; \ + } \ + } while (0) static void check_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, const char *text, - const int *want, int n_want, const char *label) -{ + qwen_tokenizer_workspace_t *ws, + const char *text, + const int *want, + int n_want, + const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL); - if (!ids) return; + if (!ids) + return; if (n != n_want) { - fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n", - label, text, n, n_want); + fprintf(stderr, "FAIL %s \"%s\": got %d ids want %d\n", label, text, n, n_want); failures++; } else { for (int i = 0; i < n; i++) { if (ids[i] != want[i]) { - fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n", - label, text, i, ids[i], want[i]); + fprintf(stderr, "FAIL %s \"%s\": ids[%d]=%d want %d\n", label, text, i, ids[i], + want[i]); failures++; } } @@ -51,25 +53,30 @@ static void check_ids(const qwen_tokenizer_t *tok, /* Concatenated decode of an encoding must reproduce the (NFC-normalized) * input - byte-level BPE is lossless. */ static void check_roundtrip(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, const char *text, - const char *want, const char *label) -{ + qwen_tokenizer_workspace_t *ws, + const char *text, + const char *want, + const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL && n > 0); - if (!ids) return; + if (!ids) + return; size_t cap = 1; for (int i = 0; i < n; i++) cap += strlen(qwen_tokenizer_decode(tok, ids[i])); char *got = (char *)malloc(cap); - if (!got) { free(ids); TEST_ASSERT(0); return; } + if (!got) { + free(ids); + TEST_ASSERT(0); + return; + } got[0] = '\0'; for (int i = 0; i < n; i++) strcat(got, qwen_tokenizer_decode(tok, ids[i])); if (strcmp(got, want) != 0) { - fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", - label, text, got, want); + fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got, + want); failures++; } free(got); @@ -79,15 +86,21 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, /* Two spellings of the same text must encode identically. */ static void check_same_ids(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, - const char *a, const char *b, const char *label) -{ + const char *a, + const char *b, + const char *label) { int na = 0, nb = 0; - int *ia = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, a, &na); - if (!ia) { TEST_ASSERT(0); return; } - int *ib = qwen_tokenizer_encode_with_workspace( - (qwen_tokenizer_t *)tok, ws, b, &nb); - if (!ib) { free(ia); TEST_ASSERT(0); return; } + int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na); + if (!ia) { + TEST_ASSERT(0); + return; + } + int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb); + if (!ib) { + free(ia); + TEST_ASSERT(0); + return; + } if (na != nb || memcmp(ia, ib, (size_t)na * sizeof(int)) != 0) { fprintf(stderr, "FAIL %s same-ids \"%s\" vs \"%s\"\n", label, a, b); failures++; @@ -96,9 +109,8 @@ static void check_same_ids(const qwen_tokenizer_t *tok, free(ia); } -static void run_cases(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, const char *label) -{ +static void +run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) { /* h e l l o -> he -> he,ll -> hell -> hello */ const int hello[] = {259}; check_ids(tok, ws, "hello", hello, 1, label); @@ -147,14 +159,14 @@ static void run_cases(const qwen_tokenizer_t *tok, /* NFC: decomposed base+combining-mark spellings must match the * precomposed codepoint (one pair per mark in the composition table). */ - check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */ - check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */ - check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */ - check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */ - check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */ - check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */ - check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */ - check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */ + check_same_ids(tok, ws, "e\xCC\x81", "\xC3\xA9", label); /* acute */ + check_same_ids(tok, ws, "a\xCC\x80", "\xC3\xA0", label); /* grave */ + check_same_ids(tok, ws, "e\xCC\x82", "\xC3\xAA", label); /* circumflex */ + check_same_ids(tok, ws, "n\xCC\x83", "\xC3\xB1", label); /* tilde */ + check_same_ids(tok, ws, "u\xCC\x88", "\xC3\xBC", label); /* diaeresis */ + check_same_ids(tok, ws, "a\xCC\x8A", "\xC3\xA5", label); /* ring */ + check_same_ids(tok, ws, "c\xCC\xA7", "\xC3\xA7", label); /* cedilla */ + check_same_ids(tok, ws, "A\xCC\x80", "\xC3\x80", label); /* upper */ check_roundtrip(tok, ws, "cafe\xCC\x81", "caf\xC3\xA9", label); /* 'x' + acute has no composition: bytes pass through untouched. */ check_roundtrip(tok, ws, "x\xCC\x81", "x\xCC\x81", label); @@ -177,30 +189,37 @@ static void run_cases(const qwen_tokenizer_t *tok, check_roundtrip(tok, ws, "\xF0\x9F\x98\x80", "\xF0\x9F\x98\x80", label); check_roundtrip(tok, ws, "tab\tand\nnl 42!", "tab\tand\nnl 42!", label); const int ellipsis[] = {'a', 0xE2, 0x80, 0xA6, 'b'}; - check_ids(tok, ws, "a\xE2\x80\xA6" "b", ellipsis, 5, label); + check_ids(tok, ws, + "a\xE2\x80\xA6" + "b", + ellipsis, 5, label); } -static void write_file(const char *dir, const char *name, const char *body) -{ +static void write_file(const char *dir, const char *name, const char *body) { char path[2048]; snprintf(path, sizeof(path), "%s/%s", dir, name); FILE *f = fopen(path, "w"); - if (!f) { perror(path); exit(2); } + if (!f) { + perror(path); + exit(2); + } fputs(body, f); fclose(f); } /* Loader rejection paths and vocab.json escape handling, in throwaway * subdirectories of the main fixture dir. */ -static void test_loader_edges(const char *root) -{ +static void test_loader_edges(const char *root) { char dir[1152], path[2048]; TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL); /* Not a JSON object. */ snprintf(dir, sizeof(dir), "%s/badjson", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "[1, 2]\n"); write_file(dir, "merges.txt", "a b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); @@ -209,7 +228,10 @@ static void test_loader_edges(const char *root) /* Missing or pair-less merges.txt is tolerated: the tokenizer loads * and encoding falls back to byte/vocab-level (no merges applied). */ snprintf(dir, sizeof(dir), "%s/nomerges", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); qwen_tokenizer_t *nom = qwen_tokenizer_load(path); @@ -225,7 +247,10 @@ static void test_loader_edges(const char *root) } snprintf(dir, sizeof(dir), "%s/emptymerges", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); write_file(dir, "merges.txt", "#version: 0.2\n\n \n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); @@ -235,7 +260,10 @@ static void test_loader_edges(const char *root) /* Escape forms in vocab keys; a malformed merge line is skipped. */ snprintf(dir, sizeof(dir), "%s/escapes", root); - if (mkdir(dir, 0755) != 0) { perror(dir); exit(2); } + if (mkdir(dir, 0755) != 0) { + perror(dir); + exit(2); + } write_file(dir, "vocab.json", "{\"a\":0, \"b\":1, \"ab\":2,\n" " \"\\u20AC\":3, \"x\\ny\":4, \"q\\\"r\":5,\n" @@ -264,12 +292,14 @@ static void test_loader_edges(const char *root) qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json"); TEST_ASSERT(rel != NULL); qwen_tokenizer_free(rel); - if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); } + if (chdir(saved_cwd) != 0) { + perror("chdir back"); + exit(2); + } } } -int main(void) -{ +int main(void) { char dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-tok-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); @@ -282,7 +312,8 @@ int main(void) snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); TEST_ASSERT(tok != NULL); - if (!tok) return 1; + if (!tok) + return 1; /* The complete byte alphabet means the integer-pair tables must exist. */ TEST_ASSERT(tok->int_merges != NULL); @@ -312,21 +343,17 @@ int main(void) { int out[8]; int n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, - &n) == 0 && n == 3); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262); /* On overflow the prefix is written and n reports the total * required count, so a caller can re-size and retry. */ n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, - &n) == -2 && n == 3); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, - &n) == -2 && n == 3); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && - n == 0); + TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); } -- cgit v1.3.1