diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /tests/test_tokenizer.c | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'tests/test_tokenizer.c')
| -rw-r--r-- | tests/test_tokenizer.c | 122 |
1 files changed, 61 insertions, 61 deletions
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index 5920977..3ef4280 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -4,7 +4,7 @@ * integer-pair fast path, and that dropping the integer tables (forcing the * string-based fallback) produces identical ids. Runs via `make test`. */ -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tok_fixture.h" #include <stdio.h> @@ -14,7 +14,7 @@ #include <unistd.h> static int failures = 0; -int qwen_verbose = 0; +int embed_verbose = 0; #define TEST_ASSERT(cond) \ do { \ @@ -24,14 +24,14 @@ int qwen_verbose = 0; } \ } while (0) -static void check_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_ids(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, const int *want, int n_want, const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL); if (!ids) return; @@ -52,19 +52,19 @@ static void check_ids(const qwen_tokenizer_t *tok, /* Concatenated decode of an encoding must reproduce the (NFC-normalized) * input - byte-level BPE is lossless. */ -static void check_roundtrip(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_roundtrip(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, const char *want, const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL && n > 0); if (!ids) return; size_t cap = 1; for (int i = 0; i < n; i++) - cap += strlen(qwen_tokenizer_decode(tok, ids[i])); + cap += strlen(embed_tokenizer_decode(tok, ids[i])); char *got = (char *)malloc(cap); if (!got) { free(ids); @@ -73,7 +73,7 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, } got[0] = '\0'; for (int i = 0; i < n; i++) - strcat(got, qwen_tokenizer_decode(tok, ids[i])); + strcat(got, embed_tokenizer_decode(tok, ids[i])); if (strcmp(got, want) != 0) { fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got, want); @@ -84,18 +84,18 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, } /* Two spellings of the same text must encode identically. */ -static void check_same_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_same_ids(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *a, const char *b, const char *label) { int na = 0, nb = 0; - int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na); + int *ia = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, a, &na); if (!ia) { TEST_ASSERT(0); return; } - int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb); + int *ib = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, b, &nb); if (!ib) { free(ia); TEST_ASSERT(0); @@ -110,7 +110,7 @@ static void check_same_ids(const qwen_tokenizer_t *tok, } static void -run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) { +run_cases(const embed_tokenizer_t *tok, embed_tokenizer_workspace_t *ws, const char *label) { /* h e l l o -> he -> he,ll -> hell -> hello */ const int hello[] = {259}; check_ids(tok, ws, "hello", hello, 1, label); @@ -212,7 +212,7 @@ static void write_file(const char *dir, const char *name, const char *body) { static void test_loader_edges(const char *root) { char dir[1152], path[2048]; - TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL); + TEST_ASSERT(embed_tokenizer_load("/nonexistent/vocab.json") == NULL); /* Not a JSON object. */ snprintf(dir, sizeof(dir), "%s/badjson", root); @@ -223,7 +223,7 @@ static void test_loader_edges(const char *root) { write_file(dir, "vocab.json", "[1, 2]\n"); write_file(dir, "merges.txt", "a b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - TEST_ASSERT(qwen_tokenizer_load(path) == NULL); + TEST_ASSERT(embed_tokenizer_load(path) == NULL); /* Missing or pair-less merges.txt is tolerated: the tokenizer loads * and encoding falls back to byte/vocab-level (no merges applied). */ @@ -234,16 +234,16 @@ static void test_loader_edges(const char *root) { } write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *nom = qwen_tokenizer_load(path); + embed_tokenizer_t *nom = embed_tokenizer_load(path); TEST_ASSERT(nom != NULL); if (nom) { - qwen_tokenizer_workspace_t *nws = qwen_tokenizer_workspace_new(); + embed_tokenizer_workspace_t *nws = embed_tokenizer_workspace_new(); int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace(nom, nws, "ab", &n); + int *ids = embed_tokenizer_encode_with_workspace(nom, nws, "ab", &n); TEST_ASSERT(ids && n == 2 && ids[0] == 0 && ids[1] == 1); free(ids); - qwen_tokenizer_workspace_free(nws); - qwen_tokenizer_free(nom); + embed_tokenizer_workspace_free(nws); + embed_tokenizer_free(nom); } snprintf(dir, sizeof(dir), "%s/emptymerges", root); @@ -254,9 +254,9 @@ static void test_loader_edges(const char *root) { write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); write_file(dir, "merges.txt", "#version: 0.2\n\n \n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *emp = qwen_tokenizer_load(path); + embed_tokenizer_t *emp = embed_tokenizer_load(path); TEST_ASSERT(emp != NULL); - qwen_tokenizer_free(emp); + embed_tokenizer_free(emp); /* Escape forms in vocab keys; a malformed merge line is skipped. */ snprintf(dir, sizeof(dir), "%s/escapes", root); @@ -270,28 +270,28 @@ static void test_loader_edges(const char *root) { " \"s\\\\t\":6, \"p\\/q\":7, \"u\\tv\":8, \"\\u0041Z\":9}\n"); write_file(dir, "merges.txt", "#version: 0.2\nmalformed-line\na b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *tok = qwen_tokenizer_load(path); + embed_tokenizer_t *tok = embed_tokenizer_load(path); TEST_ASSERT(tok != NULL); if (tok) { - TEST_ASSERT(qwen_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "x\ny") == 4); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "q\"r") == 5); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "s\\t") == 6); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "p/q") == 7); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "u\tv") == 8); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "AZ") == 9); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 2), "ab") == 0); + TEST_ASSERT(embed_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3); + TEST_ASSERT(embed_tokenizer_token_id(tok, "x\ny") == 4); + TEST_ASSERT(embed_tokenizer_token_id(tok, "q\"r") == 5); + TEST_ASSERT(embed_tokenizer_token_id(tok, "s\\t") == 6); + TEST_ASSERT(embed_tokenizer_token_id(tok, "p/q") == 7); + TEST_ASSERT(embed_tokenizer_token_id(tok, "u\tv") == 8); + TEST_ASSERT(embed_tokenizer_token_id(tok, "AZ") == 9); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 2), "ab") == 0); /* Keys outside the GPT-2 byte alphabet decode to replacement. */ - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 3), "?") == 0); - qwen_tokenizer_free(tok); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 3), "?") == 0); + embed_tokenizer_free(tok); } /* A bare filename exercises the no-directory merges-path branch. */ char saved_cwd[1024]; if (getcwd(saved_cwd, sizeof(saved_cwd)) && chdir(dir) == 0) { - qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json"); + embed_tokenizer_t *rel = embed_tokenizer_load("vocab.json"); TEST_ASSERT(rel != NULL); - qwen_tokenizer_free(rel); + embed_tokenizer_free(rel); if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); @@ -310,7 +310,7 @@ int main(void) { char vocab_path[2048]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path); TEST_ASSERT(tok != NULL); if (!tok) return 1; @@ -319,48 +319,48 @@ int main(void) { TEST_ASSERT(tok->int_merges != NULL); TEST_ASSERT(tok->cp_to_id != NULL); - qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new(); + embed_tokenizer_workspace_t *ws = embed_tokenizer_workspace_new(); TEST_ASSERT(ws != NULL); run_cases(tok, ws, "fast"); /* decode: byte tokens, merged tokens, and out-of-range ids. */ - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 'a'), "a") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 259), "hello") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, -1), "") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(NULL, 0), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 'a'), "a") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 259), "hello") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, -1), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(NULL, 0), "") == 0); /* token_id lookups. */ - TEST_ASSERT(qwen_tokenizer_token_id(tok, "hello") == 259); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "no-such-token") == -1); - TEST_ASSERT(qwen_tokenizer_token_id(NULL, "a") == -1); - TEST_ASSERT(qwen_tokenizer_token_id(tok, NULL) == -1); + TEST_ASSERT(embed_tokenizer_token_id(tok, "hello") == 259); + TEST_ASSERT(embed_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID); + TEST_ASSERT(embed_tokenizer_token_id(tok, "no-such-token") == -1); + TEST_ASSERT(embed_tokenizer_token_id(NULL, "a") == -1); + TEST_ASSERT(embed_tokenizer_token_id(tok, NULL) == -1); /* encode_into: exact fit, truncation (-2 plus the untruncated prefix), * zero capacity, empty text, and argument guards. */ { int out[8]; int n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262); /* On overflow the prefix is written and n reports the total * required count, so a caller can re-size and retry. */ n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); - TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); + TEST_ASSERT(embed_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); } - TEST_ASSERT(qwen_tokenizer_encode(tok, "", NULL) == NULL); - TEST_ASSERT(qwen_tokenizer_encode(NULL, "x", NULL) == NULL); - qwen_tokenizer_workspace_free(NULL); - qwen_tokenizer_free(NULL); + TEST_ASSERT(embed_tokenizer_encode(tok, "", NULL) == NULL); + TEST_ASSERT(embed_tokenizer_encode(NULL, "x", NULL) == NULL); + embed_tokenizer_workspace_free(NULL); + embed_tokenizer_free(NULL); /* Drop the integer tables: the string-based fallback must produce the * exact same ids. */ @@ -371,8 +371,8 @@ int main(void) { tok->cp_to_id = NULL; run_cases(tok, ws, "slow"); - qwen_tokenizer_workspace_free(ws); - qwen_tokenizer_free(tok); + embed_tokenizer_workspace_free(ws); + embed_tokenizer_free(tok); test_loader_edges(dir); |
