diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 12:51:10 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 3947e97fbf2a0b79c500ce925bcc9fe15f89899e (patch) | |
| tree | 9e1bb5894c42afd2af7f7639377c7ba49221966b /tests/test_tokenizer.c | |
| parent | 2c9592042faf7229dc801f6eaee34636267aec4c (diff) | |
| download | ffwd-3947e97fbf2a0b79c500ce925bcc9fe15f89899e.tar.gz ffwd-3947e97fbf2a0b79c500ce925bcc9fe15f89899e.zip | |
tests: drive the HTTP request path in-process in test_server
Diffstat (limited to 'tests/test_tokenizer.c')
| -rw-r--r-- | tests/test_tokenizer.c | 77 |
1 files changed, 2 insertions, 75 deletions
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index 8c20308..1de61b6 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -5,6 +5,7 @@ * string-based fallback) produces identical ids. Runs via `make test`. */ #include "qwen_tokenizer.h" +#include "tok_fixture.h" #include <stdio.h> #include <stdlib.h> @@ -21,80 +22,6 @@ int qwen_verbose = 0; } \ } while (0) -/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */ -static int byte_to_cp(int b) -{ - static int table[256]; - static int init = 0; - if (!init) { - int n = 0; - for (int i = 0; i < 256; i++) { - int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) || - (i >= 174 && i <= 255); - table[i] = normal ? i : 256 + n++; - } - init = 1; - } - return table[b]; -} - -static int cp_to_utf8(int cp, char *out) -{ - if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; } - out[0] = (char)(0xC0 | (cp >> 6)); - out[1] = (char)(0x80 | (cp & 0x3F)); - out[2] = '\0'; - return 2; -} - -/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */ -static const char *json_escape(const char *s, char *buf, size_t cap) -{ - size_t o = 0; - for (; *s && o + 2 < cap; s++) { - if (*s == '"' || *s == '\\') buf[o++] = '\\'; - buf[o++] = *s; - } - buf[o] = '\0'; - return buf; -} - -/* Merged tokens (ids 256+) and the merge list that produces them. */ -static const char *MERGED[] = { - "he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor", -}; -static const char *MERGES[] = { - "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d", - "\xC4\xA0w or", -}; -enum { N_MERGED = sizeof(MERGED) / sizeof(MERGED[0]) }; - -static int write_fixture(const char *dir) -{ - char path[2048], tok[4], esc[16]; - snprintf(path, sizeof(path), "%s/vocab.json", dir); - FILE *f = fopen(path, "w"); - if (!f) return -1; - fputs("{", f); - for (int b = 0; b < 256; b++) { - cp_to_utf8(byte_to_cp(b), tok); - fprintf(f, "%s\"%s\": %d", b ? ", " : "", - json_escape(tok, esc, sizeof(esc)), b); - } - for (int i = 0; i < N_MERGED; i++) - fprintf(f, ", \"%s\": %d", MERGED[i], 256 + i); - fputs("}", f); - fclose(f); - - snprintf(path, sizeof(path), "%s/merges.txt", dir); - f = fopen(path, "w"); - if (!f) return -1; - for (size_t i = 0; i < sizeof(MERGES) / sizeof(MERGES[0]); i++) - fprintf(f, "%s\n", MERGES[i]); - fclose(f); - return 0; -} - static void check_ids(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, const int *want, int n_want, const char *label) @@ -177,7 +104,7 @@ int main(void) char dir[1024]; snprintf(dir, sizeof(dir), "%s/pplx-tok-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - if (!mkdtemp(dir) || write_fixture(dir) != 0) { + if (!mkdtemp(dir) || tf_write_vocab(dir) != 0) { fprintf(stderr, "fixture creation failed\n"); return 2; } |
