aboutsummaryrefslogtreecommitdiff
path: root/tests/tok_fixture.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 12:51:10 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit3947e97fbf2a0b79c500ce925bcc9fe15f89899e (patch)
tree9e1bb5894c42afd2af7f7639377c7ba49221966b /tests/tok_fixture.h
parent2c9592042faf7229dc801f6eaee34636267aec4c (diff)
downloadffwd-3947e97fbf2a0b79c500ce925bcc9fe15f89899e.tar.gz
ffwd-3947e97fbf2a0b79c500ce925bcc9fe15f89899e.zip
tests: drive the HTTP request path in-process in test_server
Diffstat (limited to 'tests/tok_fixture.h')
-rw-r--r--tests/tok_fixture.h88
1 files changed, 88 insertions, 0 deletions
diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h
new file mode 100644
index 0000000..0ac81c8
--- /dev/null
+++ b/tests/tok_fixture.h
@@ -0,0 +1,88 @@
+/* tests/tok_fixture.h - synthesize a complete GPT-2-style byte-level BPE
+ * tokenizer fixture into a directory: vocab.json with all 256 byte tokens
+ * (ids 0-255, so any input tokenizes) plus a few merged tokens (ids 256+),
+ * and the matching merges.txt. Shared by the tokenizer and server tests. */
+
+#ifndef TOK_FIXTURE_H
+#define TOK_FIXTURE_H
+
+#include <stdio.h>
+#include <string.h>
+
+/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */
+static int tf_byte_to_cp(int b)
+{
+ static int table[256];
+ static int init = 0;
+ if (!init) {
+ int n = 0;
+ for (int i = 0; i < 256; i++) {
+ int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) ||
+ (i >= 174 && i <= 255);
+ table[i] = normal ? i : 256 + n++;
+ }
+ init = 1;
+ }
+ return table[b];
+}
+
+static int tf_cp_to_utf8(int cp, char *out)
+{
+ if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; }
+ out[0] = (char)(0xC0 | (cp >> 6));
+ out[1] = (char)(0x80 | (cp & 0x3F));
+ out[2] = '\0';
+ return 2;
+}
+
+/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */
+static const char *tf_json_escape(const char *s, char *buf, size_t cap)
+{
+ size_t o = 0;
+ for (; *s && o + 2 < cap; s++) {
+ if (*s == '"' || *s == '\\') buf[o++] = '\\';
+ buf[o++] = *s;
+ }
+ buf[o] = '\0';
+ return buf;
+}
+
+/* Merged tokens (ids 256+) and the merge list that produces them. */
+static const char *TF_MERGED[] = {
+ "he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor",
+};
+static const char *TF_MERGES[] = {
+ "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d",
+ "\xC4\xA0w or",
+};
+enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) };
+enum { TF_VOCAB_SIZE = 256 + TF_N_MERGED };
+
+/* Write vocab.json + merges.txt into dir. Returns 0 on success. */
+static int tf_write_vocab(const char *dir)
+{
+ char path[2048], tok[4], esc[16];
+ snprintf(path, sizeof(path), "%s/vocab.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f) return -1;
+ fputs("{", f);
+ for (int b = 0; b < 256; b++) {
+ tf_cp_to_utf8(tf_byte_to_cp(b), tok);
+ fprintf(f, "%s\"%s\": %d", b ? ", " : "",
+ tf_json_escape(tok, esc, sizeof(esc)), b);
+ }
+ for (int i = 0; i < TF_N_MERGED; i++)
+ fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i);
+ fputs("}", f);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/merges.txt", dir);
+ f = fopen(path, "w");
+ if (!f) return -1;
+ for (size_t i = 0; i < sizeof(TF_MERGES) / sizeof(TF_MERGES[0]); i++)
+ fprintf(f, "%s\n", TF_MERGES[i]);
+ fclose(f);
+ return 0;
+}
+
+#endif /* TOK_FIXTURE_H */