aboutsummaryrefslogtreecommitdiff
path: root/tests/tok_fixture.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
commitc4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch)
tree81ce5675ce886aad0309a41f4d6a696826e0a061 /tests/tok_fixture.h
parentb377ba8656bfb807ed8dda6c9f26942b50ce626d (diff)
downloadffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz
ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip
chore: run clang-format
Diffstat (limited to 'tests/tok_fixture.h')
-rw-r--r--tests/tok_fixture.h36
1 files changed, 18 insertions, 18 deletions
diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h
index 7890927..24e1563 100644
--- a/tests/tok_fixture.h
+++ b/tests/tok_fixture.h
@@ -10,15 +10,13 @@
#include <string.h>
/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */
-static int tf_byte_to_cp(int b)
-{
+static int tf_byte_to_cp(int b) {
static int table[256];
static int init = 0;
if (!init) {
int n = 0;
for (int i = 0; i < 256; i++) {
- int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) ||
- (i >= 174 && i <= 255);
+ int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) || (i >= 174 && i <= 255);
table[i] = normal ? i : 256 + n++;
}
init = 1;
@@ -26,9 +24,12 @@ static int tf_byte_to_cp(int b)
return table[b];
}
-static int tf_cp_to_utf8(int cp, char *out)
-{
- if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; }
+static int tf_cp_to_utf8(int cp, char *out) {
+ if (cp < 0x80) {
+ out[0] = (char)cp;
+ out[1] = '\0';
+ return 1;
+ }
out[0] = (char)(0xC0 | (cp >> 6));
out[1] = (char)(0x80 | (cp & 0x3F));
out[2] = '\0';
@@ -36,11 +37,11 @@ static int tf_cp_to_utf8(int cp, char *out)
}
/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */
-static const char *tf_json_escape(const char *s, char *buf, size_t cap)
-{
+static const char *tf_json_escape(const char *s, char *buf, size_t cap) {
size_t o = 0;
for (; *s && o + 2 < cap; s++) {
- if (*s == '"' || *s == '\\') buf[o++] = '\\';
+ if (*s == '"' || *s == '\\')
+ buf[o++] = '\\';
buf[o++] = *s;
}
buf[o] = '\0';
@@ -52,8 +53,7 @@ static const char *TF_MERGED[] = {
"he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor",
};
static const char *TF_MERGES[] = {
- "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d",
- "\xC4\xA0w or",
+ "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d", "\xC4\xA0w or",
};
enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) };
/* Special tokens go last so contextual and late-interaction ids resolve
@@ -65,17 +65,16 @@ enum { TF_DOCUMENT_ID = TF_QUERY_ID + 1 };
enum { TF_VOCAB_SIZE = TF_DOCUMENT_ID + 1 };
/* Write vocab.json + merges.txt into dir. Returns 0 on success. */
-static int tf_write_vocab(const char *dir)
-{
+static int tf_write_vocab(const char *dir) {
char path[2048], tok[4], esc[16];
snprintf(path, sizeof(path), "%s/vocab.json", dir);
FILE *f = fopen(path, "w");
- if (!f) return -1;
+ if (!f)
+ return -1;
fputs("{", f);
for (int b = 0; b < 256; b++) {
tf_cp_to_utf8(tf_byte_to_cp(b), tok);
- fprintf(f, "%s\"%s\": %d", b ? ", " : "",
- tf_json_escape(tok, esc, sizeof(esc)), b);
+ fprintf(f, "%s\"%s\": %d", b ? ", " : "", tf_json_escape(tok, esc, sizeof(esc)), b);
}
for (int i = 0; i < TF_N_MERGED; i++)
fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i);
@@ -88,7 +87,8 @@ static int tf_write_vocab(const char *dir)
snprintf(path, sizeof(path), "%s/merges.txt", dir);
f = fopen(path, "w");
- if (!f) return -1;
+ if (!f)
+ return -1;
for (size_t i = 0; i < sizeof(TF_MERGES) / sizeof(TF_MERGES[0]); i++)
fprintf(f, "%s\n", TF_MERGES[i]);
fclose(f);