aboutsummaryrefslogtreecommitdiff
path: root/tests/tok_fixture.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 13:04:50 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3 (patch)
tree2e441978cc9badf091976aa53bc64297432e862a /tests/tok_fixture.h
parent3947e97fbf2a0b79c500ce925bcc9fe15f89899e (diff)
downloadffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.tar.gz
ffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.zip
server: resolve contextual separator from vocab, test contextual path
Diffstat (limited to 'tests/tok_fixture.h')
-rw-r--r--tests/tok_fixture.h5
1 files changed, 4 insertions, 1 deletions
diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h
index 0ac81c8..ce7c3a3 100644
--- a/tests/tok_fixture.h
+++ b/tests/tok_fixture.h
@@ -56,7 +56,9 @@ static const char *TF_MERGES[] = {
"\xC4\xA0w or",
};
enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) };
-enum { TF_VOCAB_SIZE = 256 + TF_N_MERGED };
+/* The special token goes last so contextual-separator ids resolve in-range. */
+enum { TF_EOT_ID = 256 + TF_N_MERGED };
+enum { TF_VOCAB_SIZE = TF_EOT_ID + 1 };
/* Write vocab.json + merges.txt into dir. Returns 0 on success. */
static int tf_write_vocab(const char *dir)
@@ -73,6 +75,7 @@ static int tf_write_vocab(const char *dir)
}
for (int i = 0; i < TF_N_MERGED; i++)
fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i);
+ fprintf(f, ", \"<|endoftext|>\": %d", TF_EOT_ID);
fputs("}", f);
fclose(f);