aboutsummaryrefslogtreecommitdiff
path: root/tests/tok_fixture.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-13 02:25:35 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:10 +0200
commit81b6e93083d950ba63ee677ceb33b432cb1bf2cd (patch)
tree3b773742f2e796f99fb96f40db2fa8d83f6a77c1 /tests/tok_fixture.h
parent13c56b2218724ebcd2ef97bc386bc60930cf1ff0 (diff)
downloadffwd-81b6e93083d950ba63ee677ceb33b432cb1bf2cd.tar.gz
ffwd-81b6e93083d950ba63ee677ceb33b432cb1bf2cd.zip
server: add late-interaction rerank API
Diffstat (limited to 'tests/tok_fixture.h')
-rw-r--r--tests/tok_fixture.h11
1 files changed, 9 insertions, 2 deletions
diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h
index ce7c3a3..7890927 100644
--- a/tests/tok_fixture.h
+++ b/tests/tok_fixture.h
@@ -56,9 +56,13 @@ static const char *TF_MERGES[] = {
"\xC4\xA0w or",
};
enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) };
-/* The special token goes last so contextual-separator ids resolve in-range. */
+/* Special tokens go last so contextual and late-interaction ids resolve
+ * in-range for tiny server models. */
enum { TF_EOT_ID = 256 + TF_N_MERGED };
-enum { TF_VOCAB_SIZE = TF_EOT_ID + 1 };
+enum { TF_MASK_ID = TF_EOT_ID + 1 };
+enum { TF_QUERY_ID = TF_MASK_ID + 1 };
+enum { TF_DOCUMENT_ID = TF_QUERY_ID + 1 };
+enum { TF_VOCAB_SIZE = TF_DOCUMENT_ID + 1 };
/* Write vocab.json + merges.txt into dir. Returns 0 on success. */
static int tf_write_vocab(const char *dir)
@@ -76,6 +80,9 @@ static int tf_write_vocab(const char *dir)
for (int i = 0; i < TF_N_MERGED; i++)
fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i);
fprintf(f, ", \"<|endoftext|>\": %d", TF_EOT_ID);
+ fprintf(f, ", \"[MASK]\": %d", TF_MASK_ID);
+ fprintf(f, ", \"[Q]\": %d", TF_QUERY_ID);
+ fprintf(f, ", \"[D]\": %d", TF_DOCUMENT_ID);
fputs("}", f);
fclose(f);