aboutsummaryrefslogtreecommitdiff
path: root/tests/tiny_model.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 14:32:11 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit3f6853a7cb8fc766fea72611779915915e912501 (patch)
tree96dc47713376fc44c1e9d16de55444309f2abb4e /tests/tiny_model.h
parent4723631fe34e7aae4e9def3aeb0c923de2a50cf8 (diff)
downloadffwd-3f6853a7cb8fc766fea72611779915915e912501.tar.gz
ffwd-3f6853a7cb8fc766fea72611779915915e912501.zip
tests: cover the late-interaction API and the CLI binary
Diffstat (limited to 'tests/tiny_model.h')
-rw-r--r--tests/tiny_model.h114
1 files changed, 78 insertions, 36 deletions
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index 0a4e67b..8c23e2c 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -5,11 +5,14 @@
#ifndef TINY_MODEL_H
#define TINY_MODEL_H
+#include <errno.h>
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
static uint16_t tm_f32_to_bf16(float x)
{
@@ -54,47 +57,18 @@ typedef struct {
enum { TM_N_SPECS = 13 };
-/* Write config.json + model.safetensors with the given dimensions into dir.
+/* Write a safetensors file holding `specs` with tm_value() contents.
* dtype: "F32" or "BF16". Returns 0 on success. */
-static int tm_write_model_dims(const char *dir, const char *dtype,
- const tm_dims_t *d)
+static int tm_write_safetensors(const char *path, const char *dtype,
+ const tm_spec_t *specs, int n_specs)
{
- int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
- const tm_spec_t specs[TM_N_SPECS] = {
- {"embed_tokens.weight", d->vocab, d->hidden},
- {"layers.0.self_attn.q_proj.weight", q, d->hidden},
- {"layers.0.self_attn.k_proj.weight", kv, d->hidden},
- {"layers.0.self_attn.v_proj.weight", kv, d->hidden},
- {"layers.0.self_attn.o_proj.weight", d->hidden, q},
- {"layers.0.self_attn.q_norm.weight", d->head_dim, 0},
- {"layers.0.self_attn.k_norm.weight", d->head_dim, 0},
- {"layers.0.input_layernorm.weight", d->hidden, 0},
- {"layers.0.post_attention_layernorm.weight", d->hidden, 0},
- {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden},
- {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden},
- {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate},
- {"norm.weight", d->hidden, 0},
- };
-
- char path[2048];
- snprintf(path, sizeof(path), "%s/config.json", dir);
- FILE *f = fopen(path, "w");
- if (!f) return -1;
- fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
- "\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
- "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
- "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}",
- d->hidden, d->heads, d->kv_heads, d->head_dim,
- d->intermediate, d->vocab);
- fclose(f);
-
int bf16 = strcmp(dtype, "BF16") == 0;
size_t esize = bf16 ? 2 : 4;
char header[4096];
size_t hoff = 0, doff = 0;
hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "{");
- for (int t = 0; t < TM_N_SPECS; t++) {
+ for (int t = 0; t < n_specs; t++) {
const tm_spec_t *s = &specs[t];
size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1);
size_t end = doff + n * esize;
@@ -113,14 +87,13 @@ static int tm_write_model_dims(const char *dir, const char *dtype,
hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "}");
if (hoff >= sizeof(header) - 1) return -1;
- snprintf(path, sizeof(path), "%s/model.safetensors", dir);
- f = fopen(path, "wb");
+ FILE *f = fopen(path, "wb");
if (!f) return -1;
unsigned char len8[8];
for (int i = 0; i < 8; i++) len8[i] = (unsigned char)(hoff >> (8 * i));
fwrite(len8, 1, 8, f);
fwrite(header, 1, hoff, f);
- for (int t = 0; t < TM_N_SPECS; t++) {
+ for (int t = 0; t < n_specs; t++) {
const tm_spec_t *s = &specs[t];
size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1);
for (size_t i = 0; i < n; i++) {
@@ -137,6 +110,75 @@ static int tm_write_model_dims(const char *dir, const char *dtype,
return 0;
}
+/* Write config.json + model.safetensors with the given dimensions into dir.
+ * dtype: "F32" or "BF16". Returns 0 on success. */
+static int tm_write_model_dims(const char *dir, const char *dtype,
+ const tm_dims_t *d)
+{
+ int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
+ const tm_spec_t specs[TM_N_SPECS] = {
+ {"embed_tokens.weight", d->vocab, d->hidden},
+ {"layers.0.self_attn.q_proj.weight", q, d->hidden},
+ {"layers.0.self_attn.k_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.v_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.o_proj.weight", d->hidden, q},
+ {"layers.0.self_attn.q_norm.weight", d->head_dim, 0},
+ {"layers.0.self_attn.k_norm.weight", d->head_dim, 0},
+ {"layers.0.input_layernorm.weight", d->hidden, 0},
+ {"layers.0.post_attention_layernorm.weight", d->hidden, 0},
+ {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate},
+ {"norm.weight", d->hidden, 0},
+ };
+
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/config.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f) return -1;
+ fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
+ "\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
+ "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
+ "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}",
+ d->hidden, d->heads, d->kv_heads, d->head_dim,
+ d->intermediate, d->vocab);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/model.safetensors", dir);
+ return tm_write_safetensors(path, dtype, specs, TM_N_SPECS);
+}
+
+/* Add the 1_Dense per-token projection head that turns a base model dir into
+ * a late-interaction snapshot. weight_name is "linear.weight" for a valid
+ * head; pass another name to synthesize a malformed head. */
+static inline int tm_write_late_projection_named(const char *dir,
+ const char *dtype,
+ int token_dim, int hidden,
+ const char *weight_name)
+{
+ char path[2048];
+ snprintf(path, sizeof(path), "%s/1_Dense", dir);
+ if (mkdir(path, 0755) != 0 && errno != EEXIST) return -1;
+
+ snprintf(path, sizeof(path), "%s/1_Dense/config.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f) return -1;
+ fprintf(f, "{\"in_features\":%d,\"out_features\":%d,\"bias\":false}",
+ hidden, token_dim);
+ fclose(f);
+
+ const tm_spec_t spec = {weight_name, token_dim, hidden};
+ snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", dir);
+ return tm_write_safetensors(path, dtype, &spec, 1);
+}
+
+static inline int tm_write_late_projection(const char *dir, const char *dtype,
+ int token_dim, int hidden)
+{
+ return tm_write_late_projection_named(dir, dtype, token_dim, hidden,
+ "linear.weight");
+}
+
/* The historical default fixture: 4-dim hidden, 16-token vocab. */
static inline int tm_write_model(const char *dir, const char *dtype)
{