diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 14:32:11 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | 3f6853a7cb8fc766fea72611779915915e912501 (patch) | |
| tree | 96dc47713376fc44c1e9d16de55444309f2abb4e /tests/tiny_model.h | |
| parent | 4723631fe34e7aae4e9def3aeb0c923de2a50cf8 (diff) | |
| download | ffwd-3f6853a7cb8fc766fea72611779915915e912501.tar.gz ffwd-3f6853a7cb8fc766fea72611779915915e912501.zip | |
tests: cover the late-interaction API and the CLI binary
Diffstat (limited to 'tests/tiny_model.h')
| -rw-r--r-- | tests/tiny_model.h | 114 |
1 files changed, 78 insertions, 36 deletions
diff --git a/tests/tiny_model.h b/tests/tiny_model.h index 0a4e67b..8c23e2c 100644 --- a/tests/tiny_model.h +++ b/tests/tiny_model.h @@ -5,11 +5,14 @@ #ifndef TINY_MODEL_H #define TINY_MODEL_H +#include <errno.h> #include <math.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <sys/stat.h> +#include <sys/types.h> static uint16_t tm_f32_to_bf16(float x) { @@ -54,47 +57,18 @@ typedef struct { enum { TM_N_SPECS = 13 }; -/* Write config.json + model.safetensors with the given dimensions into dir. +/* Write a safetensors file holding `specs` with tm_value() contents. * dtype: "F32" or "BF16". Returns 0 on success. */ -static int tm_write_model_dims(const char *dir, const char *dtype, - const tm_dims_t *d) +static int tm_write_safetensors(const char *path, const char *dtype, + const tm_spec_t *specs, int n_specs) { - int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim; - const tm_spec_t specs[TM_N_SPECS] = { - {"embed_tokens.weight", d->vocab, d->hidden}, - {"layers.0.self_attn.q_proj.weight", q, d->hidden}, - {"layers.0.self_attn.k_proj.weight", kv, d->hidden}, - {"layers.0.self_attn.v_proj.weight", kv, d->hidden}, - {"layers.0.self_attn.o_proj.weight", d->hidden, q}, - {"layers.0.self_attn.q_norm.weight", d->head_dim, 0}, - {"layers.0.self_attn.k_norm.weight", d->head_dim, 0}, - {"layers.0.input_layernorm.weight", d->hidden, 0}, - {"layers.0.post_attention_layernorm.weight", d->hidden, 0}, - {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden}, - {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden}, - {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate}, - {"norm.weight", d->hidden, 0}, - }; - - char path[2048]; - snprintf(path, sizeof(path), "%s/config.json", dir); - FILE *f = fopen(path, "w"); - if (!f) return -1; - fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1," - "\"num_attention_heads\":%d,\"num_key_value_heads\":%d," - "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d," - "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}", - d->hidden, d->heads, d->kv_heads, d->head_dim, - d->intermediate, d->vocab); - fclose(f); - int bf16 = strcmp(dtype, "BF16") == 0; size_t esize = bf16 ? 2 : 4; char header[4096]; size_t hoff = 0, doff = 0; hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "{"); - for (int t = 0; t < TM_N_SPECS; t++) { + for (int t = 0; t < n_specs; t++) { const tm_spec_t *s = &specs[t]; size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1); size_t end = doff + n * esize; @@ -113,14 +87,13 @@ static int tm_write_model_dims(const char *dir, const char *dtype, hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "}"); if (hoff >= sizeof(header) - 1) return -1; - snprintf(path, sizeof(path), "%s/model.safetensors", dir); - f = fopen(path, "wb"); + FILE *f = fopen(path, "wb"); if (!f) return -1; unsigned char len8[8]; for (int i = 0; i < 8; i++) len8[i] = (unsigned char)(hoff >> (8 * i)); fwrite(len8, 1, 8, f); fwrite(header, 1, hoff, f); - for (int t = 0; t < TM_N_SPECS; t++) { + for (int t = 0; t < n_specs; t++) { const tm_spec_t *s = &specs[t]; size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1); for (size_t i = 0; i < n; i++) { @@ -137,6 +110,75 @@ static int tm_write_model_dims(const char *dir, const char *dtype, return 0; } +/* Write config.json + model.safetensors with the given dimensions into dir. + * dtype: "F32" or "BF16". Returns 0 on success. */ +static int tm_write_model_dims(const char *dir, const char *dtype, + const tm_dims_t *d) +{ + int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim; + const tm_spec_t specs[TM_N_SPECS] = { + {"embed_tokens.weight", d->vocab, d->hidden}, + {"layers.0.self_attn.q_proj.weight", q, d->hidden}, + {"layers.0.self_attn.k_proj.weight", kv, d->hidden}, + {"layers.0.self_attn.v_proj.weight", kv, d->hidden}, + {"layers.0.self_attn.o_proj.weight", d->hidden, q}, + {"layers.0.self_attn.q_norm.weight", d->head_dim, 0}, + {"layers.0.self_attn.k_norm.weight", d->head_dim, 0}, + {"layers.0.input_layernorm.weight", d->hidden, 0}, + {"layers.0.post_attention_layernorm.weight", d->hidden, 0}, + {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden}, + {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden}, + {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate}, + {"norm.weight", d->hidden, 0}, + }; + + char path[2048]; + snprintf(path, sizeof(path), "%s/config.json", dir); + FILE *f = fopen(path, "w"); + if (!f) return -1; + fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1," + "\"num_attention_heads\":%d,\"num_key_value_heads\":%d," + "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d," + "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}", + d->hidden, d->heads, d->kv_heads, d->head_dim, + d->intermediate, d->vocab); + fclose(f); + + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + return tm_write_safetensors(path, dtype, specs, TM_N_SPECS); +} + +/* Add the 1_Dense per-token projection head that turns a base model dir into + * a late-interaction snapshot. weight_name is "linear.weight" for a valid + * head; pass another name to synthesize a malformed head. */ +static inline int tm_write_late_projection_named(const char *dir, + const char *dtype, + int token_dim, int hidden, + const char *weight_name) +{ + char path[2048]; + snprintf(path, sizeof(path), "%s/1_Dense", dir); + if (mkdir(path, 0755) != 0 && errno != EEXIST) return -1; + + snprintf(path, sizeof(path), "%s/1_Dense/config.json", dir); + FILE *f = fopen(path, "w"); + if (!f) return -1; + fprintf(f, "{\"in_features\":%d,\"out_features\":%d,\"bias\":false}", + hidden, token_dim); + fclose(f); + + const tm_spec_t spec = {weight_name, token_dim, hidden}; + snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", dir); + return tm_write_safetensors(path, dtype, &spec, 1); +} + +static inline int tm_write_late_projection(const char *dir, const char *dtype, + int token_dim, int hidden) +{ + return tm_write_late_projection_named(dir, dtype, token_dim, hidden, + "linear.weight"); +} + /* The historical default fixture: 4-dim hidden, 16-token vocab. */ static inline int tm_write_model(const char *dir, const char *dtype) { |
