aboutsummaryrefslogtreecommitdiff
path: root/tests/test_xlm_roberta.c
blob: 82e41598ff09fbc60ceab8a7d6e641133cdca661 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
/* Hermetic XLM-R/RoBERTa bridge checks. */

#include "internal.h"
#include "tiny_model.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>

static int write_fixture(const char *dir) {
    const int hidden = 4, heads = 2, inter = 8, vocab = 16, max_pos = 8;
    char path[2048];
    snprintf(path, sizeof(path), "%s/config.json", dir);
    FILE *f = fopen(path, "w");
    if (!f)
        return -1;
    fprintf(f,
            "{\"model_type\":\"xlm-roberta\",\"hidden_size\":%d,"
            "\"num_hidden_layers\":1,\"num_attention_heads\":%d,"
            "\"intermediate_size\":%d,\"vocab_size\":%d,"
            "\"max_position_embeddings\":%d,\"layer_norm_eps\":1e-5,"
            "\"pad_token_id\":1,\"type_vocab_size\":1}",
            hidden, heads, inter, vocab, max_pos);
    fclose(f);

    const tm_spec_t specs[] = {
        {"embeddings.word_embeddings.weight", vocab, hidden},
        {"embeddings.position_embeddings.weight", max_pos, hidden},
        {"embeddings.token_type_embeddings.weight", 1, hidden},
        {"embeddings.LayerNorm.weight", hidden, 0},
        {"embeddings.LayerNorm.bias", hidden, 0},
        {"encoder.layer.0.attention.self.query.weight", hidden, hidden},
        {"encoder.layer.0.attention.self.query.bias", hidden, 0},
        {"encoder.layer.0.attention.self.key.weight", hidden, hidden},
        {"encoder.layer.0.attention.self.key.bias", hidden, 0},
        {"encoder.layer.0.attention.self.value.weight", hidden, hidden},
        {"encoder.layer.0.attention.self.value.bias", hidden, 0},
        {"encoder.layer.0.attention.output.dense.weight", hidden, hidden},
        {"encoder.layer.0.attention.output.dense.bias", hidden, 0},
        {"encoder.layer.0.attention.output.LayerNorm.weight", hidden, 0},
        {"encoder.layer.0.attention.output.LayerNorm.bias", hidden, 0},
        {"encoder.layer.0.intermediate.dense.weight", inter, hidden},
        {"encoder.layer.0.intermediate.dense.bias", inter, 0},
        {"encoder.layer.0.output.dense.weight", hidden, inter},
        {"encoder.layer.0.output.dense.bias", hidden, 0},
        {"encoder.layer.0.output.LayerNorm.weight", hidden, 0},
        {"encoder.layer.0.output.LayerNorm.bias", hidden, 0},
    };
    snprintf(path, sizeof(path), "%s/model.safetensors", dir);
    return tm_write_safetensors_with_prefix(path, "F32", specs,
                                            (int)(sizeof(specs) / sizeof(specs[0])), "roberta.");
}

int main(void) {
    const char *tmp = getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp";
    char dir[1024];
    snprintf(dir, sizeof(dir), "%s/ffwd-xlm-roberta-test-XXXXXX", tmp);
    if (!mkdtemp(dir) || write_fixture(dir) != 0) {
        fprintf(stderr, "fixture creation failed\n");
        return 2;
    }

    ffwd_model_t *model = ffwd_model_load(dir);
    ffwd_workspace_t *ws = ffwd_workspace_new(model);
    const ffwd_config_t *cfg = ffwd_model_config(model);
    if (!model || !ws || !cfg) {
        fprintf(stderr, "model setup failed\n");
        return 1;
    }
    if (cfg->family != FFWD_FAMILY_BERT || cfg->position_id_offset != 2 ||
        cfg->type_vocab_size != 1 || cfg->pooling_mode != FFWD_POOL_MEAN ||
        cfg->attention_mode != FFWD_ATTENTION_BIDIRECTIONAL) {
        fprintf(stderr, "XLM-R config bridge wrong\n");
        return 1;
    }

    float out[4];
    int ids_ok[] = {0, 4, 5, 6, 7, 2};
    if (ffwd_model_encode_into(model, ws, ids_ok, 6, out) != 0) {
        fprintf(stderr, "XLM-R forward failed at max valid position\n");
        return 1;
    }
    int ids_too_long[] = {0, 4, 5, 6, 7, 8, 2};
    if (ffwd_model_encode_into(model, ws, ids_too_long, 7, out) == 0) {
        fprintf(stderr, "XLM-R position overflow was accepted\n");
        return 1;
    }

    ffwd_workspace_free(ws);
    ffwd_model_free(model);
    puts("ok: XLM-R config, roberta prefix, type_vocab_size=1, position offset");
    return 0;
}