aboutsummaryrefslogtreecommitdiff
path: root/tools/test_safetensors.c
blob: 5a00c5499df8ff265907bd3b93e7d50d5db0adcf (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#define _POSIX_C_SOURCE 200809L
#include "safetensors.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define CHECK(condition) do {                                               \
    if (!(condition)) {                                                     \
        fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition);       \
        exit(1);                                                            \
    }                                                                       \
} while (0)

static safetensors_file_t *read_header(const char *header) {
    char path[] = "build/safetensors-test-XXXXXX";
    int fd = mkstemp(path);
    CHECK(fd >= 0);
    FILE *file = fdopen(fd, "wb");
    CHECK(file);
    size_t length = strlen(header);
    uint64_t padded = (length + 7) & ~(uint64_t)7;
    const float values[] = {1.25f, -2.5f};
    CHECK(fwrite(&padded, sizeof(padded), 1, file) == 1);
    CHECK(fwrite(header, 1, length, file) == length);
    for (size_t i = length; i < padded; i++)
        CHECK(fputc(' ', file) != EOF);
    CHECK(fwrite(values, sizeof(values), 1, file) == 1);
    CHECK(fclose(file) == 0);
    safetensors_file_t *sf = safetensors_open(path);
    CHECK(unlink(path) == 0);
    return sf;
}

int main(void) {
    safetensors_file_t *sf = read_header(
        "{\"__metadata__\":{\"format\":\"pt\"},"
        "\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,8]}}");
    CHECK(sf && sf->num_tensors == 1);
    const safetensor_t *tensor = safetensors_find(sf, "weight");
    CHECK(tensor && tensor->dtype == DTYPE_F32 && tensor->ndim == 1 && tensor->shape[0] == 2);
    CHECK(tensor->data_offset == 0 && tensor->data_size == 8);
    const float *values = safetensors_data(sf, tensor);
    CHECK(values[0] == 1.25f && values[1] == -2.5f);
    CHECK(!safetensors_find(sf, "missing"));
    safetensors_close(sf);

    sf = read_header("{\"position_ids\":{\"dtype\":\"I64\",\"shape\":[1,1],\"data_offsets\":[0,8]}}");
    CHECK(sf && safetensors_find(sf, "position_ids")->dtype == DTYPE_UNKNOWN);
    safetensors_close(sf);
    safetensors_close(NULL);

    const char *invalid[] = {
        "{",
        "[]",
        "{\"weight\":null}",
        "{\"weight\":{\"shape\":[2],\"data_offsets\":[0,8]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":2,\"data_offsets\":[0,8]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[-1],\"data_offsets\":[0,8]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1.5],\"data_offsets\":[0,8]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[9223372036854775808],\"data_offsets\":[0,8]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1,1,1,1,1,1,1,1,1],\"data_offsets\":[0,8]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[-1,8]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[8,0]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,9]}}",
        "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,18446744073709551616]}}",
    };
    for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++)
        CHECK(!read_header(invalid[i]));
    puts("PASS: single-file tensor loading, metadata, lookup, and malformed header rejection.");
    return 0;
}