aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/safetensors.c
blob: 546f51d49affc7faa1ff2677c9d186fc9ecf1477 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#include "safetensors.h"

#include <yyjson.h>

#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>

static int parse_header(safetensors_file_t *sf) {
    yyjson_doc *doc = yyjson_read((char *)sf->data + 8, sf->header_size, 0);
    if (!doc)
        return -1;
    yyjson_val *root = yyjson_doc_get_root(doc);
    if (!yyjson_is_obj(root))
        goto fail;

    size_t index, count;
    yyjson_val *key, *entry;
    yyjson_obj_foreach(root, index, count, key, entry) {
        if (yyjson_equals_str(key, "__metadata__"))
            continue;
        if (!yyjson_is_obj(entry) || sf->num_tensors >= SAFETENSORS_MAX_TENSORS)
            goto fail;
        safetensor_t *tensor = &sf->tensors[sf->num_tensors];
        size_t name_len = yyjson_get_len(key);
        if (name_len >= sizeof(tensor->name))
            goto fail;
        memcpy(tensor->name, yyjson_get_str(key), name_len + 1);

        yyjson_val *dtype = yyjson_obj_get(entry, "dtype");
        yyjson_val *shape = yyjson_obj_get(entry, "shape");
        yyjson_val *offsets = yyjson_obj_get(entry, "data_offsets");
        if (!yyjson_is_str(dtype) || !yyjson_is_arr(shape) ||
            yyjson_arr_size(shape) > sizeof(tensor->shape) / sizeof(tensor->shape[0]) ||
            !yyjson_is_arr(offsets) || yyjson_arr_size(offsets) != 2)
            goto fail;
        tensor->dtype = yyjson_equals_str(dtype, "F32") ? DTYPE_F32 : DTYPE_UNKNOWN;
        tensor->ndim = (int)yyjson_arr_size(shape);
        for (int i = 0; i < tensor->ndim; i++) {
            yyjson_val *dim = yyjson_arr_get(shape, (size_t)i);
            if (!yyjson_is_uint(dim) || yyjson_get_uint(dim) > INT64_MAX)
                goto fail;
            tensor->shape[i] = (int64_t)yyjson_get_uint(dim);
        }
        yyjson_val *start = yyjson_arr_get(offsets, 0);
        yyjson_val *end = yyjson_arr_get(offsets, 1);
        if (!yyjson_is_uint(start) || !yyjson_is_uint(end))
            goto fail;
        uint64_t first = yyjson_get_uint(start), last = yyjson_get_uint(end);
        if (first > last || last > sf->file_size - 8 - sf->header_size)
            goto fail;
        tensor->data_offset = (size_t)first;
        tensor->data_size = (size_t)(last - first);
        sf->num_tensors++;
    }
    yyjson_doc_free(doc);
    return 0;

fail:
    yyjson_doc_free(doc);
    return -1;
}

safetensors_file_t *safetensors_open(const char *path) {
    int fd = open(path, O_RDONLY);
    if (fd < 0)
        return NULL;

    struct stat st;
    if (fstat(fd, &st) < 0 || st.st_size < 8) {
        close(fd);
        return NULL;
    }

    size_t file_size = (size_t)st.st_size;
    void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
    close(fd);
    if (data == MAP_FAILED)
        return NULL;

    uint64_t header_size = 0;
    memcpy(&header_size, data, 8);
    if (header_size > file_size - 8) {
        munmap(data, file_size);
        return NULL;
    }

    safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t));
    if (!sf) {
        munmap(data, file_size);
        return NULL;
    }

    sf->data = data;
    sf->file_size = file_size;
    sf->header_size = (size_t)header_size;
    if (parse_header(sf) != 0) {
        safetensors_close(sf);
        return NULL;
    }
    return sf;
}

void safetensors_close(safetensors_file_t *sf) {
    if (!sf)
        return;
    if (sf->data)
        munmap(sf->data, sf->file_size);
    free(sf);
}

const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name) {
    for (int i = 0; i < sf->num_tensors; i++) {
        if (strcmp(sf->tensors[i].name, name) == 0)
            return &sf->tensors[i];
    }
    return NULL;
}

const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
    return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
}