1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
#include "safetensors.h"
#include <yyjson.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
static int parse_header(safetensors_file_t *sf) {
yyjson_doc *doc = yyjson_read((char *)sf->data + 8, sf->header_size, 0);
if (!doc)
return -1;
yyjson_val *root = yyjson_doc_get_root(doc);
if (!yyjson_is_obj(root))
goto fail;
size_t index, count;
yyjson_val *key, *entry;
yyjson_obj_foreach(root, index, count, key, entry) {
if (yyjson_equals_str(key, "__metadata__"))
continue;
if (!yyjson_is_obj(entry) || sf->num_tensors >= SAFETENSORS_MAX_TENSORS)
goto fail;
safetensor_t *tensor = &sf->tensors[sf->num_tensors];
size_t name_len = yyjson_get_len(key);
if (name_len >= sizeof(tensor->name))
goto fail;
memcpy(tensor->name, yyjson_get_str(key), name_len + 1);
yyjson_val *dtype = yyjson_obj_get(entry, "dtype");
yyjson_val *shape = yyjson_obj_get(entry, "shape");
yyjson_val *offsets = yyjson_obj_get(entry, "data_offsets");
if (!yyjson_is_str(dtype) || !yyjson_is_arr(shape) ||
yyjson_arr_size(shape) > sizeof(tensor->shape) / sizeof(tensor->shape[0]) ||
!yyjson_is_arr(offsets) || yyjson_arr_size(offsets) != 2)
goto fail;
tensor->dtype = yyjson_equals_str(dtype, "F32") ? DTYPE_F32 : DTYPE_UNKNOWN;
tensor->ndim = (int)yyjson_arr_size(shape);
for (int i = 0; i < tensor->ndim; i++) {
yyjson_val *dim = yyjson_arr_get(shape, (size_t)i);
if (!yyjson_is_uint(dim) || yyjson_get_uint(dim) > INT64_MAX)
goto fail;
tensor->shape[i] = (int64_t)yyjson_get_uint(dim);
}
yyjson_val *start = yyjson_arr_get(offsets, 0);
yyjson_val *end = yyjson_arr_get(offsets, 1);
if (!yyjson_is_uint(start) || !yyjson_is_uint(end))
goto fail;
uint64_t first = yyjson_get_uint(start), last = yyjson_get_uint(end);
if (first > last || last > sf->file_size - 8 - sf->header_size)
goto fail;
tensor->data_offset = (size_t)first;
tensor->data_size = (size_t)(last - first);
sf->num_tensors++;
}
yyjson_doc_free(doc);
return 0;
fail:
yyjson_doc_free(doc);
return -1;
}
safetensors_file_t *safetensors_open(const char *path) {
int fd = open(path, O_RDONLY);
if (fd < 0)
return NULL;
struct stat st;
if (fstat(fd, &st) < 0 || st.st_size < 8) {
close(fd);
return NULL;
}
size_t file_size = (size_t)st.st_size;
void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
if (data == MAP_FAILED)
return NULL;
uint64_t header_size = 0;
memcpy(&header_size, data, 8);
if (header_size > file_size - 8) {
munmap(data, file_size);
return NULL;
}
safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t));
if (!sf) {
munmap(data, file_size);
return NULL;
}
sf->data = data;
sf->file_size = file_size;
sf->header_size = (size_t)header_size;
if (parse_header(sf) != 0) {
safetensors_close(sf);
return NULL;
}
return sf;
}
void safetensors_close(safetensors_file_t *sf) {
if (!sf)
return;
if (sf->data)
munmap(sf->data, sf->file_size);
free(sf);
}
const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name) {
for (int i = 0; i < sf->num_tensors; i++) {
if (strcmp(sf->tensors[i].name, name) == 0)
return &sf->tensors[i];
}
return NULL;
}
const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
}
|