1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
#define _POSIX_C_SOURCE 200809L
#include "safetensors.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#define CHECK(condition) do { \
if (!(condition)) { \
fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
exit(1); \
} \
} while (0)
static safetensors_file_t *read_header(const char *header) {
char path[] = "build/safetensors-test-XXXXXX";
int fd = mkstemp(path);
CHECK(fd >= 0);
FILE *file = fdopen(fd, "wb");
CHECK(file);
size_t length = strlen(header);
uint64_t padded = (length + 7) & ~(uint64_t)7;
const float values[] = {1.25f, -2.5f};
CHECK(fwrite(&padded, sizeof(padded), 1, file) == 1);
CHECK(fwrite(header, 1, length, file) == length);
for (size_t i = length; i < padded; i++)
CHECK(fputc(' ', file) != EOF);
CHECK(fwrite(values, sizeof(values), 1, file) == 1);
CHECK(fclose(file) == 0);
safetensors_file_t *sf = safetensors_open(path);
CHECK(unlink(path) == 0);
return sf;
}
int main(void) {
safetensors_file_t *sf = read_header(
"{\"__metadata__\":{\"format\":\"pt\"},"
"\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,8]}}");
CHECK(sf && sf->num_tensors == 1);
const safetensor_t *tensor = safetensors_find(sf, "weight");
CHECK(tensor && tensor->dtype == DTYPE_F32 && tensor->ndim == 1 && tensor->shape[0] == 2);
CHECK(tensor->data_offset == 0 && tensor->data_size == 8);
const float *values = safetensors_data(sf, tensor);
CHECK(values[0] == 1.25f && values[1] == -2.5f);
CHECK(!safetensors_find(sf, "missing"));
safetensors_close(sf);
sf = read_header("{\"position_ids\":{\"dtype\":\"I64\",\"shape\":[1,1],\"data_offsets\":[0,8]}}");
CHECK(sf && safetensors_find(sf, "position_ids")->dtype == DTYPE_UNKNOWN);
safetensors_close(sf);
safetensors_close(NULL);
const char *invalid[] = {
"{",
"[]",
"{\"weight\":null}",
"{\"weight\":{\"shape\":[2],\"data_offsets\":[0,8]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":2,\"data_offsets\":[0,8]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[-1],\"data_offsets\":[0,8]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[1.5],\"data_offsets\":[0,8]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[9223372036854775808],\"data_offsets\":[0,8]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[1,1,1,1,1,1,1,1,1],\"data_offsets\":[0,8]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[-1,8]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[8,0]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,9]}}",
"{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,18446744073709551616]}}",
};
for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++)
CHECK(!read_header(invalid[i]));
puts("PASS: single-file tensor loading, metadata, lookup, and malformed header rejection.");
return 0;
}
|