#include "safetensors.h" #include #include #include #include #include #include #include static int parse_header(safetensors_file_t *sf) { yyjson_doc *doc = yyjson_read((char *)sf->data + 8, sf->header_size, 0); if (!doc) return -1; yyjson_val *root = yyjson_doc_get_root(doc); if (!yyjson_is_obj(root)) goto fail; size_t index, count; yyjson_val *key, *entry; yyjson_obj_foreach(root, index, count, key, entry) { if (yyjson_equals_str(key, "__metadata__")) continue; if (!yyjson_is_obj(entry) || sf->num_tensors >= SAFETENSORS_MAX_TENSORS) goto fail; safetensor_t *tensor = &sf->tensors[sf->num_tensors]; size_t name_len = yyjson_get_len(key); if (name_len >= sizeof(tensor->name)) goto fail; memcpy(tensor->name, yyjson_get_str(key), name_len + 1); yyjson_val *dtype = yyjson_obj_get(entry, "dtype"); yyjson_val *shape = yyjson_obj_get(entry, "shape"); yyjson_val *offsets = yyjson_obj_get(entry, "data_offsets"); if (!yyjson_is_str(dtype) || !yyjson_is_arr(shape) || yyjson_arr_size(shape) > sizeof(tensor->shape) / sizeof(tensor->shape[0]) || !yyjson_is_arr(offsets) || yyjson_arr_size(offsets) != 2) goto fail; tensor->dtype = yyjson_equals_str(dtype, "F32") ? DTYPE_F32 : DTYPE_UNKNOWN; tensor->ndim = (int)yyjson_arr_size(shape); for (int i = 0; i < tensor->ndim; i++) { yyjson_val *dim = yyjson_arr_get(shape, (size_t)i); if (!yyjson_is_uint(dim) || yyjson_get_uint(dim) > INT64_MAX) goto fail; tensor->shape[i] = (int64_t)yyjson_get_uint(dim); } yyjson_val *start = yyjson_arr_get(offsets, 0); yyjson_val *end = yyjson_arr_get(offsets, 1); if (!yyjson_is_uint(start) || !yyjson_is_uint(end)) goto fail; uint64_t first = yyjson_get_uint(start), last = yyjson_get_uint(end); if (first > last || last > sf->file_size - 8 - sf->header_size) goto fail; tensor->data_offset = (size_t)first; tensor->data_size = (size_t)(last - first); sf->num_tensors++; } yyjson_doc_free(doc); return 0; fail: yyjson_doc_free(doc); return -1; } safetensors_file_t *safetensors_open(const char *path) { int fd = open(path, O_RDONLY); if (fd < 0) return NULL; struct stat st; if (fstat(fd, &st) < 0 || st.st_size < 8) { close(fd); return NULL; } size_t file_size = (size_t)st.st_size; void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); if (data == MAP_FAILED) return NULL; uint64_t header_size = 0; memcpy(&header_size, data, 8); if (header_size > file_size - 8) { munmap(data, file_size); return NULL; } safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t)); if (!sf) { munmap(data, file_size); return NULL; } sf->data = data; sf->file_size = file_size; sf->header_size = (size_t)header_size; if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; } return sf; } void safetensors_close(safetensors_file_t *sf) { if (!sf) return; if (sf->data) munmap(sf->data, sf->file_size); free(sf); } const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name) { for (int i = 0; i < sf->num_tensors; i++) { if (strcmp(sf->tensors[i].name, name) == 0) return &sf->tensors[i]; } return NULL; } const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { return (const char *)sf->data + 8 + sf->header_size + t->data_offset; }