aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-22 01:03:17 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:21 +0200
commit49663acfb43a08b7c21e5414fe0ca2a80027deec (patch)
treec9192d60257ab162ebef4b86f49ba2c19046f9e0
parent6d4f018051c3681588c36e88fed0ab48876f9a80 (diff)
downloadffwd-49663acfb43a08b7c21e5414fe0ca2a80027deec.tar.gz
ffwd-49663acfb43a08b7c21e5414fe0ca2a80027deec.zip
safetensors: harden parser limits
-rw-r--r--qwen_asr_safetensors.c122
1 files changed, 99 insertions, 23 deletions
diff --git a/qwen_asr_safetensors.c b/qwen_asr_safetensors.c
index 7afe899..86136f7 100644
--- a/qwen_asr_safetensors.c
+++ b/qwen_asr_safetensors.c
@@ -40,16 +40,18 @@ static int parse_string(const char **p, char *out, size_t max_len) {
return 0;
}
-static int64_t parse_int(const char **p) {
+static int parse_int(const char **p, int64_t *out) {
skip_whitespace(p);
int64_t val = 0;
int neg = 0;
if (**p == '-') { neg = 1; (*p)++; }
+ if (**p < '0' || **p > '9') return -1;
while (**p >= '0' && **p <= '9') {
val = val * 10 + (**p - '0');
(*p)++;
}
- return neg ? -val : val;
+ *out = neg ? -val : val;
+ return 0;
}
static safetensor_dtype_t parse_dtype(const char *s) {
@@ -91,25 +93,33 @@ static int parse_tensor_entry(const char **p, safetensor_t *t) {
if (**p != '[') return -1;
(*p)++;
t->ndim = 0;
- while (**p && **p != ']' && t->ndim < 8) {
+ while (**p && **p != ']') {
skip_whitespace(p);
if (**p == ',') { (*p)++; continue; }
- t->shape[t->ndim++] = parse_int(p);
+ if (t->ndim >= 8) return -1;
+ if (parse_int(p, &t->shape[t->ndim]) != 0) return -1;
+ t->ndim++;
}
- if (**p == ']') (*p)++;
+ if (**p != ']') return -1;
+ (*p)++;
} else if (strcmp(key, "data_offsets") == 0) {
if (**p != '[') return -1;
(*p)++;
skip_whitespace(p);
- size_t start = (size_t)parse_int(p);
+ int64_t start_i;
+ if (parse_int(p, &start_i) != 0) return -1;
skip_whitespace(p);
- if (**p == ',') (*p)++;
+ if (**p != ',') return -1;
+ (*p)++;
skip_whitespace(p);
- size_t end = (size_t)parse_int(p);
- t->data_offset = start;
- t->data_size = end - start;
+ int64_t end_i;
+ if (parse_int(p, &end_i) != 0) return -1;
+ if (start_i < 0 || end_i < start_i) return -1;
+ t->data_offset = (size_t)start_i;
+ t->data_size = (size_t)(end_i - start_i);
skip_whitespace(p);
- if (**p == ']') (*p)++;
+ if (**p != ']') return -1;
+ (*p)++;
} else {
/* Skip unknown value */
if (**p == '"') {
@@ -138,7 +148,8 @@ static int parse_tensor_entry(const char **p, safetensor_t *t) {
}
}
}
- if (**p == '}') (*p)++;
+ if (**p != '}') return -1;
+ (*p)++;
return 0;
}
@@ -150,7 +161,7 @@ static int parse_header(safetensors_file_t *sf) {
sf->num_tensors = 0;
- while (*p && *p != '}' && sf->num_tensors < SAFETENSORS_MAX_TENSORS) {
+ while (*p && *p != '}') {
skip_whitespace(&p);
if (*p == ',') { p++; continue; }
if (*p == '}') break;
@@ -174,11 +185,18 @@ static int parse_header(safetensors_file_t *sf) {
continue;
}
+ if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) {
+ fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n",
+ sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS);
+ return -1;
+ }
+
safetensor_t *t = &sf->tensors[sf->num_tensors];
snprintf(t->name, sizeof(t->name), "%s", name);
if (parse_tensor_entry(&p, t) != 0) return -1;
sf->num_tensors++;
}
+ if (*p != '}') return -1;
return 0;
}
@@ -301,6 +319,23 @@ void safetensors_print_all(const safetensors_file_t *sf) {
* Multi-shard operations
* ======================================================================== */
+static int cmp_shard_names(const void *a, const void *b) {
+ const char *sa = (const char *)a;
+ const char *sb = (const char *)b;
+ return strcmp(sa, sb);
+}
+
+static int parse_shard_name(const char *name, int *idx, int *total) {
+ int i = 0, n = 0, consumed = 0;
+ if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2)
+ return -1;
+ if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n)
+ return -1;
+ *idx = i;
+ *total = n;
+ return 0;
+}
+
multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
if (!ms) return NULL;
@@ -316,25 +351,52 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
return ms;
}
- /* Try multi-shard: model-00001-of-NNNNN.safetensors */
- for (int i = 1; i <= SAFETENSORS_MAX_SHARDS; i++) {
- snprintf(path, sizeof(path), "%s/model-%05d-of-%05d.safetensors",
- model_dir, i, i); /* placeholder - need to detect count */
- /* We don't know the total count yet, try common patterns */
- break;
- }
-
/* Scan directory for shard files */
DIR *dir = opendir(model_dir);
if (!dir) { free(ms); return NULL; }
struct dirent *entry;
char shard_names[SAFETENSORS_MAX_SHARDS][256];
+ int seen[SAFETENSORS_MAX_SHARDS] = {0};
int n_shards = 0;
+ int expected_total = 0;
- while ((entry = readdir(dir)) != NULL && n_shards < SAFETENSORS_MAX_SHARDS) {
+ while ((entry = readdir(dir)) != NULL) {
if (strncmp(entry->d_name, "model-", 6) == 0 &&
strstr(entry->d_name, ".safetensors") != NULL) {
+ int shard_idx = 0, shard_total = 0;
+ if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) {
+ fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n",
+ entry->d_name);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ if (shard_total > SAFETENSORS_MAX_SHARDS) {
+ fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n",
+ shard_total, SAFETENSORS_MAX_SHARDS);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ if (expected_total == 0) {
+ expected_total = shard_total;
+ } else if (expected_total != shard_total) {
+ fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n",
+ entry->d_name);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ if (seen[shard_idx - 1]) {
+ fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n",
+ shard_idx);
+ closedir(dir);
+ free(ms);
+ return NULL;
+ }
+ seen[shard_idx - 1] = 1;
+
snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]),
"%s", entry->d_name);
n_shards++;
@@ -347,9 +409,23 @@ multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
free(ms);
return NULL;
}
+ if (expected_total != n_shards) {
+ fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n",
+ expected_total, n_shards, model_dir);
+ free(ms);
+ return NULL;
+ }
+ for (int i = 0; i < expected_total; i++) {
+ if (!seen[i]) {
+ fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n",
+ i + 1, expected_total, model_dir);
+ free(ms);
+ return NULL;
+ }
+ }
/* Sort shard names to ensure consistent ordering */
- qsort(shard_names, n_shards, 256, (int(*)(const void*,const void*))strcmp);
+ qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names);
/* Open each shard */
for (int i = 0; i < n_shards; i++) {